diff --git a/.circleci/config.yml b/.circleci/config.yml index 92928eb86e..1460364639 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,124 +1,131 @@ -version: 2.0 +# These environment variables must be set in CircleCI UI +# +# DOCKERHUB_REPO - docker hub repo, format: / +# DOCKER_USER +# DOCKER_PASS +# + +version: 2 jobs: - unit-tests: - environment: - COMPOSE_FILE: .circleci/docker-compose.circle.yml - COMPOSE_PROJECT_NAME: redash + build: docker: - - image: circleci/buildpack-deps:xenial + - image: docker:18.02.0-ce + working_directory: ~/mozilla/redash steps: - - setup_remote_docker - checkout + - setup_remote_docker - run: - name: Build Docker Images - command: | - set -x - docker-compose up -d - sleep 10 - - run: - name: Create Test Database - command: docker-compose run --rm postgres psql -h postgres -U postgres -c "create database tests;" - - run: - name: Run Tests - command: docker-compose run --name tests redash tests --junitxml=junit.xml tests/ - - run: - name: Copy Test Results - command: | - mkdir -p /tmp/test-results/unit-tests - docker cp tests:/app/coverage.xml ./coverage.xml - docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml - - store_test_results: - path: /tmp/test-results - - store_artifacts: - path: coverage.xml - build-tarball: + command: docker build -t app:build . + no_output_timeout: 20m + + test: docker: - - image: circleci/node:8 + - image: circleci/node:6.14.3-stretch + - image: redis + - image: circleci/postgres:10-alpine-postgis + working_directory: ~/mozilla/redash steps: - checkout - - run: npm install - - run: npm run build - - run: .circleci/update_version - - run: .circleci/pack + - run: mkdir -p /tmp/test-reports/pytest + - run: sudo apt-get update + - run: sudo apt-get install -y python-pip python-dev + - run: sudo apt-get install -y redis-tools redis-server + - run: sudo pip install --upgrade setuptools + - run: sudo pip install -r requirements_dev.txt + - run: sudo pip install -r requirements.txt + - run: sudo npm install + - run: sudo npm run bundle + - run: sudo npm run build + - run: + command: pytest --junitxml=/tmp/test-reports/pytest/junit.xml tests/ + environment: + REDASH_REDIS_URL: redis://localhost:6379/0 + REDASH_DATABASE_URL: "postgresql://postgres@localhost/postgres" - store_artifacts: - path: /tmp/artifacts/ - build-docker-image: - docker: - - image: circleci/buildpack-deps:xenial - steps: - - setup_remote_docker - - checkout - - run: .circleci/update_version - - run: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: docker build -t redash/redash:$(.circleci/docker_tag) . - - run: docker push redash/redash:$(.circleci/docker_tag) - integration-tests: - working_directory: ~/redash - machine: true - environment: - REDASH_SERVER_URL : "http://127.0.0.1:5000/" - DOCKER_IMAGE: mozilla/redash-ui-tests + path: /tmp/test-reports/ + destination: tr1 + - store_test_results: + path: /tmp/test-reports/ + + deploy-master: + machine: + enable: true + working_directory: ~/mozilla/redash steps: - checkout - run: - name: Install Docker Compose - command: | - set -x - pip install --upgrade pip - pip install docker-compose>=1.18 - docker-compose --version - - run: - name: Pull redash images + name: Deploy to Dockerhub + no_output_timeout: 20m command: | - set -x - docker-compose -f docker-compose.yml up --no-start - sleep 10 - - run: - name: Pull redash-ui-tests - command: docker pull "${DOCKER_IMAGE}":latest + ./bin/deploy "master" + + deploy-rc: + machine: + enable: true + working_directory: ~/mozilla/redash + steps: + - checkout - run: - name: Setup redash instance + name: Deploy to Dockerhub + no_output_timeout: 20m command: | - set -x - docker-compose run --rm --user root server create_db - docker-compose run --rm postgres psql -h postgres -U postgres -c "create database tests" - docker-compose run --rm --user root server /app/manage.py users create_root root@example.com "rootuser" --password "IAMROOT" --org default - docker-compose run --rm --user root server /app/manage.py ds new "ui-tests" --type "url" --options '{"title": "uitests"}' - docker-compose run -d -p 5000:5000 --user root server - docker-compose start postgres + ./bin/deploy "rc" + + deploy-milestone: + machine: + enable: true + working_directory: ~/mozilla/redash + steps: + - checkout - run: - name: Run tests + name: Deploy milestone to Dockerhub + no_output_timeout: 20m command: | - set -x - docker run --net="host" --env REDASH_SERVER_URL="${REDASH_SERVER_URL}" "${DOCKER_IMAGE}" - - store_artifacts: - path: report.html + ./bin/deploy "$CIRCLE_TAG" + ./bin/alias "$CIRCLE_TAG" "latest" + + workflows: version: 2 - integration_tests: + build-test-deploy: jobs: - - integration-tests: + - build: filters: - branches: - only: master - build: - jobs: - - unit-tests - - build-tarball: - requires: - - unit-tests - filters: - tags: - only: /v[0-9]+(\.[0-9\-a-z]+)*/ - branches: - only: - - master - - release - - build-docker-image: - requires: - - unit-tests - filters: - branches: - ignore: /.*/ - tags: - only: /v[0-9]+(\.[0-9\-a-z]+)*/ \ No newline at end of file + tags: + only: /.*/ + branches: + ignore: + - gh-pages + + - test: + filters: + tags: + only: /.*/ + branches: + ignore: + - gh-pages + + - deploy-master: + requires: + - test + filters: + branches: + only: + - master + + - deploy-rc: + requires: + - test + filters: + branches: + only: + - release + + - deploy-milestone: + requires: + - test + filters: + tags: + only: /^m[0-9]+(\.[0-9]+)?$/ + branches: + ignore: /.*/ diff --git a/.pyup.yml b/.pyup.yml new file mode 100644 index 0000000000..af68611aed --- /dev/null +++ b/.pyup.yml @@ -0,0 +1,6 @@ +schedule: "every day" +search: False +update: insecure +requirements: + - requirements.txt: + update: insecure diff --git a/Dockerfile b/Dockerfile index e289bbde18..cbb333eba2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,16 @@ COPY requirements.txt requirements_dev.txt requirements_all_ds.txt ./ RUN pip install -r requirements.txt -r requirements_dev.txt -r requirements_all_ds.txt COPY . ./ -RUN npm install && npm run build && rm -rf node_modules + +# Upgrade node to LTS 6.11.2 +RUN cd ~ +RUN wget https://nodejs.org/download/release/v6.11.2/node-v6.11.2-linux-x64.tar.gz +RUN sudo tar --strip-components 1 -xzvf node-v* -C /usr/local + +# Upgrade npm +RUN npm upgrade npm + +RUN npm install && npm run bundle && npm run build && rm -rf node_modules RUN chown -R redash /app USER redash diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..c776130b5a --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +.PHONY: build bundle compose_build create_database tests test_db clean + +compose_build: + docker-compose build + +test_db: + docker-compose run --rm postgres psql -h postgres -U postgres -c "create database tests" + +create_database: + docker-compose run server create_db + +clean: + docker ps -a -q | xargs docker kill;docker ps -a -q | xargs docker rm + +bundle: + docker-compose run server bin/bundle-extensions + +tests: + docker-compose run server tests + +build: bundle + npm run build diff --git a/bin/alias b/bin/alias new file mode 100755 index 0000000000..1d4a32a19b --- /dev/null +++ b/bin/alias @@ -0,0 +1,12 @@ +#!/bin/bash + +set -eo pipefail + +[ ! -z $DOCKERHUB_REPO ] && [ $# -eq 2 ] + +VERSION="$1" +ALIAS="$2" + +docker login -u $DOCKER_USER -p $DOCKER_PASS +docker tag $DOCKERHUB_REPO:$VERSION $DOCKERHUB_REPO:$ALIAS +docker push $DOCKERHUB_REPO:$ALIAS diff --git a/bin/bundle-extensions b/bin/bundle-extensions new file mode 100755 index 0000000000..8416aab776 --- /dev/null +++ b/bin/bundle-extensions @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +import os +from subprocess import call +from distutils.dir_util import copy_tree + +from pkg_resources import iter_entry_points, resource_filename, resource_isdir + + + +# Make a directory for extensions and set it as an environment variable +# to be picked up by webpack. +EXTENSIONS_RELATIVE_PATH = os.path.join('client', 'app', 'extensions') +EXTENSIONS_DIRECTORY = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + EXTENSIONS_RELATIVE_PATH) + +if not os.path.exists(EXTENSIONS_DIRECTORY): + os.makedirs(EXTENSIONS_DIRECTORY) +os.environ["EXTENSIONS_DIRECTORY"] = EXTENSIONS_RELATIVE_PATH + +for entry_point in iter_entry_points('redash.extensions'): + # This is where the frontend code for an extension lives + # inside of its package. + content_folder_relative = os.path.join( + entry_point.name, 'bundle') + (root_module, _) = os.path.splitext(entry_point.module_name) + + if not resource_isdir(root_module, content_folder_relative): + continue + + content_folder = resource_filename(root_module, content_folder_relative) + + # This is where we place our extensions folder. + destination = os.path.join( + EXTENSIONS_DIRECTORY, + entry_point.name) + + copy_tree(content_folder, destination) diff --git a/bin/deploy b/bin/deploy new file mode 100755 index 0000000000..2bdb54ed4d --- /dev/null +++ b/bin/deploy @@ -0,0 +1,19 @@ +#!/bin/bash + +set -eo pipefail + +[ ! -z $DOCKERHUB_REPO ] && [ $# -eq 1 ] + +VERSION="$1" + +printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ + "$CIRCLE_SHA1" \ + "$VERSION" \ + "$CIRCLE_PROJECT_USERNAME" \ + "$CIRCLE_PROJECT_REPONAME" \ + "$CIRCLE_BUILD_URL" \ +> version.json + +docker login -u $DOCKER_USER -p $DOCKER_PASS +docker build -t $DOCKERHUB_REPO:$VERSION . +docker push $DOCKERHUB_REPO:$VERSION diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint index 0d45eb5482..a91be66fc8 100755 --- a/bin/docker-entrypoint +++ b/bin/docker-entrypoint @@ -2,23 +2,32 @@ set -e worker() { + /app/manage.py db upgrade WORKERS_COUNT=${WORKERS_COUNT:-2} QUEUES=${QUEUES:-queries,scheduled_queries,celery} + MAX_MEMORY=$(($(/usr/bin/awk '/MemTotal/ {print $2}' /proc/meminfo)/4)) echo "Starting $WORKERS_COUNT workers for queues: $QUEUES..." - exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair + exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo \ + --max-tasks-per-child=10 \ + --max-memory-per-child=$MAX_MEMORY \ + -Ofair } scheduler() { + /app/manage.py db upgrade WORKERS_COUNT=${WORKERS_COUNT:-1} QUEUES=${QUEUES:-celery} echo "Starting scheduler and $WORKERS_COUNT workers for queues: $QUEUES..." - exec /usr/local/bin/celery worker --app=redash.worker --beat -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair + exec /usr/local/bin/celery worker --app=redash.worker --beat -c$WORKERS_COUNT -Q$QUEUES -linfo \ + --max-tasks-per-child=10 \ + -Ofair } server() { + /app/manage.py db upgrade exec /usr/local/bin/gunicorn -b 0.0.0.0:5000 --name redash -w${REDASH_WEB_WORKERS:-4} redash.wsgi:app } diff --git a/client/app/assets/less/redash/query.less b/client/app/assets/less/redash/query.less index 30231788a3..c689f05b05 100644 --- a/client/app/assets/less/redash/query.less +++ b/client/app/assets/less/redash/query.less @@ -458,6 +458,7 @@ a.label-tag { .datasource-small { visibility: hidden; + display: none !important; } .query-fullscreen .query-metadata__mobile { @@ -576,6 +577,7 @@ nav .rg-bottom { .datasource-small { visibility: visible; + display: inline-block !important; } .query-fullscreen { diff --git a/client/app/components/dashboards/widget.js b/client/app/components/dashboards/widget.js index ecd061464b..50eda3d21e 100644 --- a/client/app/components/dashboards/widget.js +++ b/client/app/components/dashboards/widget.js @@ -66,8 +66,6 @@ function DashboardWidgetCtrl($location, $uibModal, $window, Events, currentUser) return; } - Events.record('delete', 'widget', this.widget.id); - this.widget.delete().then(() => { if (this.deleted) { this.deleted({}); diff --git a/client/app/components/parameters.html b/client/app/components/parameters.html index 9aa5a76c87..ac8bbe7aac 100644 --- a/client/app/components/parameters.html +++ b/client/app/components/parameters.html @@ -1,5 +1,5 @@
diff --git a/client/app/components/parameters.js b/client/app/components/parameters.js index 56e3c7fdfe..2975d3aa30 100644 --- a/client/app/components/parameters.js +++ b/client/app/components/parameters.js @@ -152,6 +152,7 @@ function ParametersDirective($location, $uibModal) { }, }); }; + scope.hideParameters = $location.search().hideParameters; }, }; } diff --git a/client/app/components/queries/query-editor.js b/client/app/components/queries/query-editor.js index 3c7d9f6fe4..f0555be12a 100644 --- a/client/app/components/queries/query-editor.js +++ b/client/app/components/queries/query-editor.js @@ -93,18 +93,21 @@ function queryEditor(QuerySnippet, $timeout) { editor.getSession().setMode(newMode); }); - $scope.$watch('schema', (newSchema, oldSchema) => { + $scope.$watch('autoCompleteSchema', (newSchema, oldSchema) => { if (newSchema !== oldSchema) { if (newSchema === undefined) { return; } - const tokensCount = newSchema.reduce((totalLength, table) => totalLength + table.columns.length, 0); - // If there are too many tokens we disable live autocomplete, - // as it makes typing slower. - if (tokensCount > 5000) { + const tokensCount = + newSchema.reduce((totalLength, table) => totalLength + table.columns.length, 0); + // If there are too many tokens or if it's requested via the UI + // we disable live autocomplete, as it makes typing slower. + if (tokensCount > 5000 || !$scope.$parent.autocompleteQuery) { editor.setOption('enableLiveAutocompletion', false); + editor.setOption('enableBasicAutocompletion', false); } else { editor.setOption('enableLiveAutocompletion', true); + editor.setOption('enableBasicAutocompletion', true); } } }); @@ -112,6 +115,10 @@ function queryEditor(QuerySnippet, $timeout) { $scope.$parent.$on('angular-resizable.resizing', () => { editor.resize(); }); + $scope.$parent.$watch('autocompleteQuery', () => { + editor.setOption('enableLiveAutocompletion', $scope.$parent.autocompleteQuery); + editor.setOption('enableBasicAutocompletion', $scope.$parent.autocompleteQuery); + }); editor.focus(); }, @@ -119,31 +126,44 @@ function queryEditor(QuerySnippet, $timeout) { const schemaCompleter = { getCompletions(state, session, pos, prefix, callback) { - if (prefix.length === 0 || !$scope.schema) { + // make a variable for the auto completion in the query editor + $scope.autoCompleteSchema = $scope.schema; // removeExtraSchemaInfo( + + if (prefix.length === 0 || !$scope.autoCompleteSchema) { callback(null, []); return; } - if (!$scope.schema.keywords) { + if (!$scope.autoCompleteSchema.keywords) { const keywords = {}; - $scope.schema.forEach((table) => { + $scope.autoCompleteSchema.forEach((table) => { keywords[table.name] = 'Table'; - table.columns.forEach((c) => { - keywords[c] = 'Column'; + table.columns.forEach((c) => { // autoCompleteColumns + if (c.charAt(c.length - 1) === ')') { + let parensStartAt = c.indexOf('(') - 1; + c = c.substring(0, parensStartAt); + parensStartAt = 1; // linter complains without this line + } + // remove '[P] ' for partition keys + if (c.charAt(0) === '[') { + c = c.substring(4, c.length); + } + // keywords[c] = 'Column'; // dups columns keywords[`${table.name}.${c}`] = 'Column'; }); }); - $scope.schema.keywords = map(keywords, (v, k) => ({ - name: k, - value: k, - score: 0, - meta: v, - })); + $scope.autoCompleteSchema.keywords = map(keywords, (v, k) => + ({ + name: k, + value: k, + score: 0, + meta: v, + })); } - callback(null, $scope.schema.keywords); + callback(null, $scope.autoCompleteSchema.keywords); }, }; diff --git a/client/app/components/queries/schedule-dialog.html b/client/app/components/queries/schedule-dialog.html index 8f1ab21541..aca492cdfe 100644 --- a/client/app/components/queries/schedule-dialog.html +++ b/client/app/components/queries/schedule-dialog.html @@ -15,4 +15,11 @@

Refresh Schedule

+ + diff --git a/client/app/components/queries/schedule-dialog.js b/client/app/components/queries/schedule-dialog.js index 1fc60c3925..41c29e031c 100644 --- a/client/app/components/queries/schedule-dialog.js +++ b/client/app/components/queries/schedule-dialog.js @@ -103,11 +103,32 @@ function queryRefreshSelect(clientConfig, Policy) { }; } +function scheduleUntil() { + return { + restrict: 'E', + scope: { + query: '=', + saveQuery: '=', + }, + template: '', + }; +} + +function scheduleKeepResults() { + return { + restrict: 'E', + scope: { + query: '=', + saveQuery: '=', + }, + template: '', + }; +} + const ScheduleForm = { controller() { this.query = this.resolve.query; this.saveQuery = this.resolve.saveQuery; - if (this.query.hasDailySchedule()) { this.refreshType = 'daily'; } else { @@ -125,5 +146,7 @@ const ScheduleForm = { export default function init(ngModule) { ngModule.directive('queryTimePicker', queryTimePicker); ngModule.directive('queryRefreshSelect', queryRefreshSelect); + ngModule.directive('scheduleUntil', scheduleUntil); + ngModule.directive('scheduleKeepResults', scheduleKeepResults); ngModule.component('scheduleDialog', ScheduleForm); } diff --git a/client/app/components/queries/schema-browser.html b/client/app/components/queries/schema-browser.html index a98e27aec6..ed5c9c1d15 100644 --- a/client/app/components/queries/schema-browser.html +++ b/client/app/components/queries/schema-browser.html @@ -6,10 +6,20 @@ ng-click="$ctrl.onRefresh()"> + +
-
+
diff --git a/client/app/components/queries/schema-browser.js b/client/app/components/queries/schema-browser.js index 499852be75..9c072051d7 100644 --- a/client/app/components/queries/schema-browser.js +++ b/client/app/components/queries/schema-browser.js @@ -3,6 +3,9 @@ import template from './schema-browser.html'; function SchemaBrowserCtrl($rootScope, $scope) { 'ngInject'; + this.versionToggle = false; + this.versionFilter = 'abcdefghijklmnop'; + this.showTable = (table) => { table.collapsed = !table.collapsed; $scope.$broadcast('vsRepeatTrigger'); @@ -21,6 +24,15 @@ function SchemaBrowserCtrl($rootScope, $scope) { this.isEmpty = function isEmpty() { return this.schema === undefined || this.schema.length === 0; }; + this.flipToggleVersionedTables = (versionToggle, toggleString) => { + if (versionToggle === false) { + this.versionToggle = true; + this.versionFilter = toggleString; + } else { + this.versionToggle = false; + this.versionFilter = 'abcdefghijklmnop'; + } + }; this.itemSelected = ($event, hierarchy) => { $rootScope.$broadcast('query-editor.command', 'paste', hierarchy.join('.')); @@ -32,7 +44,9 @@ function SchemaBrowserCtrl($rootScope, $scope) { const SchemaBrowser = { bindings: { schema: '<', + tabletogglestring: '<', onRefresh: '&', + flipToggleVersionedTables: '&', }, controller: SchemaBrowserCtrl, template, diff --git a/client/app/config/index.js b/client/app/config/index.js index aecdaf14b0..d51affad70 100644 --- a/client/app/config/index.js +++ b/client/app/config/index.js @@ -82,6 +82,11 @@ function registerComponents() { registerAll(context); } +function registerExtensions() { + const context = require.context('%', true, /^((?![\\/]test[\\/]).)*\.js$/); + registerAll(context); +} + function registerServices() { const context = require.context('@/services', true, /^((?![\\/]test[\\/]).)*\.js$/); registerAll(context); @@ -142,6 +147,7 @@ markdownFilter(ngModule); dateTimeFilter(ngModule); registerComponents(); registerPages(); +registerExtensions(); registerVisualizations(ngModule); export default ngModule; diff --git a/client/app/pages/admin/outdated-queries/index.js b/client/app/pages/admin/outdated-queries/index.js index aa5a54ad5d..e1dfd1e280 100644 --- a/client/app/pages/admin/outdated-queries/index.js +++ b/client/app/pages/admin/outdated-queries/index.js @@ -3,8 +3,7 @@ import moment from 'moment'; import { Paginator } from '@/lib/pagination'; import template from './outdated-queries.html'; -function OutdatedQueriesCtrl($scope, Events, $http, $timeout) { - Events.record('view', 'page', 'admin/outdated_queries'); +function OutdatedQueriesCtrl($scope, $http, $timeout) { $scope.autoUpdate = true; this.queries = new Paginator([], { itemsPerPage: 50 }); diff --git a/client/app/pages/admin/tasks/index.js b/client/app/pages/admin/tasks/index.js index 53d9007ea9..bceb11e53c 100644 --- a/client/app/pages/admin/tasks/index.js +++ b/client/app/pages/admin/tasks/index.js @@ -3,8 +3,7 @@ import moment from 'moment'; import { Paginator } from '@/lib/pagination'; import template from './tasks.html'; -function TasksCtrl($scope, $location, $http, $timeout, Events) { - Events.record('view', 'page', 'admin/tasks'); +function TasksCtrl($scope, $location, $http, $timeout) { $scope.autoUpdate = true; $scope.selectedTab = 'in_progress'; diff --git a/client/app/pages/alert/index.js b/client/app/pages/alert/index.js index f33f791574..e7f5f008e6 100644 --- a/client/app/pages/alert/index.js +++ b/client/app/pages/alert/index.js @@ -6,8 +6,6 @@ function AlertCtrl($routeParams, $location, $sce, toastr, currentUser, Query, Ev if (this.alertId === 'new') { Events.record('view', 'page', 'alerts/new'); - } else { - Events.record('view', 'alert', this.alertId); } this.trustAsHtml = html => $sce.trustAsHtml(html); diff --git a/client/app/pages/alerts-list/index.js b/client/app/pages/alerts-list/index.js index 25cf0689a0..19869bcc5b 100644 --- a/client/app/pages/alerts-list/index.js +++ b/client/app/pages/alerts-list/index.js @@ -8,9 +8,7 @@ const stateClass = { }; class AlertsListCtrl { - constructor(Events, Alert) { - Events.record('view', 'page', 'alerts'); - + constructor(Alert) { this.showEmptyState = false; this.showList = false; diff --git a/client/app/pages/dashboards/dashboard.js b/client/app/pages/dashboards/dashboard.js index 875c22a5cf..8035b2a41f 100644 --- a/client/app/pages/dashboards/dashboard.js +++ b/client/app/pages/dashboards/dashboard.js @@ -179,7 +179,6 @@ function DashboardCtrl( (dashboard) => { this.dashboard = dashboard; this.isDashboardOwner = currentUser.id === dashboard.user.id || currentUser.hasPermission('admin'); - Events.record('view', 'dashboard', dashboard.id); renderDashboard(dashboard, force); if ($location.search().edit === true) { @@ -229,7 +228,6 @@ function DashboardCtrl( this.archiveDashboard = () => { const archive = () => { - Events.record('archive', 'dashboard', this.dashboard.id); this.dashboard.$delete(); }; diff --git a/client/app/pages/data-sources/list.html b/client/app/pages/data-sources/list.html index 56af90e071..fd23dfc516 100644 --- a/client/app/pages/data-sources/list.html +++ b/client/app/pages/data-sources/list.html @@ -9,7 +9,7 @@
{{dataSource.name}} -

{{dataSource.name}}

+

{{dataSource.name}}

- {{dataSource.type_name}}
diff --git a/client/app/pages/data-sources/list.js b/client/app/pages/data-sources/list.js index 12a6b107c3..7ae3bff7aa 100644 --- a/client/app/pages/data-sources/list.js +++ b/client/app/pages/data-sources/list.js @@ -1,9 +1,7 @@ import settingsMenu from '@/lib/settings-menu'; import template from './list.html'; -function DataSourcesCtrl(Policy, Events, DataSource) { - Events.record('view', 'page', 'admin/data_sources'); - +function DataSourcesCtrl(Policy, DataSource) { this.policy = Policy; this.dataSources = DataSource.query(); } diff --git a/client/app/pages/data-sources/show.js b/client/app/pages/data-sources/show.js index 435f41ab90..ab2c461ba0 100644 --- a/client/app/pages/data-sources/show.js +++ b/client/app/pages/data-sources/show.js @@ -6,9 +6,8 @@ const logger = debug('redash:http'); function DataSourceCtrl( $scope, $route, $routeParams, $http, $location, toastr, - currentUser, AlertDialog, Events, DataSource, + currentUser, AlertDialog, DataSource, ) { - Events.record('view', 'page', 'admin/data_source'); $scope.dataSource = $route.current.locals.dataSource; $scope.dataSourceId = $routeParams.dataSourceId; @@ -45,8 +44,6 @@ function DataSourceCtrl( function deleteDataSource(callback) { const doDelete = () => { - Events.record('delete', 'datasource', $scope.dataSource.id); - $scope.dataSource.$delete(() => { toastr.success('Data source deleted successfully.'); $location.path('/data_sources/'); @@ -64,8 +61,6 @@ function DataSourceCtrl( } function testConnection(callback) { - Events.record('test', 'datasource', $scope.dataSource.id); - DataSource.test({ id: $scope.dataSource.id }, (httpResponse) => { if (httpResponse.ok) { toastr.success('Success'); diff --git a/client/app/pages/destinations/list.js b/client/app/pages/destinations/list.js index 5e96eb2be7..84a87327d3 100644 --- a/client/app/pages/destinations/list.js +++ b/client/app/pages/destinations/list.js @@ -1,9 +1,7 @@ import settingsMenu from '@/lib/settings-menu'; import template from './list.html'; -function DestinationsCtrl($scope, $location, toastr, currentUser, Events, Destination) { - Events.record('view', 'page', 'admin/destinations'); - +function DestinationsCtrl($scope, $location, toastr, currentUser, Destination) { $scope.destinations = Destination.query(); } diff --git a/client/app/pages/destinations/show.js b/client/app/pages/destinations/show.js index 795d8b77a5..bdba532ebb 100644 --- a/client/app/pages/destinations/show.js +++ b/client/app/pages/destinations/show.js @@ -6,9 +6,8 @@ const logger = debug('redash:http'); function DestinationCtrl( $scope, $route, $routeParams, $http, $location, toastr, - currentUser, AlertDialog, Events, Destination, + currentUser, AlertDialog, Destination, ) { - Events.record('view', 'page', 'admin/destination'); $scope.destination = $route.current.locals.destination; $scope.destinationId = $routeParams.destinationId; @@ -34,8 +33,6 @@ function DestinationCtrl( $scope.delete = () => { const doDelete = () => { - Events.record('delete', 'destination', $scope.destination.id); - $scope.destination.$delete(() => { toastr.success('Destination deleted successfully.'); $location.path('/destinations/'); diff --git a/client/app/pages/groups/data-sources.js b/client/app/pages/groups/data-sources.js index 1e0a94f9f3..b571981709 100644 --- a/client/app/pages/groups/data-sources.js +++ b/client/app/pages/groups/data-sources.js @@ -1,8 +1,7 @@ import { includes } from 'lodash'; import template from './data-sources.html'; -function GroupDataSourcesCtrl($scope, $routeParams, $http, Events, Group, DataSource) { - Events.record('view', 'group_data_sources', $scope.groupId); +function GroupDataSourcesCtrl($scope, $routeParams, $http, Group, DataSource) { $scope.group = Group.get({ id: $routeParams.groupId }); $scope.dataSources = Group.dataSources({ id: $routeParams.groupId }); $scope.newDataSource = {}; diff --git a/client/app/pages/groups/list.js b/client/app/pages/groups/list.js index 48b1480cfc..3ce9f60174 100644 --- a/client/app/pages/groups/list.js +++ b/client/app/pages/groups/list.js @@ -2,8 +2,7 @@ import settingsMenu from '@/lib/settings-menu'; import { Paginator } from '@/lib/pagination'; import template from './list.html'; -function GroupsCtrl($scope, $uibModal, currentUser, Events, Group) { - Events.record('view', 'page', 'groups'); +function GroupsCtrl($scope, $uibModal, currentUser, Group) { $scope.currentUser = currentUser; $scope.groups = new Paginator([], { itemsPerPage: 20 }); Group.query((groups) => { diff --git a/client/app/pages/groups/show.js b/client/app/pages/groups/show.js index 3a6eb01250..46978b53b5 100644 --- a/client/app/pages/groups/show.js +++ b/client/app/pages/groups/show.js @@ -1,9 +1,7 @@ import { includes } from 'lodash'; import template from './show.html'; -function GroupCtrl($scope, $routeParams, $http, currentUser, Events, Group, User) { - Events.record('view', 'group', $scope.groupId); - +function GroupCtrl($scope, $routeParams, $http, currentUser, Group, User) { $scope.currentUser = currentUser; $scope.group = Group.get({ id: $routeParams.groupId }); $scope.members = Group.members({ id: $routeParams.groupId }); diff --git a/client/app/pages/queries-list/index.js b/client/app/pages/queries-list/index.js index 257768652e..cc509a4aff 100644 --- a/client/app/pages/queries-list/index.js +++ b/client/app/pages/queries-list/index.js @@ -20,9 +20,7 @@ class QueriesListCtrl { this.pageSize = parseInt($location.search().page_size || 20, 10); this.pageSizeOptions = [5, 10, 20, 50, 100]; - if (isString(this.term) && this.term !== '') { - Events.record('search', 'query', '', { term: this.term }); - } else { + if (!isString(this.term)) { this.term = ''; } diff --git a/client/app/pages/queries/add-to-dashboard.html b/client/app/pages/queries/add-to-dashboard.html new file mode 100644 index 0000000000..1f5e6f027a --- /dev/null +++ b/client/app/pages/queries/add-to-dashboard.html @@ -0,0 +1,23 @@ +
+ +

Add to Dashboard

+
+
+ +
+
+ + + + {{$select.selected.name}} + +
+ + +
+
+ +
diff --git a/client/app/pages/queries/add-to-dashboard.js b/client/app/pages/queries/add-to-dashboard.js new file mode 100644 index 0000000000..292727c141 --- /dev/null +++ b/client/app/pages/queries/add-to-dashboard.js @@ -0,0 +1,70 @@ +import template from './add-to-dashboard.html'; + +const AddToDashboardForm = { + controller($sce, Dashboard, currentUser, toastr, Query, Widget) { + 'ngInject'; + + this.query = this.resolve.query; + this.vis = this.resolve.vis; + this.saveAddToDashbosard = this.resolve.saveAddToDashboard; + this.saveInProgress = false; + + this.trustAsHtml = html => $sce.trustAsHtml(html); + + this.onDashboardSelected = (dash) => { + // add widget to dashboard + this.saveInProgress = true; + this.widgetSize = 1; + this.selectedVis = null; + this.query = {}; + this.selected_query = this.query.id; + this.type = 'visualization'; + this.isVisualization = () => this.type === 'visualization'; + + const widget = new Widget({ + visualization_id: this.vis && this.vis.id, + dashboard_id: dash.id, + options: {}, + width: this.widgetSize, + type: this.type, + }); + + // (response) + widget.save().then(() => { + // (dashboard) + this.selectedDashboard = Dashboard.get({ slug: dash.slug }, () => {}); + this.close(); + }).catch(() => { + toastr.error('Widget can not be added'); + }).finally(() => { + this.saveInProgress = false; + }); + }; + + this.selectedDashboard = null; + + this.searchDashboards = (term) => { // , limitToUsersDashboards + if (!term || term.length < 3) { + return; + } + + Dashboard.get({ + q: term, + include_drafts: true, + }, (results) => { + this.dashboards = results.results; + }); + }; + }, + bindings: { + resolve: '<', + close: '&', + dismiss: '&', + vis: '<', + }, + template, +}; + +export default function (ngModule) { + ngModule.component('addToDashboardDialog', AddToDashboardForm); +} diff --git a/client/app/pages/queries/compare-query-dialog.css b/client/app/pages/queries/compare-query-dialog.css new file mode 100644 index 0000000000..ce2d01370e --- /dev/null +++ b/client/app/pages/queries/compare-query-dialog.css @@ -0,0 +1,54 @@ +/* Compare Query Version container */ +/* Offers slight visual improvement (alignment) to modern UAs */ +.compare-query-version { + display: flex; + justify-content: space-between; + align-items: center; +} + +.diff-removed { + background-color: rgba(208, 2, 27, 0.3); +} + +.diff-added { + background-color: rgba(65, 117, 5, 0.3); +} + +.query-diff-container span { + display: inline-block; + border-radius: 3px; + line-height: 20px; + vertical-align: middle; + margin: 0 5px 0 0; +} + +.query-diff-container > div:not(.compare-query-version-controls) { + float: left; + width: calc(50% - 5px); + margin: 0 10px 0 0; +} + +.compare-query-version { + background-color: #f5f5f5; + padding: 5px; + border: 1px solid #ccc; + margin-right: 15px; + border-radius: 3px; +} + +.diff-content { + border: 1px solid #ccc; + background-color: #f5f5f5; + border-radius: 3px; + padding: 15px; +} + +.query-diff-container > div:last-child { + margin: 0; +} + +.compare-query-version-controls { + display: flex; + align-items: center; + margin-bottom: 25px; +} diff --git a/client/app/pages/queries/compare-query-dialog.html b/client/app/pages/queries/compare-query-dialog.html new file mode 100644 index 0000000000..5214046055 --- /dev/null +++ b/client/app/pages/queries/compare-query-dialog.html @@ -0,0 +1,33 @@ +
+ +

Compare Query

+
+
+
+
+ Compare current version to + +
+ +
+
+
Current Version {{$ctrl.currentQuery.version}}
+
+

+ + {{part.value}} + +

+
+
+
+
Previous Version {{$ctrl.previousQueryVersion + 1}}
+
+

{{$ctrl.previousQuery}}

+
+
+
diff --git a/client/app/pages/queries/compare-query-dialog.js b/client/app/pages/queries/compare-query-dialog.js new file mode 100644 index 0000000000..fb4338971a --- /dev/null +++ b/client/app/pages/queries/compare-query-dialog.js @@ -0,0 +1,63 @@ +import * as jsDiff from 'diff'; +import template from './compare-query-dialog.html'; +import './compare-query-dialog.css'; + +const CompareQueryDialog = { + controller: ['clientConfig', '$http', function doCompare(clientConfig, $http) { + this.currentQuery = this.resolve.query; + + this.previousQuery = ''; + this.currentDiff = []; + this.previousDiff = []; + this.versions = []; + this.previousQueryVersion = this.currentQuery.version - 2; // due to 0-indexed versions[] + + this.compareQueries = (isInitialLoad) => { + if (!isInitialLoad) { + this.previousQueryVersion = document.getElementById('version-choice').value - 1; // due to 0-indexed versions[] + } + + this.previousQuery = this.versions[this.previousQueryVersion].change.query.current; + this.currentDiff = jsDiff.diffChars(this.previousQuery, this.currentQuery.query); + document.querySelector('.compare-query-revert-wrapper').classList.remove('hidden'); + }; + + this.revertQuery = () => { + this.resolve.query.query = this.previousQuery; + this.resolve.saveQuery(); + + // Close modal. + this.dismiss(); + }; + + $http.get(`/api/queries/${this.currentQuery.id}/version`).then((response) => { + this.versions = response.data; + + const compare = (a, b) => { + if (a.object_version < b.object_version) { + return -1; + } else if (a.object_version > b.object_version) { + return 1; + } + return 0; + }; + + this.versions.sort(compare); + this.compareQueries(true); + }); + }], + scope: { + query: '=', + saveQuery: '<', + }, + bindings: { + resolve: '<', + close: '&', + dismiss: '&', + }, + template, +}; + +export default function (ngModule) { + ngModule.component('compareQueryDialog', CompareQueryDialog); +} diff --git a/client/app/pages/queries/query.html b/client/app/pages/queries/query.html index 4724cdb428..0344a3f10d 100644 --- a/client/app/pages/queries/query.html +++ b/client/app/pages/queries/query.html @@ -45,7 +45,7 @@

-
+
@@ -67,6 +67,9 @@

  • Show API Key
    • +
  • + Query Versions +

    • @@ -81,10 +84,12 @@

    {{ds.name}} + +

    - +
     
    @@ -157,9 +162,16 @@

    + + + Autocomplete + + + +

    - \ No newline at end of file + diff --git a/client/app/pages/queries/source-view.js b/client/app/pages/queries/source-view.js index dfcb779bfb..25bc8a52ce 100644 --- a/client/app/pages/queries/source-view.js +++ b/client/app/pages/queries/source-view.js @@ -102,6 +102,11 @@ function QuerySourceCtrl( }); }; + $scope.autocompleteQuery = true; + $scope.toggleAutocompleteQuery = () => { + $scope.autocompleteQuery = !$scope.autocompleteQuery; + }; + $scope.$watch('query.query', (newQueryText) => { $scope.isDirty = newQueryText !== queryText; }); diff --git a/client/app/pages/queries/view.js b/client/app/pages/queries/view.js index 3072cda76c..c78277bfd0 100644 --- a/client/app/pages/queries/view.js +++ b/client/app/pages/queries/view.js @@ -136,7 +136,6 @@ function QueryViewCtrl( KeyboardShortcuts.unbind(shortcuts); }); - Events.record('view', 'query', $scope.query.id); if ($scope.query.hasResult() || $scope.query.paramsRequired()) { getQueryResult(); } @@ -178,8 +177,6 @@ function QueryViewCtrl( }; $scope.duplicateQuery = () => { - Events.record('fork', 'query', $scope.query.id); - Query.fork({ id: $scope.query.id }, (newQuery) => { $location.url(newQuery.getSourceLink()).replace(); }); @@ -208,6 +205,7 @@ function QueryViewCtrl( } else { request = pick($scope.query, [ 'schedule', + 'schedule_resultset_size', 'query', 'id', 'description', @@ -323,7 +321,6 @@ function QueryViewCtrl( $scope.dataSource = find($scope.dataSources, ds => ds.id === $scope.query.data_source_id); getSchema(); - $scope.executeQuery(); }; $scope.setVisualizationTab = (visualization) => { @@ -353,6 +350,21 @@ function QueryViewCtrl( }); }; + $scope.compareQueryVersion = () => { + if (!$scope.query.query) { + return; + } + + $uibModal.open({ + windowClass: 'modal-xl', + component: 'compareQueryDialog', + resolve: { + query: $scope.query, + saveQuery: () => $scope.saveQuery, + }, + }); + }; + $scope.$watch('query.name', () => { Title.set($scope.query.name); }); @@ -440,6 +452,18 @@ function QueryViewCtrl( }); }; + $scope.openAddToDashboardForm = (vis) => { + $uibModal.open({ + component: 'addToDashboardDialog', + size: 'sm', + resolve: { + query: $scope.query, + vis, + saveAddToDashboard: () => $scope.saveAddToDashboard, + }, + }); + }; + $scope.showEmbedDialog = (query, visId) => { const visualization = getVisualization(visId); $uibModal.open({ @@ -472,6 +496,17 @@ function QueryViewCtrl( }, }); }; + + $scope.moreMenuIsPopulated = () => { + const menuParent = document.getElementById('query-more-menu'); + + if (menuParent) { + if (menuParent.querySelectorAll('.dropdown-menu li').length) { + return true; + } + } + return false; + }; } export default function init(ngModule) { diff --git a/client/app/pages/query-snippets/edit.js b/client/app/pages/query-snippets/edit.js index 9522a70c9a..4419478d3f 100644 --- a/client/app/pages/query-snippets/edit.js +++ b/client/app/pages/query-snippets/edit.js @@ -3,7 +3,6 @@ import template from './edit.html'; function SnippetCtrl($routeParams, $http, $location, toastr, currentUser, AlertDialog, Events, QuerySnippet) { this.snippetId = $routeParams.snippetId; - Events.record('view', 'query_snippet', this.snippetId); this.editorOptions = { mode: 'snippets', diff --git a/client/app/pages/query-snippets/list.js b/client/app/pages/query-snippets/list.js index ee0f268218..48d12c070e 100644 --- a/client/app/pages/query-snippets/list.js +++ b/client/app/pages/query-snippets/list.js @@ -2,9 +2,7 @@ import settingsMenu from '@/lib/settings-menu'; import { Paginator } from '@/lib/pagination'; import template from './list.html'; -function SnippetsCtrl($location, currentUser, Events, QuerySnippet) { - Events.record('view', 'page', 'query_snippets'); - +function SnippetsCtrl($location, currentUser, QuerySnippet) { this.snippets = new Paginator([], { itemsPerPage: 20 }); QuerySnippet.query((snippets) => { this.snippets.updateRows(snippets); diff --git a/client/app/pages/users/list.html b/client/app/pages/users/list.html index b2a62243ac..c5cfd05f6c 100644 --- a/client/app/pages/users/list.html +++ b/client/app/pages/users/list.html @@ -46,6 +46,10 @@ Joined + + Last Active At + + @@ -62,6 +66,9 @@ + + +
    @@ -74,4 +81,4 @@
    - \ No newline at end of file + diff --git a/client/app/pages/users/list.js b/client/app/pages/users/list.js index 2d6d5e4f3c..3e10805283 100644 --- a/client/app/pages/users/list.js +++ b/client/app/pages/users/list.js @@ -3,9 +3,7 @@ import settingsMenu from '@/lib/settings-menu'; import { LivePaginator } from '@/lib/pagination'; import template from './list.html'; -function UsersCtrl($location, currentUser, Policy, Events, User) { - Events.record('view', 'page', 'users'); - +function UsersCtrl($location, currentUser, Policy, User) { this.currentUser = currentUser; if ($location.path() === '/users/disabled') { this.currentPage = 'disabled_users'; diff --git a/client/app/pages/users/show.js b/client/app/pages/users/show.js index aec7abe39f..e6f17be3af 100644 --- a/client/app/pages/users/show.js +++ b/client/app/pages/users/show.js @@ -6,7 +6,7 @@ import './settings.less'; function UserCtrl( $scope, $routeParams, $http, $location, toastr, - clientConfig, currentUser, Events, User, + clientConfig, currentUser, User, ) { $scope.userId = $routeParams.userId; $scope.currentUser = currentUser; @@ -16,7 +16,6 @@ function UserCtrl( $scope.userId = currentUser.id; } - Events.record('view', 'user', $scope.userId); $scope.canEdit = currentUser.hasPermission('admin') || currentUser.id === parseInt($scope.userId, 10); $scope.showSettings = false; $scope.showPasswordSettings = false; diff --git a/client/app/services/query-result.js b/client/app/services/query-result.js index 1533b2b948..c9b3ac1a46 100644 --- a/client/app/services/query-result.js +++ b/client/app/services/query-result.js @@ -54,6 +54,7 @@ function addPointToSeries(point, seriesCollection, seriesName) { function QueryResultService($resource, $timeout, $q, QueryResultError) { const QueryResultResource = $resource('api/query_results/:id', { id: '@id' }, { post: { method: 'POST' } }); + const QueryResultSetResource = $resource('api/queries/:id/resultset', { id: '@id' }); const Job = $resource('api/jobs/:id', { id: '@id' }); const statuses = { 1: 'waiting', @@ -452,6 +453,15 @@ function QueryResultService($resource, $timeout, $q, QueryResultError) { return queryResult; } + static getResultSet(queryId) { + const queryResult = new QueryResult(); + + QueryResultSetResource.get({ id: queryId }, (response) => { + queryResult.update(response); + }); + + return queryResult; + } loadResult(tryCount) { this.isLoadingResult = true; QueryResultResource.get( @@ -496,8 +506,16 @@ function QueryResultService($resource, $timeout, $q, QueryResultError) { } }, (error) => { logger('Connection error', error); - // TODO: use QueryResultError, or better yet: exception/reject of promise. - this.update({ job: { error: 'failed communicating with server. Please check your Internet connection and try again.', status: 4 } }); + this.update({ + job: { + error: 'Failed communicating with server. Retrying...', + status: 4, + id: this.job.id, + }, + }); + $timeout(() => { + this.refreshStatus(query); + }, 3000); }); } diff --git a/client/app/services/query.js b/client/app/services/query.js index 9f525caff5..6e03eeaee7 100644 --- a/client/app/services/query.js +++ b/client/app/services/query.js @@ -144,7 +144,7 @@ class Parameter { }; } return { - [`p_${this.name}`]: this.value, + [`p_${this.name}_${this.queryId}`]: this.value, }; } @@ -156,7 +156,7 @@ class Parameter { this.setValue([query[keyStart], query[keyEnd]]); } } else { - const key = `p_${this.name}`; + const key = `p_${this.name}_${this.queryId}`; if (has(query, key)) { this.setValue(query[key]); } @@ -219,7 +219,9 @@ class Parameters { }); const parameterExists = p => includes(parameterNames, p.name); - this.query.options.parameters = this.query.options.parameters.filter(parameterExists).map(p => new Parameter(p)); + this.query.options.parameters = this.query.options.parameters + .filter(parameterExists) + .map(p => new Parameter(Object.assign({ queryId: this.query.id }, p))); } initFromQueryString(query) { @@ -402,6 +404,10 @@ function QueryResource( .format('HH:mm'); }; + Query.prototype.hasScheduleExpiry = function hasScheduleExpiry() { + return (this.schedule && this.schedule_until); + }; + Query.prototype.hasResult = function hasResult() { return !!(this.latest_query_data || this.latest_query_data_id); }; @@ -443,7 +449,11 @@ function QueryResource( this.latest_query_data_id = null; } - if (this.latest_query_data && maxAge !== 0) { + if (this.schedule_resultset_size) { + if (!this.queryResult) { + this.queryResult = QueryResult.getResultSet(this.id); + } + } else if (this.latest_query_data && maxAge !== 0) { if (!this.queryResult) { this.queryResult = new QueryResult({ query_result: this.latest_query_data, @@ -480,7 +490,7 @@ function QueryResource( params += '&'; } - params += `p_${encodeURIComponent(name)}=${encodeURIComponent(value)}`; + params += `p_${encodeURIComponent(name)}_${this.id}=${encodeURIComponent(value)}`; }); } diff --git a/client/app/visualizations/chart/chart-editor.html b/client/app/visualizations/chart/chart-editor.html index e7133ee956..8f4b0837b2 100644 --- a/client/app/visualizations/chart/chart-editor.html +++ b/client/app/visualizations/chart/chart-editor.html @@ -196,6 +196,12 @@ Show Labels + +
    + + + How many characters should X Axis Labels be truncated at in the legend? +
    diff --git a/client/app/visualizations/chart/index.js b/client/app/visualizations/chart/index.js index 4946aeb431..e8a95c131d 100644 --- a/client/app/visualizations/chart/index.js +++ b/client/app/visualizations/chart/index.js @@ -282,6 +282,19 @@ function ChartEditor(ColorPalette, clientConfig) { scope.options.legend = { enabled: true }; } + scope.$watch('options.globalSeriesType', (newType, oldType) => { + const defaultXAxisLength = 10; + if (!has(scope.options, 'xAxisLabelLength')) { + scope.options.xAxisLabelLength = defaultXAxisLength; + } + if (oldType !== newType) { + scope.options.xAxisLabelLength = defaultXAxisLength; + if (newType === 'pie') { + scope.options.xAxisLabelLength = 300; + } + } + }, true); + if (scope.columnNames) { each(scope.options.columnMapping, (value, key) => { if (scope.columnNames.length > 0 && !includes(scope.columnNames, key)) { diff --git a/client/app/visualizations/chart/plotly/utils.js b/client/app/visualizations/chart/plotly/utils.js index 537b7c7fe9..733b3d9643 100644 --- a/client/app/visualizations/chart/plotly/utils.js +++ b/client/app/visualizations/chart/plotly/utils.js @@ -209,6 +209,19 @@ function getUnifiedXAxisValues(seriesList, sorted) { return sorted ? sortBy(result, identity) : result; } +const DEFAULT_XAXIS_LABEL_LENGTH = 300; + +// We only truncate category x-axis labels because the other types +// are correctly formatted by Plotly. +function truncateCategoryAxis(oldXLabel, options) { + const xAxisLabelLength = parseInt(options.xAxisLabelLength, 10) || DEFAULT_XAXIS_LABEL_LENGTH; + + if (options && options.xAxis && options.xAxis.type === 'category') { + return String(oldXLabel).substr(0, xAxisLabelLength); + } + return oldXLabel; +} + function preparePieData(seriesList, options) { const { cellWidth, cellHeight, xPadding, yPadding, cellsInRow, hasX, @@ -260,9 +273,17 @@ function preparePieData(seriesList, options) { }); }); + const colorPalette = ColorPaletteArray.slice(); return { values: map(serie.data, i => i.y), - labels: map(serie.data, row => (hasX ? normalizeValue(row.x) : `Slice ${index}`)), + labels: map(serie.data, (row, rowIdx) => { + const rowX = hasX ? truncateCategoryAxis(normalizeValue(row.x), options) : `Slice ${index}`; + const rowOpts = options.seriesOptions[rowX]; + if (rowOpts) { + colorPalette[rowIdx] = rowOpts.color; + } + return rowX; + }), type: 'pie', hole: 0.4, marker: { @@ -317,7 +338,7 @@ function prepareChartData(seriesList, options) { const yValues = []; const yErrorValues = []; each(data, (row) => { - const x = normalizeValue(row.x); + const x = truncateCategoryAxis(normalizeValue(row.x), options); const y = normalizeValue(row.y); const yError = normalizeValue(row.yError); const size = normalizeValue(row.size); diff --git a/client/app/visualizations/edit-visualization-dialog.css b/client/app/visualizations/edit-visualization-dialog.css new file mode 100644 index 0000000000..3e84b755b2 --- /dev/null +++ b/client/app/visualizations/edit-visualization-dialog.css @@ -0,0 +1,5 @@ +/* Edit Visualization Dialog specific CSS */ + +.slight-padding { + padding: 5px; +} \ No newline at end of file diff --git a/client/app/visualizations/edit-visualization-dialog.html b/client/app/visualizations/edit-visualization-dialog.html index 28791ee2ca..4d9b531b5c 100644 --- a/client/app/visualizations/edit-visualization-dialog.html +++ b/client/app/visualizations/edit-visualization-dialog.html @@ -34,10 +34,18 @@

    Visualization Editor

    +
    +
    +
    +
    +
    +
    +
    +
    - +
    diff --git a/client/app/visualizations/edit-visualization-dialog.js b/client/app/visualizations/edit-visualization-dialog.js index 7c855d20be..197edcf31d 100644 --- a/client/app/visualizations/edit-visualization-dialog.js +++ b/client/app/visualizations/edit-visualization-dialog.js @@ -1,6 +1,7 @@ import { map } from 'lodash'; import { copy } from 'angular'; import template from './edit-visualization-dialog.html'; +import './edit-visualization-dialog.css'; const EditVisualizationDialog = { template, @@ -21,6 +22,8 @@ const EditVisualizationDialog = { // Don't allow to change type after creating visualization this.canChangeType = !(this.visualization && this.visualization.id); + this.warning_three_column_groupby = 'You have more than 2 columns in your result set. To ensure the chart is accurate, please do one of the following:
    • Change the SQL query to give 2 result columns. You can CONCAT() columns together if you wish.
    • Select column(s) to group by.
    '; + this.warning_three_column_stacking = 'You have more than 2 columns in your result set. You may wish to make the Stacking option equal to `Enabled` or `Percent`.'; this.newVisualization = () => ({ @@ -48,6 +51,24 @@ const EditVisualizationDialog = { } }; + this.has3plusColumnsFunction = () => { + let has3plusColumns = false; + if ((JSON.stringify(this.visualization.options.columnMapping).match(/,/g) || []).length > 2) { + has3plusColumns = true; + } + return has3plusColumns; + }; + + this.disableSubmit = () => { + if (this.visualization.options.globalSeriesType === 'column' + && this.has3plusColumnsFunction() + && !JSON.stringify(this.visualization.options.columnMapping).includes('"":') + && JSON.stringify(this.visualization.options.columnMapping).includes('unused')) { + return true; + } + return false; + }; + this.submit = () => { if (this.visualization.id) { Events.record('update', 'visualization', this.visualization.id, { type: this.visualization.type }); diff --git a/docker-compose.yml b/docker-compose.yml index b454410bff..16d6f13582 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,3 +41,13 @@ services: # tests. command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF" restart: unless-stopped + flower: + image: mher/flower:latest + command: flower + environment: + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + ports: + - "5555:5555" + links: + - redis diff --git a/migrations/versions/15041b7085fe_.py b/migrations/versions/15041b7085fe_.py new file mode 100644 index 0000000000..fcb10aa78f --- /dev/null +++ b/migrations/versions/15041b7085fe_.py @@ -0,0 +1,24 @@ +"""empty message + +Revision ID: 15041b7085fe +Revises: f9571a5ab4f3, 969126bd800f +Create Date: 2018-02-14 17:52:17.138127 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '15041b7085fe' +down_revision = ('f9571a5ab4f3', '969126bd800f') +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/migrations/versions/2ba47e9812b1_.py b/migrations/versions/2ba47e9812b1_.py new file mode 100644 index 0000000000..93d0f59268 --- /dev/null +++ b/migrations/versions/2ba47e9812b1_.py @@ -0,0 +1,24 @@ +"""empty message + +Revision ID: 2ba47e9812b1 +Revises: 71477dadd6ef, 9d7678c47452 +Create Date: 2018-07-25 16:09:54.769289 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '2ba47e9812b1' +down_revision = ('71477dadd6ef', '9d7678c47452', ) +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/migrations/versions/40384fa03dd1_.py b/migrations/versions/40384fa03dd1_.py new file mode 100644 index 0000000000..f2c53711c0 --- /dev/null +++ b/migrations/versions/40384fa03dd1_.py @@ -0,0 +1,40 @@ +"""Upgrade 'data_scanned' column to form used in upstream + +Revision ID: 40384fa03dd1 +Revises: 58f810489c47 +Create Date: 2018-01-18 18:44:04.917081 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.sql.expression import func, cast + +# revision identifiers, used by Alembic. +revision = '40384fa03dd1' +down_revision = 'fbc0849e2674' +branch_labels = None +depends_on = None + + +def upgrade(): + qr = sa.sql.table('query_results', + sa.sql.column('data_scanned', sa.String), + sa.sql.column('data', sa.String)) + op.execute( + qr.update() + .where(qr.c.data_scanned != '') + .where(qr.c.data_scanned != 'error') + .where(qr.c.data_scanned != 'N/A') + .values(data=cast( + func.jsonb_set(cast(qr.c.data, JSONB), + '{metadata}', + cast('{"data_scanned": ' + + qr.c.data_scanned + '}', + JSONB)), + sa.String))) + op.drop_column('query_results', 'data_scanned') + + +def downgrade(): + op.add_column('query_results', sa.Column('data_scanned', sa.String(length=255), nullable=True)) diff --git a/migrations/versions/58f810489c47_.py b/migrations/versions/58f810489c47_.py new file mode 100644 index 0000000000..1ed4190288 --- /dev/null +++ b/migrations/versions/58f810489c47_.py @@ -0,0 +1,28 @@ +"""add 'data_scanned' column to query_results + +Revision ID: 58f810489c47 +Revises: eb2f788f997e +Create Date: 2017-06-25 21:24:54.942119 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '58f810489c47' +down_revision = 'eb2f788f997e' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('query_results', sa.Column('data_scanned', sa.String(length=255), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('query_results', 'data_scanned') + # ### end Alembic commands ### diff --git a/migrations/versions/9d7678c47452_.py b/migrations/versions/9d7678c47452_.py new file mode 100644 index 0000000000..d351153c87 --- /dev/null +++ b/migrations/versions/9d7678c47452_.py @@ -0,0 +1,34 @@ +"""Incremental query results aggregation + +Revision ID: 9d7678c47452 +Revises: 15041b7085fe +Create Date: 2018-03-08 04:36:12.802199 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '9d7678c47452' +down_revision = '15041b7085fe' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table('query_resultsets', + sa.Column('query_id', sa.Integer(), nullable=False), + sa.Column('result_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['query_id'], ['queries.id'], ), + sa.ForeignKeyConstraint(['result_id'], ['query_results.id'], ), + sa.PrimaryKeyConstraint('query_id', 'result_id') + ) + op.add_column(u'queries', sa.Column('schedule_resultset_size', sa.Integer(), nullable=True)) +1 + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column(u'queries', 'schedule_resultset_size') + op.drop_table('query_resultsets') + # ### end Alembic commands ### diff --git a/migrations/versions/eb2f788f997e_.py b/migrations/versions/eb2f788f997e_.py new file mode 100644 index 0000000000..71fd2bd5b3 --- /dev/null +++ b/migrations/versions/eb2f788f997e_.py @@ -0,0 +1,27 @@ +"""Add 'schedule_until' column to queries. + +Revision ID: eb2f788f997e +Revises: d1eae8b9893e +Create Date: 2017-03-02 12:20:00.029066 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'eb2f788f997e' +down_revision = 'd1eae8b9893e' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + 'queries', + sa.Column('schedule_until', sa.DateTime(timezone=True), nullable=True)) + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('queries', 'schedule_until') diff --git a/migrations/versions/f9571a5ab4f3_.py b/migrations/versions/f9571a5ab4f3_.py new file mode 100644 index 0000000000..da1ba02d6d --- /dev/null +++ b/migrations/versions/f9571a5ab4f3_.py @@ -0,0 +1,28 @@ +"""Rename 'image_url' to 'profile_image_url' + + a revision was changed after we pulled it from upstream in m12, so it had to + be fixed here. + + +Revision ID: f9571a5ab4f3 +Revises: 40384fa03dd1 +Create Date: 2018-01-18 18:04:07.943843 +""" +from alembic import op + + +# revision identifiers, used by Alembic. +revision = 'f9571a5ab4f3' +down_revision = '40384fa03dd1' +branch_labels = None +depends_on = None + + +def upgrade(): + # Upstream changed the column name in migration revision 7671dca4e604 -- + # see git revision 62e5e3892603502c5f3a6da277c33c73510b8819 + op.alter_column('users', 'image_url', new_column_name='profile_image_url') + + +def downgrade(): + op.alter_column('users', 'profile_image_url', new_column_name='image_url') diff --git a/migrations/versions/fbc0849e2674_.py b/migrations/versions/fbc0849e2674_.py new file mode 100644 index 0000000000..6195141496 --- /dev/null +++ b/migrations/versions/fbc0849e2674_.py @@ -0,0 +1,26 @@ +""" +Merge upstream fulltext search + +This formerly merged the fulltext search changes (6b5be7e0a0ef, 5ec5c84ba61e) +with upstream's 7671dca4e604 - but then those changes moved in the revision +graph to be direct descendants of that upstream revision, so the merge point +has been moved. + +Revision ID: fbc0849e2674 +Revises: 6b5be7e0a0ef, eb2f788f997e +Create Date: 2017-12-12 04:45:34.360587 +""" + +# revision identifiers, used by Alembic. +revision = 'fbc0849e2674' +down_revision = ('6b5be7e0a0ef', '58f810489c47') +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/package.json b/package.json index f3899c9c44..5f53bc5fbd 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "scripts": { "start": "webpack-dev-server", "dev": "REDASH_BACKEND=https://dev.redashapp.com npm start", + "bundle": "bin/bundle-extensions", "build": "rm -rf ./client/dist/ && NODE_ENV=production webpack", "watch": "webpack --watch --progress --colors -d", "analyze": "rm -rf ./client/dist/ && BUNDLE_ANALYZER=on webpack", @@ -51,6 +52,7 @@ "d3": "^3.5.17", "d3-cloud": "^1.2.4", "debug": "^3.1.0", + "diff": "^3.3.0", "font-awesome": "^4.7.0", "gridstack": "^0.3.0", "jquery": "^3.2.1", diff --git a/redash/__init__.py b/redash/__init__.py index bd9a8e5859..233f44196b 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -128,6 +128,11 @@ def create_app(load_admin=True): app.config['SQLALCHEMY_DATABASE_URI'] = settings.SQLALCHEMY_DATABASE_URI app.config.update(settings.all_settings()) + def set_response_headers(response): + response.headers['X-Content-Type-Options'] = 'nosniff' + return response + + app.after_request(set_response_headers) provision_app(app) db.init_app(app) migrate.init_app(app, db) diff --git a/redash/authentication/remote_user_auth.py b/redash/authentication/remote_user_auth.py index 9a78da77ab..f159eb7dc9 100644 --- a/redash/authentication/remote_user_auth.py +++ b/redash/authentication/remote_user_auth.py @@ -30,6 +30,21 @@ def login(org_slug=None): logger.error("Cannot use remote user for login when it's not provided in the request (looked in headers['" + settings.REMOTE_USER_HEADER + "'])") return redirect(url_for('redash.index', next=next_path, org_slug=org_slug)) + # Check if there is a header of user groups and if yes + # check it against a list of allowed user groups from the settings + if settings.REMOTE_GROUPS_ENABLED: + remote_groups = settings.set_from_string( + request.headers.get(settings.REMOTE_GROUPS_HEADER) or '' + ) + allowed_groups = settings.REMOTE_GROUPS_ALLOWED + if not allowed_groups.intersection(remote_groups): + logger.error( + "User groups provided in the %s header are not " + "matching the allowed groups.", + settings.REMOTE_GROUPS_HEADER + ) + return redirect(url_for('redash.index', next=next_path)) + logger.info("Logging in " + email + " via remote user") user = create_and_login_user(current_org, email, email) diff --git a/redash/extensions.py b/redash/extensions.py index 78125842d3..e00a8b164c 100644 --- a/redash/extensions.py +++ b/redash/extensions.py @@ -1,4 +1,5 @@ -from pkg_resources import iter_entry_points +import os +from pkg_resources import iter_entry_points, resource_isdir, resource_listdir def init_extensions(app): @@ -10,5 +11,15 @@ def init_extensions(app): for entry_point in iter_entry_points('redash.extensions'): app.logger.info('Loading Redash extension %s.', entry_point.name) - extension = entry_point.load() - app.redash_extensions[entry_point.name] = extension(app) + try: + extension = entry_point.load() + app.redash_extensions[entry_point.name] = extension(app) + except ImportError: + app.logger.info('%s does not have a callable and will not be loaded.', entry_point.name) + (root_module, _) = os.path.splitext(entry_point.module_name) + content_folder_relative = os.path.join(entry_point.name, 'bundle') + + # If it's a frontend extension only, store a list of files in the bundle directory. + if resource_isdir(root_module, content_folder_relative): + app.redash_extensions[entry_point.name] = resource_listdir( + root_module, content_folder_relative) diff --git a/redash/handlers/admin.py b/redash/handlers/admin.py index 919dc91924..51b0c0ca05 100644 --- a/redash/handlers/admin.py +++ b/redash/handlers/admin.py @@ -1,10 +1,12 @@ import json +import time from flask import request -from flask_login import login_required +from flask_login import current_user, login_required from redash import models, redis_connection +from redash.authentication import current_org from redash.handlers import routes -from redash.handlers.base import json_response +from redash.handlers.base import json_response, record_event from redash.permissions import require_super_admin from redash.tasks.queries import QueryTaskTracker @@ -23,6 +25,13 @@ def outdated_queries(): else: outdated_queries = [] + record_event(current_org, current_user, { + 'action': 'view', + 'object_type': 'api_call', + 'object_id': 'admin/outdated_queries', + 'timestamp': int(time.time()), + }) + return json_response( dict(queries=[q.to_dict(with_stats=True, with_last_modified_by=False) for q in outdated_queries], @@ -41,6 +50,12 @@ def queries_tasks(): waiting = QueryTaskTracker.all(QueryTaskTracker.WAITING_LIST, limit=waiting_limit) in_progress = QueryTaskTracker.all(QueryTaskTracker.IN_PROGRESS_LIST, limit=progress_limit) done = QueryTaskTracker.all(QueryTaskTracker.DONE_LIST, limit=done_limit) + record_event(current_org, current_user, { + 'action': 'view', + 'object_type': 'api_call', + 'object_id': 'admin/tasks', + 'timestamp': int(time.time()), + }) response = { 'waiting': [t.data for t in waiting if t is not None], diff --git a/redash/handlers/alerts.py b/redash/handlers/alerts.py index 569e5e2753..f5bfc0e13c 100644 --- a/redash/handlers/alerts.py +++ b/redash/handlers/alerts.py @@ -15,7 +15,14 @@ class AlertResource(BaseResource): def get(self, alert_id): alert = get_object_or_404(models.Alert.get_by_id_and_org, alert_id, self.current_org) require_access(alert.groups, self.current_user, view_only) + self.record_event({ + 'action': 'view', + 'timestamp': int(time.time()), + 'object_id': alert.id, + 'object_type': 'alert' + }) return serialize_alert(alert) + return alert.to_dict() def post(self, alert_id): req = request.get_json(True) diff --git a/redash/handlers/api.py b/redash/handlers/api.py index f8ef199857..4518bffcce 100644 --- a/redash/handlers/api.py +++ b/redash/handlers/api.py @@ -6,11 +6,11 @@ from redash.handlers.base import org_scoped_rule from redash.handlers.permissions import ObjectPermissionsListResource, CheckPermissionResource from redash.handlers.alerts import AlertResource, AlertListResource, AlertSubscriptionListResource, AlertSubscriptionResource -from redash.handlers.dashboards import DashboardListResource, DashboardResource, DashboardShareResource, PublicDashboardResource +from redash.handlers.dashboards import DashboardListResource, DashboardResource, DashboardShareResource, PublicDashboardResource from redash.handlers.data_sources import DataSourceTypeListResource, DataSourceListResource, DataSourceSchemaResource, DataSourceResource, DataSourcePauseResource, DataSourceTestResource from redash.handlers.events import EventsResource -from redash.handlers.queries import QueryForkResource, QueryRefreshResource, QueryListResource, QueryRecentResource, QuerySearchResource, QueryResource, MyQueriesResource -from redash.handlers.query_results import QueryResultListResource, QueryResultResource, JobResource +from redash.handlers.queries import QueryForkResource, QueryRefreshResource, QueryListResource, QueryRecentResource, QuerySearchResource, QueryResource, MyQueriesResource, QueryVersionListResource, ChangeResource +from redash.handlers.query_results import QueryResultListResource, QueryResultResource, JobResource, QueryResultSetResource from redash.handlers.users import UserResource, UserListResource, UserInviteResource, UserResetPasswordResource, UserDisableResource from redash.handlers.visualizations import VisualizationListResource from redash.handlers.visualizations import VisualizationResource @@ -84,6 +84,9 @@ def json_representation(data, code, headers=None): api.add_org_resource(QueryRefreshResource, '/api/queries//refresh', endpoint='query_refresh') api.add_org_resource(QueryResource, '/api/queries/', endpoint='query') api.add_org_resource(QueryForkResource, '/api/queries//fork', endpoint='query_fork') +api.add_org_resource(QueryResultSetResource, '/api/queries//resultset', endpoint='query_aggregate_results') +api.add_org_resource(QueryVersionListResource, '/api/queries//version', endpoint='query_versions') +api.add_org_resource(ChangeResource, '/api/changes/', endpoint='changes') api.add_org_resource(ObjectPermissionsListResource, '/api///acl', endpoint='object_permissions') api.add_org_resource(CheckPermissionResource, '/api///acl/', endpoint='check_permissions') diff --git a/redash/handlers/dashboards.py b/redash/handlers/dashboards.py index da7c4e01f3..20ee945715 100644 --- a/redash/handlers/dashboards.py +++ b/redash/handlers/dashboards.py @@ -10,6 +10,7 @@ from redash.permissions import (can_modify, require_admin_or_owner, require_object_modify_permission, require_permission) +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import StaleDataError @@ -46,6 +47,7 @@ def get(self): self.current_user.group_ids, self.current_user.id, search_term, + 'include_drafts' in request.args, ) else: results = models.Dashboard.all( @@ -86,6 +88,7 @@ def post(self): user=self.current_user, is_draft=True, layout='[]') + dashboard.record_changes(changed_by=self.current_user) models.db.session.add(dashboard) models.db.session.commit() return serialize_dashboard(dashboard) @@ -137,6 +140,12 @@ def get(self, dashboard_slug=None): response['can_edit'] = can_modify(dashboard, self.current_user) + self.record_event({ + 'action': 'view', + 'object_id': dashboard.id, + 'object_type': 'dashboard', + }) + return response @require_permission('edit_dashboard') @@ -173,9 +182,18 @@ def post(self, dashboard_slug): try: models.db.session.commit() except StaleDataError: + models.db.session.rollback() abort(409) + except IntegrityError: + models.db.session.rollback() + abort(400) result = serialize_dashboard(dashboard, with_widgets=True, user=self.current_user) + self.record_event({ + 'action': 'edit', + 'object_id': dashboard.id, + 'object_type': 'dashboard', + }) return result @require_permission('edit_dashboard') @@ -193,6 +211,11 @@ def delete(self, dashboard_slug): models.db.session.add(dashboard) d = serialize_dashboard(dashboard, with_widgets=True, user=self.current_user) models.db.session.commit() + self.record_event({ + 'action': 'archive', + 'object_id': dashboard.id, + 'object_type': 'dashboard', + }) return d diff --git a/redash/handlers/data_sources.py b/redash/handlers/data_sources.py index be94b1d028..72fd83dce2 100644 --- a/redash/handlers/data_sources.py +++ b/redash/handlers/data_sources.py @@ -25,7 +25,13 @@ class DataSourceResource(BaseResource): @require_admin def get(self, data_source_id): data_source = models.DataSource.get_by_id_and_org(data_source_id, self.current_org) - return data_source.to_dict(all=True) + ds = data_source.to_dict(all=True) + self.record_event({ + 'action': 'view', + 'object_id': data_source.id, + 'object_type': 'data_source', + }) + return ds @require_admin def post(self, data_source_id): @@ -48,6 +54,7 @@ def post(self, data_source_id): try: models.db.session.commit() except IntegrityError as e: + models.db.session.rollback() if req['name'] in e.message: abort(400, message="Data source with the name {} already exists.".format(req['name'])) @@ -59,6 +66,11 @@ def post(self, data_source_id): def delete(self, data_source_id): data_source = models.DataSource.get_by_id_and_org(data_source_id, self.current_org) data_source.delete() + self.record_event({ + 'action': 'delete', + 'object_id': data_source_id, + 'object_type': 'datasource', + }) return make_response('', 204) @@ -83,6 +95,11 @@ def get(self): except AttributeError: logging.exception("Error with DataSource#to_dict (data source id: %d)", ds.id) + self.record_event({ + 'action': 'view', + 'object_id': 'admin/data_sources', + 'object_type': 'api_call', + }) return sorted(response.values(), key=lambda d: d['name'].lower()) @require_admin @@ -111,6 +128,7 @@ def post(self): models.db.session.commit() except IntegrityError as e: + models.db.session.rollback() if req['name'] in e.message: abort(400, message="Data source with the name {} already exists.".format(req['name'])) @@ -186,6 +204,12 @@ class DataSourceTestResource(BaseResource): def post(self, data_source_id): data_source = get_object_or_404(models.DataSource.get_by_id_and_org, data_source_id, self.current_org) + self.record_event({ + 'action': 'test', + 'object_id': data_source_id, + 'object_type': 'datasource', + }) + try: data_source.query_runner.test_connection() except Exception as e: diff --git a/redash/handlers/destinations.py b/redash/handlers/destinations.py index c1895b7321..254e51f078 100644 --- a/redash/handlers/destinations.py +++ b/redash/handlers/destinations.py @@ -19,7 +19,13 @@ class DestinationResource(BaseResource): @require_admin def get(self, destination_id): destination = models.NotificationDestination.get_by_id_and_org(destination_id, self.current_org) - return destination.to_dict(all=True) + d = destination.to_dict(all=True) + self.record_event({ + 'action': 'view', + 'object_id': destination_id, + 'object_type': 'destination' + }) + return d @require_admin def post(self, destination_id): @@ -48,6 +54,12 @@ def delete(self, destination_id): models.db.session.delete(destination) models.db.session.commit() + self.record_event({ + 'action': 'delete', + 'object_id': destination_id, + 'object_type': 'destination', + }) + return make_response('', 204) @@ -63,6 +75,12 @@ def get(self): d = ds.to_dict() response[ds.id] = d + self.record_event({ + 'action': 'view', + 'object_id': 'admin/destinations', + 'object_type': 'api_call', + }) + return response.values() @require_admin diff --git a/redash/handlers/groups.py b/redash/handlers/groups.py index 7790044468..ba72346b10 100644 --- a/redash/handlers/groups.py +++ b/redash/handlers/groups.py @@ -30,6 +30,12 @@ def get(self): groups = models.Group.query.filter( models.Group.id.in_(self.current_user.group_ids)) + self.record_event({ + 'action': 'view', + 'object_id': 'groups', + 'object_type': 'api_call', + }) + return [g.to_dict() for g in groups] @@ -59,6 +65,12 @@ def get(self, group_id): group = models.Group.get_by_id_and_org(group_id, self.current_org) + self.record_event({ + 'action': 'view', + 'object_id': group_id, + 'object_type': 'group', + }) + return group.to_dict() @require_admin @@ -154,6 +166,12 @@ def get(self, group_id): data_sources = (models.DataSource.query .join(models.DataSourceGroup) .filter(models.DataSourceGroup.group == group)) + + self.record_event({ + 'action': 'view', + 'object_id': group_id, + 'object_type': 'group_data_sources', + }) return [ds.to_dict(with_permissions_for=group) for ds in data_sources] diff --git a/redash/handlers/queries.py b/redash/handlers/queries.py index e2fd67a07b..86b24b7296 100644 --- a/redash/handlers/queries.py +++ b/redash/handlers/queries.py @@ -68,6 +68,11 @@ def get(self): return [] include_drafts = request.args.get('include_drafts') is not None + self.record_event({ + 'action': 'search', + 'object_id': term, + 'object_type': 'query', + }) # this redirects to the new query list API that is aware of search new_location = url_for( @@ -103,6 +108,8 @@ def post(self): :json string query: Query text :>json string query_hash: Hash of query text :>json string schedule: Schedule interval, in seconds, for repeated execution of this query + :json string api_key: Key for public access to this query's results. :>json boolean is_archived: Whether this query is displayed in indexes and search results or not. :>json boolean is_draft: Whether this query is a draft or not @@ -139,7 +147,10 @@ def post(self): query_def['data_source'] = data_source query_def['org'] = self.current_org query_def['is_draft'] = True + if query_def.get('schedule_resultset_size') == 1: + query_def['schedule_resultset_size'] = None query = models.Query.create(**query_def) + query.record_changes(changed_by=self.current_user) models.db.session.add(query) models.db.session.commit() @@ -273,6 +284,7 @@ def post(self, query_id): try: self.update_model(query, query_def) + query.record_changes(self.current_user) models.db.session.commit() except StaleDataError: abort(409) @@ -293,6 +305,12 @@ def get(self, query_id): result = QuerySerializer(q, with_visualizations=True).serialize() result['can_edit'] = can_modify(q, self.current_user) + + self.record_event({ + 'action': 'view', + 'object_id': query_id, + 'object_type': 'query', + }) return result # TODO: move to resource of its own? (POST /queries/{id}/archive) @@ -322,6 +340,11 @@ def post(self, query_id): require_access(query.data_source.groups, self.current_user, not_view_only) forked_query = query.fork(self.current_user) models.db.session.commit() + self.record_event({ + 'action': 'fork', + 'object_id': query_id, + 'object_type': 'query', + }) return QuerySerializer(forked_query, with_visualizations=True).serialize() @@ -351,3 +374,15 @@ def post(self, query_id): class QueryTagsResource(BaseResource): def get(self): return {t[0]: t[1] for t in models.Query.all_tags(self.current_user, True)} + +class QueryVersionListResource(BaseResource): + @require_permission('view_query') + def get(self, query_id): + results = models.Change.list_versions(models.Query.get_by_id(query_id)) + return [q.to_dict() for q in results] + + +class ChangeResource(BaseResource): + @require_permission('view_query') + def get(self, change_id): + return models.Change.query.get(change_id).to_dict() diff --git a/redash/handlers/query_results.py b/redash/handlers/query_results.py index 9f08493e54..5da925e5c3 100644 --- a/redash/handlers/query_results.py +++ b/redash/handlers/query_results.py @@ -132,6 +132,33 @@ def post(self): ONE_YEAR = 60 * 60 * 24 * 365.25 +class QueryResultSetResource(BaseResource): + @require_permission('view_query') + def get(self, query_id=None, filetype='json'): + query = get_object_or_404(models.Query.get_by_id_and_org, query_id, self.current_org) + if not query.schedule_resultset_size: + abort(404, message="query does not keep multiple results") + + # Synthesize a result set from the last N results. + total = len(query.query_results) + offset = max(total - query.schedule_resultset_size, 0) + results = [qr.to_dict() for qr in query.query_results[offset:]] + if not results: + aggregate_result = {} + else: + # Start a synthetic data set with the data from the first result... + aggregate_result = results[0].copy() + aggregate_result['data'] = {'columns': results[0]['data']['columns'], + 'rows': []} + # .. then add each subsequent result set into it. + for r in results: + aggregate_result['data']['rows'].extend(r['data']['rows']) + + data = json.dumps({'query_result': aggregate_result}, cls=utils.JSONEncoder) + headers = {'Content-Type': "application/json"} + return make_response(data, 200, headers) + + class QueryResultResource(BaseResource): @staticmethod def add_cors_headers(headers): @@ -194,7 +221,7 @@ def get(self, query_id=None, query_result_id=None, filetype='json'): query_result = run_query_sync(query.data_source, parameter_values, query.query_text, max_age=max_age) elif query.latest_query_data_id is not None: query_result = get_object_or_404(models.QueryResult.get_by_id_and_org, query.latest_query_data_id, self.current_org) - + if query is not None and query_result is not None and self.current_user.is_api_user(): if query.query_hash != query_result.query_hash: abort(404, message='No cached result found for this query.') diff --git a/redash/handlers/query_snippets.py b/redash/handlers/query_snippets.py index fc74865771..fbc6a2871c 100644 --- a/redash/handlers/query_snippets.py +++ b/redash/handlers/query_snippets.py @@ -11,6 +11,11 @@ class QuerySnippetResource(BaseResource): def get(self, snippet_id): snippet = get_object_or_404(models.QuerySnippet.get_by_id_and_org, snippet_id, self.current_org) + self.record_event({ + 'action': 'view', + 'object_id': snippet_id, + 'object_type': 'query_snippet', + }) return snippet.to_dict() def post(self, snippet_id): @@ -69,5 +74,10 @@ def post(self): return snippet.to_dict() def get(self): + self.record_event({ + 'action': 'view', + 'object_id': 'query_snippets', + 'object_type': 'api_call', + }) return [snippet.to_dict() for snippet in models.QuerySnippet.all(org=self.current_org)] diff --git a/redash/handlers/users.py b/redash/handlers/users.py index d0fa73a9ab..586420aff9 100644 --- a/redash/handlers/users.py +++ b/redash/handlers/users.py @@ -49,13 +49,13 @@ def serialize_user(user): if group: user_groups.append({'id': group.id, 'name': group.name}) - + d['groups'] = user_groups return d search_term = request.args.get('q', '') - + if request.args.get('disabled', None) is not None: users = models.User.all_disabled(self.current_org) else: @@ -63,9 +63,14 @@ def serialize_user(user): if search_term: users = models.User.search(users, search_term) - + users = order_results(users) + self.record_event({ + 'action': 'view', + 'object_id': 'users', + 'object_type': 'api_call', + }) return paginate(users, page, page_size, serialize_user) @require_admin @@ -87,6 +92,7 @@ def post(self): models.db.session.add(user) models.db.session.commit() except IntegrityError as e: + models.db.session.rollback() if "email" in e.message: abort(400, message='Email already taken.') abort(500) @@ -138,7 +144,11 @@ class UserResource(BaseResource): def get(self, user_id): require_permission_or_owner('list_users', user_id) user = get_object_or_404(models.User.get_by_id_and_org, user_id, self.current_org) - + self.record_event({ + 'action': 'view', + 'object_id': user_id, + 'object_type': 'user', + }) return user.to_dict(with_api_key=is_admin_or_owner(user_id)) def post(self, user_id): @@ -170,7 +180,7 @@ def post(self, user_id): message = "Email already taken." else: message = "Error updating record" - + models.db.session.rollback() abort(400, message=message) self.record_event({ diff --git a/redash/handlers/visualizations.py b/redash/handlers/visualizations.py index 79bcf7a528..bcddac5440 100644 --- a/redash/handlers/visualizations.py +++ b/redash/handlers/visualizations.py @@ -49,5 +49,10 @@ def post(self, visualization_id): def delete(self, visualization_id): vis = get_object_or_404(models.Visualization.get_by_id_and_org, visualization_id, self.current_org) require_object_modify_permission(vis.query_rel, self.current_user) + self.record_event({ + 'action': 'delete', + 'object_id': visualization_id, + 'object_type': 'visualization', + }) models.db.session.delete(vis) models.db.session.commit() diff --git a/redash/handlers/widgets.py b/redash/handlers/widgets.py index d88908dac2..cae7246221 100644 --- a/redash/handlers/widgets.py +++ b/redash/handlers/widgets.py @@ -78,3 +78,8 @@ def delete(self, widget_id): require_object_modify_permission(widget.dashboard, self.current_user) models.db.session.delete(widget) models.db.session.commit() + self.record_event({ + 'action': 'delete', + 'object_id': widget_id, + 'object_type': 'widget', + }) diff --git a/redash/metrics/celery.py b/redash/metrics/celery.py index 25b63af136..ef7ed68cf2 100644 --- a/redash/metrics/celery.py +++ b/redash/metrics/celery.py @@ -12,7 +12,7 @@ @task_prerun.connect -def task_prerun_handler(signal, sender, task_id, task, args, kwargs): +def task_prerun_handler(signal, sender, task_id, task, args, kwargs, **kw): try: tasks_start_time[task_id] = time.time() except Exception: @@ -30,7 +30,7 @@ def metric_name(name, tags): @task_postrun.connect -def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, state): +def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, state, **kw): try: run_time = 1000 * (time.time() - tasks_start_time.pop(task_id)) diff --git a/redash/models.py b/redash/models.py index 895ead6196..0776bb5345 100644 --- a/redash/models.py +++ b/redash/models.py @@ -199,10 +199,6 @@ class ChangeTrackingMixin(object): skipped_fields = ('id', 'created_at', 'updated_at', 'version') _clean_values = None - def __init__(self, *a, **kw): - super(ChangeTrackingMixin, self).__init__(*a, **kw) - self.record_changes(self.user) - def prep_cleanvalues(self): self.__dict__['_clean_values'] = {} for attr in inspect(self.__class__).column_attrs: @@ -213,10 +209,10 @@ def prep_cleanvalues(self): def __setattr__(self, key, value): if self._clean_values is None: self.prep_cleanvalues() - for attr in inspect(self.__class__).column_attrs: - col, = attr.columns - previous = getattr(self, attr.key, None) - self._clean_values[col.name] = previous + + if key in inspect(self.__class__).column_attrs: + previous = getattr(self, key, None) + self._clean_values[key] = previous super(ChangeTrackingMixin, self).__setattr__(key, value) @@ -227,13 +223,19 @@ def record_changes(self, changed_by): for attr in inspect(self.__class__).column_attrs: col, = attr.columns if attr.key not in self.skipped_fields: - changes[col.name] = {'previous': self._clean_values[col.name], - 'current': getattr(self, attr.key)} + prev = self._clean_values[col.name] + current = getattr(self, attr.key) + if prev != current: + changes[col.name] = {'previous': prev, 'current': current} - db.session.add(Change(object=self, - object_version=self.version, - user=changed_by, - change=changes)) + if changes: + self.version = (self.version or 0) + 1 + change = Change(object=self, + object_version=self.version, + user=changed_by, + change=changes) + db.session.add(change) + return change class BelongsToOrgMixin(object): @@ -473,6 +475,8 @@ def to_dict(self, with_api_key=False): if with_api_key: d['api_key'] = self.api_key + d['last_active_at'] = Event.query.filter(Event.user_id == self.id).with_entities(Event.created_at).order_by(Event.created_at.desc()).first() + return d def is_api_user(self): @@ -670,6 +674,8 @@ def add_group(self, group, view_only=False): db.session.add(dsg) return dsg + setattr(self, 'data_source_groups', dsg) + def remove_group(self, group): db.session.query(DataSourceGroup).filter( DataSourceGroup.group == group, @@ -741,9 +747,11 @@ def to_dict(self): def unused(cls, days=7): age_threshold = datetime.datetime.now() - datetime.timedelta(days=days) - unused_results = (db.session.query(QueryResult.id).filter( - Query.id == None, QueryResult.retrieved_at < age_threshold) - .outerjoin(Query)) + unused_results = db.session.query(QueryResult.id).filter( + QueryResult.retrieved_at < age_threshold, + Query.id == None, + ~QueryResultSet.query.filter(QueryResultSet.result_id == QueryResult.id).exists() + ).outerjoin(Query) return unused_results @@ -785,6 +793,8 @@ def store_result(cls, org, data_source, query_hash, query, data, run_time, retri for q in queries: q.latest_query_data = query_result db.session.add(q) + if q.schedule_resultset_size > 0: + q.query_results.append(query_result) query_ids = [q.id for q in queries] logging.info("Updated %s queries with result (%s).", len(query_ids), query_hash) @@ -857,13 +867,14 @@ def should_schedule_next(previous_iteration, now, schedule, failures): class Query(ChangeTrackingMixin, TimestampMixin, BelongsToOrgMixin, db.Model): id = Column(db.Integer, primary_key=True) - version = Column(db.Integer, default=1) + version = Column(db.Integer, default=0) org_id = Column(db.Integer, db.ForeignKey('organizations.id')) org = db.relationship(Organization, backref="queries") data_source_id = Column(db.Integer, db.ForeignKey("data_sources.id"), nullable=True) data_source = db.relationship(DataSource, backref='queries') latest_query_data_id = Column(db.Integer, db.ForeignKey("query_results.id"), nullable=True) latest_query_data = db.relationship(QueryResult) + query_results = db.relationship("QueryResult", secondary="query_resultsets") name = Column(db.String(255)) description = Column(db.String(4096), nullable=True) query_text = Column("query", db.Text) @@ -878,6 +889,8 @@ class Query(ChangeTrackingMixin, TimestampMixin, BelongsToOrgMixin, db.Model): is_draft = Column(db.Boolean, default=True, index=True) schedule = Column(db.String(10), nullable=True) schedule_failures = Column(db.Integer, default=0) + schedule_until = Column(db.DateTime(True), nullable=True) + schedule_resultset_size = Column(db.Integer, nullable=True) visualizations = db.relationship("Visualization", cascade="all, delete-orphan") options = Column(MutableDict.as_mutable(PseudoJSON), default={}) search_vector = Column(TSVectorType('id', 'name', 'description', 'query', @@ -998,7 +1011,9 @@ def by_user(cls, user): def outdated_queries(cls): queries = (db.session.query(Query) .options(joinedload(Query.latest_query_data).load_only('retrieved_at')) - .filter(Query.schedule != None) + .filter(Query.schedule != None, + (Query.schedule_until == None) | + (Query.schedule_until > db.func.now())) .order_by(Query.id)) now = utils.utcnow() @@ -1025,6 +1040,37 @@ def search(cls, term, group_ids, user_id=None, include_drafts=False, limit=None) # sort the result using the weight as defined in the search vector column return all_queries.search(term, sort=True).limit(limit) + @classmethod + def delete_stale_resultsets(cls): + delete_count = 0 + texts = [c[0] for c in db.session.query(Query.query_text) + .filter(Query.schedule_resultset_size != None).distinct()] + for text in texts: + queries = (Query.query.filter(Query.query_text == text, + Query.schedule_resultset_size != None) + .order_by(Query.schedule_resultset_size.desc())) + # Multiple queries with the same text may request multiple result sets + # be kept. We start with the one that keeps the most, and delete both + # the unneeded bridge rows and result sets. + first_query = queries.first() + if first_query is not None and first_query.schedule_resultset_size: + resultsets = QueryResultSet.query.filter(QueryResultSet.query_rel == first_query).order_by(QueryResultSet.result_id) + resultset_count = resultsets.count() + if resultset_count > first_query.schedule_resultset_size: + n_to_delete = resultset_count - first_query.schedule_resultset_size + r_ids = [r.result_id for r in resultsets][:n_to_delete] + QueryResultSet.query.filter(QueryResultSet.result_id.in_(r_ids)).delete(synchronize_session=False) + delete_count += QueryResult.query.filter(QueryResult.id.in_(r_ids)).delete(synchronize_session=False) + # By this point there are no stale result sets left. + # Delete unneeded bridge rows for the remaining queries. + for q in queries[1:]: + resultsets = db.session.query(QueryResultSet.result_id).filter(QueryResultSet.query_rel == q).order_by(QueryResultSet.result_id) + n_to_delete = resultsets.count() - q.schedule_resultset_size + if n_to_delete > 0: + stale_r = QueryResultSet.query.filter(QueryResultSet.result_id.in_(resultsets.limit(n_to_delete).subquery())) + stale_r.delete(synchronize_session=False) + return delete_count + @classmethod def search_by_user(cls, term, user, limit=None): return cls.by_user(user).search(term, sort=True).limit(limit) @@ -1063,6 +1109,7 @@ def fork(self, user): kwargs = {a: getattr(self, a) for a in forked_list} forked_query = Query.create(name=u'Copy of (#{}) {}'.format(self.id, self.name), user=user, **kwargs) + forked_query.record_changes(changed_by=user) for v in self.visualizations: if v.type == 'TABLE': @@ -1105,6 +1152,16 @@ def __repr__(self): return '' % (self.id, self.name or 'untitled') +class QueryResultSet(db.Model): + query_id = Column(db.Integer, db.ForeignKey("queries.id"), + primary_key=True) + query_rel = db.relationship(Query) + result_id = Column(db.Integer, db.ForeignKey("query_results.id"), + primary_key=True) + result = db.relationship(QueryResult) + __tablename__ = 'query_resultsets' + + @vectorizer(db.Integer) def integer_vectorizer(column): return db.func.cast(column, db.Text) @@ -1236,7 +1293,6 @@ def to_dict(self, full=True): 'id': self.id, 'object_id': self.object_id, 'object_type': self.object_type, - 'change_type': self.change_type, 'object_version': self.object_version, 'change': self.change, 'created_at': self.created_at @@ -1256,6 +1312,12 @@ def last_change(cls, obj): cls.object_type == obj.__class__.__tablename__).order_by( cls.object_version.desc()).first() + @classmethod + def list_versions(cls, query): + return cls.query.filter( + cls.object_id == query.id, + cls.object_type == 'queries') + class Alert(TimestampMixin, db.Model): UNKNOWN_STATE = 'unknown' @@ -1347,7 +1409,7 @@ class Dashboard(ChangeTrackingMixin, TimestampMixin, BelongsToOrgMixin, db.Model } @classmethod - def all(cls, org, group_ids, user_id): + def all(cls, org, group_ids, user_id, include_drafts=False): query = ( Dashboard.query .options(joinedload(Dashboard.user)) @@ -1363,14 +1425,14 @@ def all(cls, org, group_ids, user_id): Dashboard.org == org) .distinct()) - query = query.filter(or_(Dashboard.user_id == user_id, Dashboard.is_draft == False)) + query = query.filter(or_(Dashboard.user_id == user_id, Dashboard.is_draft == include_drafts)) return query @classmethod - def search(cls, org, groups_ids, user_id, search_term): + def search(cls, org, groups_ids, user_id, search_term, include_drafts): # TODO: switch to FTS - return cls.all(org, groups_ids, user_id).filter(cls.name.ilike(u'%{}%'.format(search_term))) + return cls.all(org, groups_ids, user_id, include_drafts).filter(cls.name.ilike(u'%{}%'.format(search_term))) @classmethod def all_tags(cls, org, user): diff --git a/redash/monitor.py b/redash/monitor.py index f1f241eb26..41fdc0ddb5 100644 --- a/redash/monitor.py +++ b/redash/monitor.py @@ -35,7 +35,7 @@ def get_queues_status(): 'data_sources': ', '.join(sources), 'size': redis_connection.llen(queue) } - + queues['celery'] = { 'size': redis_connection.llen('celery'), 'data_sources': '' diff --git a/redash/query_runner/__init__.py b/redash/query_runner/__init__.py index 1a5357dbad..81900351d8 100644 --- a/redash/query_runner/__init__.py +++ b/redash/query_runner/__init__.py @@ -1,14 +1,16 @@ -import sys import logging import json +import sys + +import requests -from collections import OrderedDict from redash import settings logger = logging.getLogger(__name__) __all__ = [ 'BaseQueryRunner', + 'BaseHTTPQueryRunner', 'InterruptException', 'BaseSQLQueryRunner', 'TYPE_DATETIME', @@ -51,6 +53,7 @@ class NotSupported(Exception): class BaseQueryRunner(object): noop_query = None + configuration_properties = None def __init__(self, configuration): self.syntax = 'sql' @@ -76,6 +79,12 @@ def annotate_query(cls): def configuration_schema(cls): return {} + @classmethod + def add_configuration_property(cls, property, value): + if cls.configuration_properties is None: + raise NotImplementedError() + cls.configuration_properties[property] = value + def test_connection(self): if self.noop_query is None: raise NotImplementedError() @@ -143,6 +152,103 @@ def _get_tables_stats(self, tables_dict): tables_dict[t]['size'] = res[0]['cnt'] +class BaseHTTPQueryRunner(BaseQueryRunner): + response_error = "Endpoint returned unexpected status code" + requires_authentication = False + url_title = 'URL base path' + username_title = 'HTTP Basic Auth Username' + password_title = 'HTTP Basic Auth Password' + configuration_properties = { + 'url': { + 'type': 'string', + 'title': url_title, + }, + 'username': { + 'type': 'string', + 'title': username_title, + }, + 'password': { + 'type': 'string', + 'title': password_title, + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": ( + "This string will be used to toggle visibility of " + "tables in the schema browser when editing a query " + "in order to remove non-useful tables from sight." + ), + } + } + + @classmethod + def configuration_schema(cls): + schema = { + 'type': 'object', + 'properties': cls.configuration_properties, + 'required': ['url'], + 'secret': ['password'] + } + if cls.requires_authentication: + schema['required'] += ['username', 'password'] + return schema + + def get_auth(self): + username = self.configuration.get('username') + password = self.configuration.get('password') + if username and password: + return (username, password) + if self.requires_authentication: + raise ValueError("Username and Password required") + else: + return None + + def get_response(self, url, auth=None, **kwargs): + # Get authentication values if not given + if auth is None: + auth = self.get_auth() + + # Then call requests to get the response from the given endpoint + # URL optionally, with the additional requests parameters. + error = None + response = None + try: + response = requests.get(url, auth=auth, **kwargs) + # Raise a requests HTTP exception with the appropriate reason + # for 4xx and 5xx response status codes which is later caught + # and passed back. + response.raise_for_status() + + # Any other responses (e.g. 2xx and 3xx): + if response.status_code != 200: + error = '{} ({}).'.format( + self.response_error, + response.status_code, + ) + + except requests.HTTPError as exc: + logger.exception(exc) + error = ( + "Failed to execute query. " + "Return Code: {} Reason: {}".format( + response.status_code, + response.text + ) + ) + except requests.RequestException as exc: + # Catch all other requests exceptions and return the error. + logger.exception(exc) + error = str(exc) + except Exception as exc: + # Catch any other exceptions, log it and reraise it. + logger.exception(exc) + raise sys.exc_info()[1], None, sys.exc_info()[2] + + return response, error + + query_runners = {} diff --git a/redash/query_runner/athena.py b/redash/query_runner/athena.py index a07952934f..57d3a092a2 100644 --- a/redash/query_runner/athena.py +++ b/redash/query_runner/athena.py @@ -80,6 +80,12 @@ def configuration_schema(cls): 'type': 'boolean', 'title': 'Use Glue Data Catalog', }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } }, 'required': ['region', 's3_staging_dir'], 'order': ['region', 'aws_access_key', 'aws_secret_key', 's3_staging_dir', 'schema'], @@ -143,9 +149,10 @@ def get_schema(self, get_stats=False): schema = {} query = """ - SELECT table_schema, table_name, column_name + SELECT table_schema, table_name, column_name, data_type as column_type, comment as extra_info FROM information_schema.columns WHERE table_schema NOT IN ('information_schema') + ORDER BY 1, 5 DESC """ results, error = self.run_query(query, None) @@ -157,7 +164,16 @@ def get_schema(self, get_stats=False): table_name = '{0}.{1}'.format(row['table_schema'], row['table_name']) if table_name not in schema: schema[table_name] = {'name': table_name, 'columns': []} - schema[table_name]['columns'].append(row['column_name']) + + if row['extra_info'] == 'Partition Key': + schema[table_name]['columns'].append('[P] ' + row['column_name'] + ' (' + row['column_type'] + ')') + elif row['column_type'] == 'integer' or row['column_type'] == 'varchar' or row['column_type'] == 'timestamp' or row['column_type'] == 'boolean' or row['column_type'] == 'bigint': + schema[table_name]['columns'].append(row['column_name'] + ' (' + row['column_type'] + ')') + elif row['column_type'][0:2] == 'row' or row['column_type'][0:2] == 'map' or row['column_type'][0:2] == 'arr': + schema[table_name]['columns'].append(row['column_name'] + ' (row or map or array)') + else: + schema[table_name]['columns'].append(row['column_name']) + return schema.values() diff --git a/redash/query_runner/axibase_tsd.py b/redash/query_runner/axibase_tsd.py index 9737b6bc87..9d50cd3b26 100644 --- a/redash/query_runner/axibase_tsd.py +++ b/redash/query_runner/axibase_tsd.py @@ -133,6 +133,12 @@ def configuration_schema(cls): 'trust_certificate': { 'type': 'boolean', 'title': 'Trust SSL Certificate' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, 'required': ['username', 'password', 'hostname', 'protocol', 'port'], diff --git a/redash/query_runner/big_query.py b/redash/query_runner/big_query.py index 7e44661d2d..a35ca86206 100644 --- a/redash/query_runner/big_query.py +++ b/redash/query_runner/big_query.py @@ -83,6 +83,47 @@ def _get_query_results(jobs, project_id, location, job_id, start_index): class BigQuery(BaseQueryRunner): noop_query = "SELECT 1" + configuration_properties = { + 'projectId': { + 'type': 'string', + 'title': 'Project ID' + }, + 'jsonKeyFile': { + "type": "string", + 'title': 'JSON Key File' + }, + 'totalMBytesProcessedLimit': { + "type": "number", + 'title': 'Scanned Data Limit (MB)' + }, + 'userDefinedFunctionResourceUri': { + "type": "string", + 'title': 'UDF Source URIs (i.e. gs://bucket/date_utils.js, gs://bucket/string_utils.js )' + }, + 'useStandardSql': { + "type": "boolean", + 'title': "Use Standard SQL (Beta)", + }, + 'location': { + "type": "string", + "title": "Processing Location", + "default": "US", + }, + 'loadSchema': { + "type": "boolean", + "title": "Load Schema" + }, + 'maximumBillingTier': { + "type": "number", + "title": "Maximum Billing Tier" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } @classmethod def enabled(cls): @@ -92,41 +133,7 @@ def enabled(cls): def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'projectId': { - 'type': 'string', - 'title': 'Project ID' - }, - 'jsonKeyFile': { - "type": "string", - 'title': 'JSON Key File' - }, - 'totalMBytesProcessedLimit': { - "type": "number", - 'title': 'Scanned Data Limit (MB)' - }, - 'userDefinedFunctionResourceUri': { - "type": "string", - 'title': 'UDF Source URIs (i.e. gs://bucket/date_utils.js, gs://bucket/string_utils.js )' - }, - 'useStandardSql': { - "type": "boolean", - 'title': "Use Standard SQL (Beta)", - }, - 'location': { - "type": "string", - "title": "Processing Location", - "default": "US", - }, - 'loadSchema': { - "type": "boolean", - "title": "Load Schema" - }, - 'maximumBillingTier': { - "type": "number", - "title": "Maximum Billing Tier" - } - }, + 'properties': cls.configuration_properties, 'required': ['jsonKeyFile', 'projectId'], "order": ['projectId', 'jsonKeyFile', 'loadSchema', 'useStandardSql', 'location', 'totalMBytesProcessedLimit', 'maximumBillingTier', 'userDefinedFunctionResourceUri'], 'secret': ['jsonKeyFile'] diff --git a/redash/query_runner/cass.py b/redash/query_runner/cass.py index 37bcc98cbb..eca59fda64 100644 --- a/redash/query_runner/cass.py +++ b/redash/query_runner/cass.py @@ -27,6 +27,43 @@ def default(self, o): class Cassandra(BaseQueryRunner): noop_query = "SELECT dateof(now()) FROM system.local" + configuration_properties = { + 'host': { + 'type': 'string', + }, + 'port': { + 'type': 'number', + 'default': 9042, + }, + 'keyspace': { + 'type': 'string', + 'title': 'Keyspace name' + }, + 'username': { + 'type': 'string', + 'title': 'Username' + }, + 'password': { + 'type': 'string', + 'title': 'Password' + }, + 'protocol': { + 'type': 'number', + 'title': 'Protocol Version', + 'default': 3 + }, + 'timeout': { + 'type': 'number', + 'title': 'Timeout', + 'default': 10 + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } @classmethod def enabled(cls): @@ -36,37 +73,7 @@ def enabled(cls): def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'host': { - 'type': 'string', - }, - 'port': { - 'type': 'number', - 'default': 9042, - }, - 'keyspace': { - 'type': 'string', - 'title': 'Keyspace name' - }, - 'username': { - 'type': 'string', - 'title': 'Username' - }, - 'password': { - 'type': 'string', - 'title': 'Password' - }, - 'protocol': { - 'type': 'number', - 'title': 'Protocol Version', - 'default': 3 - }, - 'timeout': { - 'type': 'number', - 'title': 'Timeout', - 'default': 10 - } - }, + 'properties': cls.configuration_properties, 'required': ['keyspace', 'host'] } diff --git a/redash/query_runner/clickhouse.py b/redash/query_runner/clickhouse.py index 00fc2b578b..93c7cd6423 100644 --- a/redash/query_runner/clickhouse.py +++ b/redash/query_runner/clickhouse.py @@ -29,6 +29,12 @@ def configuration_schema(cls): "dbname": { "type": "string", "title": "Database Name" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, "required": ["dbname"], diff --git a/redash/query_runner/dynamodb_sql.py b/redash/query_runner/dynamodb_sql.py index 6c7a980326..ccd071b5e5 100644 --- a/redash/query_runner/dynamodb_sql.py +++ b/redash/query_runner/dynamodb_sql.py @@ -33,22 +33,31 @@ class DynamoDBSQL(BaseSQLQueryRunner): + noop_query = "SELECT 1" + configuration_properties = { + "region": { + "type": "string", + "default": "us-east-1" + }, + "access_key": { + "type": "string", + }, + "secret_key": { + "type": "string", + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } + @classmethod def configuration_schema(cls): return { "type": "object", - "properties": { - "region": { - "type": "string", - "default": "us-east-1" - }, - "access_key": { - "type": "string", - }, - "secret_key": { - "type": "string", - } - }, + "properties": cls.configuration_properties, "required": ["access_key", "secret_key"], "secret": ["secret_key"] } diff --git a/redash/query_runner/elasticsearch.py b/redash/query_runner/elasticsearch.py index 7b2a06ec37..f6268f42d7 100644 --- a/redash/query_runner/elasticsearch.py +++ b/redash/query_runner/elasticsearch.py @@ -45,25 +45,32 @@ class BaseElasticSearch(BaseQueryRunner): DEBUG_ENABLED = False + configuration_properties = { + 'server': { + 'type': 'string', + 'title': 'Base URL' + }, + 'basic_auth_user': { + 'type': 'string', + 'title': 'Basic Auth User' + }, + 'basic_auth_password': { + 'type': 'string', + 'title': 'Basic Auth Password' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } @classmethod def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'server': { - 'type': 'string', - 'title': 'Base URL' - }, - 'basic_auth_user': { - 'type': 'string', - 'title': 'Basic Auth User' - }, - 'basic_auth_password': { - 'type': 'string', - 'title': 'Basic Auth Password' - } - }, + 'properties': cls.configuration_properties, "secret": ["basic_auth_password"], "required": ["server"] } diff --git a/redash/query_runner/google_analytics.py b/redash/query_runner/google_analytics.py index cd14724b66..eb6457624f 100644 --- a/redash/query_runner/google_analytics.py +++ b/redash/query_runner/google_analytics.py @@ -103,6 +103,12 @@ def configuration_schema(cls): 'jsonKeyFile': { "type": "string", 'title': 'JSON Key File' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, 'required': ['jsonKeyFile'], diff --git a/redash/query_runner/google_spreadsheets.py b/redash/query_runner/google_spreadsheets.py index 61b7a62d5d..1071437971 100644 --- a/redash/query_runner/google_spreadsheets.py +++ b/redash/query_runner/google_spreadsheets.py @@ -148,6 +148,19 @@ def request(self, *args, **kwargs): class GoogleSpreadsheet(BaseQueryRunner): + configuration_properties = { + 'jsonKeyFile': { + "type": "string", + 'title': 'JSON Key File' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } + @classmethod def annotate_query(cls): return False @@ -164,12 +177,7 @@ def enabled(cls): def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'jsonKeyFile': { - "type": "string", - 'title': 'JSON Key File' - } - }, + 'properties': cls.configuration_properties, 'required': ['jsonKeyFile'], 'secret': ['jsonKeyFile'] } diff --git a/redash/query_runner/graphite.py b/redash/query_runner/graphite.py index 023ec04940..217bd55a16 100644 --- a/redash/query_runner/graphite.py +++ b/redash/query_runner/graphite.py @@ -42,6 +42,12 @@ def configuration_schema(cls): 'verify': { 'type': 'boolean', 'title': 'Verify SSL certificate' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, 'required': ['url'], diff --git a/redash/query_runner/hive_ds.py b/redash/query_runner/hive_ds.py index 2d3aa4a303..a77759a15b 100644 --- a/redash/query_runner/hive_ds.py +++ b/redash/query_runner/hive_ds.py @@ -36,25 +36,32 @@ class Hive(BaseSQLQueryRunner): noop_query = "SELECT 1" + configuration_properties = { + "host": { + "type": "string" + }, + "port": { + "type": "number" + }, + "database": { + "type": "string" + }, + "username": { + "type": "string" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } @classmethod def configuration_schema(cls): return { "type": "object", - "properties": { - "host": { - "type": "string" - }, - "port": { - "type": "number" - }, - "database": { - "type": "string" - }, - "username": { - "type": "string" - } - }, + "properties": cls.configuration_properties, "required": ["host"] } diff --git a/redash/query_runner/impala_ds.py b/redash/query_runner/impala_ds.py index 0f412ffac6..cb6dc2fb1e 100644 --- a/redash/query_runner/impala_ds.py +++ b/redash/query_runner/impala_ds.py @@ -36,38 +36,45 @@ class Impala(BaseSQLQueryRunner): noop_query = "show schemas" + configuration_properties = { + "host": { + "type": "string" + }, + "port": { + "type": "number" + }, + "protocol": { + "type": "string", + "title": "Please specify beeswax or hiveserver2" + }, + "database": { + "type": "string" + }, + "use_ldap": { + "type": "boolean" + }, + "ldap_user": { + "type": "string" + }, + "ldap_password": { + "type": "string" + }, + "timeout": { + "type": "number" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } @classmethod def configuration_schema(cls): return { "type": "object", - "properties": { - "host": { - "type": "string" - }, - "port": { - "type": "number" - }, - "protocol": { - "type": "string", - "title": "Please specify beeswax or hiveserver2" - }, - "database": { - "type": "string" - }, - "use_ldap": { - "type": "boolean" - }, - "ldap_user": { - "type": "string" - }, - "ldap_password": { - "type": "string" - }, - "timeout": { - "type": "number" - } - }, + "properties": cls.configuration_properties, "required": ["host"], "secret": ["ldap_password"] } diff --git a/redash/query_runner/influx_db.py b/redash/query_runner/influx_db.py index 5d830daa46..8ae0a24ab2 100644 --- a/redash/query_runner/influx_db.py +++ b/redash/query_runner/influx_db.py @@ -50,16 +50,23 @@ def _transform_result(results): class InfluxDB(BaseQueryRunner): noop_query = "show measurements limit 1" + configuration_properties = { + 'url': { + 'type': 'string' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } @classmethod def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'url': { - 'type': 'string' - } - }, + 'properties': cls.configuration_properties, 'required': ['url'] } diff --git a/redash/query_runner/jql.py b/redash/query_runner/jql.py index 37b1f345c6..9beb7c53ae 100644 --- a/redash/query_runner/jql.py +++ b/redash/query_runner/jql.py @@ -1,5 +1,4 @@ import json -import requests import re from collections import OrderedDict @@ -137,28 +136,13 @@ def get_dict_output_field_name(cls,field_name, member_name): return None -class JiraJQL(BaseQueryRunner): +class JiraJQL(BaseHTTPQueryRunner): noop_query = '{"queryType": "count"}' - - @classmethod - def configuration_schema(cls): - return { - 'type': 'object', - 'properties': { - 'url': { - 'type': 'string', - 'title': 'JIRA URL' - }, - 'username': { - 'type': 'string', - }, - 'password': { - 'type': 'string' - } - }, - 'required': ['url', 'username', 'password'], - 'secret': ['password'] - } + response_error = "JIRA returned unexpected status code" + requires_authentication = True + url_title = 'JIRA URL' + username_title = 'Username' + password_title = 'Password' @classmethod def name(cls): @@ -186,13 +170,9 @@ def run_query(self, query, user): else: query['maxResults'] = query.get('maxResults', 1000) - response = requests.get(jql_url, params=query, auth=(self.configuration.get('username'), self.configuration.get('password'))) - - if response.status_code == 401 or response.status_code == 403: - return None, "Authentication error. Please check username/password." - - if response.status_code != 200: - return None, "JIRA returned unexpected status code ({})".format(response.status_code) + response, error = self.get_response(jql_url, params=query) + if error is not None: + return None, error data = response.json() @@ -206,4 +186,3 @@ def run_query(self, query, user): return None, "Query cancelled by user." register(JiraJQL) - diff --git a/redash/query_runner/memsql_ds.py b/redash/query_runner/memsql_ds.py index 66e9eddc3d..d54211ab25 100644 --- a/redash/query_runner/memsql_ds.py +++ b/redash/query_runner/memsql_ds.py @@ -56,6 +56,12 @@ def configuration_schema(cls): }, "password": { "type": "string" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, diff --git a/redash/query_runner/mongodb.py b/redash/query_runner/mongodb.py index 91323a8a19..dd920c5455 100644 --- a/redash/query_runner/mongodb.py +++ b/redash/query_runner/mongodb.py @@ -118,24 +118,32 @@ def parse_results(results): class MongoDB(BaseQueryRunner): + configuration_properties = { + 'connectionString': { + 'type': 'string', + 'title': 'Connection String' + }, + 'dbName': { + 'type': 'string', + 'title': "Database Name" + }, + 'replicaSetName': { + 'type': 'string', + 'title': 'Replica Set Name' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } + @classmethod def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'connectionString': { - 'type': 'string', - 'title': 'Connection String' - }, - 'dbName': { - 'type': 'string', - 'title': "Database Name" - }, - 'replicaSetName': { - 'type': 'string', - 'title': 'Replica Set Name' - }, - }, + 'properties': cls.configuration_properties, 'required': ['connectionString', 'dbName'] } diff --git a/redash/query_runner/mssql.py b/redash/query_runner/mssql.py index 57ee300a9c..50e38ba1d7 100644 --- a/redash/query_runner/mssql.py +++ b/redash/query_runner/mssql.py @@ -35,41 +35,48 @@ def default(self, o): class SqlServer(BaseSQLQueryRunner): noop_query = "SELECT 1" + configuration_properties = { + "user": { + "type": "string" + }, + "password": { + "type": "string" + }, + "server": { + "type": "string", + "default": "127.0.0.1" + }, + "port": { + "type": "number", + "default": 1433 + }, + "tds_version": { + "type": "string", + "default": "7.0", + "title": "TDS Version" + }, + "charset": { + "type": "string", + "default": "UTF-8", + "title": "Character Set" + }, + "db": { + "type": "string", + "title": "Database Name" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } @classmethod def configuration_schema(cls): return { "type": "object", - "properties": { - "user": { - "type": "string" - }, - "password": { - "type": "string" - }, - "server": { - "type": "string", - "default": "127.0.0.1" - }, - "port": { - "type": "number", - "default": 1433 - }, - "tds_version": { - "type": "string", - "default": "7.0", - "title": "TDS Version" - }, - "charset": { - "type": "string", - "default": "UTF-8", - "title": "Character Set" - }, - "db": { - "type": "string", - "title": "Database Name" - } - }, + "properties": cls.configuration_properties, "required": ["db"], "secret": ["password"] } diff --git a/redash/query_runner/mysql.py b/redash/query_runner/mysql.py index fafa5edf16..35babdc3b8 100644 --- a/redash/query_runner/mysql.py +++ b/redash/query_runner/mysql.py @@ -29,6 +29,33 @@ class Mysql(BaseSQLQueryRunner): noop_query = "SELECT 1" + configuration_properties = { + 'host': { + 'type': 'string', + 'default': '127.0.0.1' + }, + 'user': { + 'type': 'string' + }, + 'passwd': { + 'type': 'string', + 'title': 'Password' + }, + 'db': { + 'type': 'string', + 'title': 'Database name' + }, + 'port': { + 'type': 'number', + 'default': 3306, + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } @classmethod def configuration_schema(cls): @@ -36,27 +63,7 @@ def configuration_schema(cls): schema = { 'type': 'object', - 'properties': { - 'host': { - 'type': 'string', - 'default': '127.0.0.1' - }, - 'user': { - 'type': 'string' - }, - 'passwd': { - 'type': 'string', - 'title': 'Password' - }, - 'db': { - 'type': 'string', - 'title': 'Database name' - }, - 'port': { - 'type': 'number', - 'default': 3306, - } - }, + 'properties': cls.configuration_properties, "order": ['host', 'port', 'user', 'passwd', 'db'], 'required': ['db'], 'secret': ['passwd'] @@ -79,7 +86,7 @@ def configuration_schema(cls): 'ssl_key': { 'type': 'string', 'title': 'Path to private key file (SSL)' - } + }, }) return schema @@ -91,7 +98,7 @@ def name(cls): @classmethod def enabled(cls): try: - import MySQLdb + import pymysql except ImportError: return False @@ -101,7 +108,8 @@ def _get_tables(self, schema): query = """ SELECT col.table_schema, col.table_name, - col.column_name + col.column_name, + col.column_type FROM `information_schema`.`columns` col WHERE col.table_schema NOT IN ('information_schema', 'performance_schema', 'mysql'); """ @@ -122,16 +130,16 @@ def _get_tables(self, schema): if table_name not in schema: schema[table_name] = {'name': table_name, 'columns': []} - schema[table_name]['columns'].append(row['column_name']) + schema[table_name]['columns'].append(row['column_name'] + ' (' + row['column_type'] + ')') return schema.values() def run_query(self, query, user): - import MySQLdb + import pymysql connection = None try: - connection = MySQLdb.connect(host=self.configuration.get('host', ''), + connection = pymysql.connect(host=self.configuration.get('host', ''), user=self.configuration.get('user', ''), passwd=self.configuration.get('passwd', ''), db=self.configuration['db'], @@ -161,7 +169,7 @@ def run_query(self, query, user): error = "No data was returned." cursor.close() - except MySQLdb.Error as e: + except pymysql.Error as e: json_data = None error = e.args[1] except KeyboardInterrupt: diff --git a/redash/query_runner/oracle.py b/redash/query_runner/oracle.py index 5bb8f70f2f..489ad9a849 100644 --- a/redash/query_runner/oracle.py +++ b/redash/query_runner/oracle.py @@ -31,8 +31,33 @@ logger = logging.getLogger(__name__) + class Oracle(BaseSQLQueryRunner): noop_query = "SELECT 1 FROM dual" + configuration_properties = { + "user": { + "type": "string" + }, + "password": { + "type": "string" + }, + "host": { + "type": "string" + }, + "port": { + "type": "number" + }, + "servicename": { + "type": "string", + "title": "DSN Service Name" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } @classmethod def get_col_type(cls, col_type, scale): @@ -49,24 +74,7 @@ def enabled(cls): def configuration_schema(cls): return { "type": "object", - "properties": { - "user": { - "type": "string" - }, - "password": { - "type": "string" - }, - "host": { - "type": "string" - }, - "port": { - "type": "number" - }, - "servicename": { - "type": "string", - "title": "DSN Service Name" - } - }, + "properties": cls.configuration_properties, "required": ["servicename", "user", "password", "host", "port"], "secret": ["password"] } diff --git a/redash/query_runner/pg.py b/redash/query_runner/pg.py index 348f5c925b..8590846e69 100644 --- a/redash/query_runner/pg.py +++ b/redash/query_runner/pg.py @@ -47,36 +47,43 @@ def _wait(conn, timeout=None): class PostgreSQL(BaseSQLQueryRunner): noop_query = "SELECT 1" + configuration_properties = { + "user": { + "type": "string" + }, + "password": { + "type": "string" + }, + "host": { + "type": "string", + "default": "127.0.0.1" + }, + "port": { + "type": "number", + "default": 5432 + }, + "dbname": { + "type": "string", + "title": "Database Name" + }, + "sslmode": { + "type": "string", + "title": "SSL Mode", + "default": "prefer" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } @classmethod def configuration_schema(cls): return { "type": "object", - "properties": { - "user": { - "type": "string" - }, - "password": { - "type": "string" - }, - "host": { - "type": "string", - "default": "127.0.0.1" - }, - "port": { - "type": "number", - "default": 5432 - }, - "dbname": { - "type": "string", - "title": "Database Name" - }, - "sslmode": { - "type": "string", - "title": "SSL Mode", - "default": "prefer" - } - }, + "properties": cls.configuration_properties, "order": ['host', 'port', 'user', 'password'], "required": ["dbname"], "secret": ["password"] @@ -103,7 +110,7 @@ def _get_definitions(self, schema, query): if table_name not in schema: schema[table_name] = {'name': table_name, 'columns': []} - schema[table_name]['columns'].append(row['column_name']) + schema[table_name]['columns'].append(row['column_name'] + ' (' + row['column_type'] + ')') def _get_tables(self, schema): ''' @@ -123,6 +130,7 @@ def _get_tables(self, schema): query = """ SELECT s.nspname as table_schema, c.relname as table_name, + t.typname as column_type, a.attname as column_name FROM pg_class c JOIN pg_namespace s @@ -132,6 +140,8 @@ def _get_tables(self, schema): ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped + JOIN pg_type t + ON c.reltype = t.oid WHERE c.relkind IN ('r', 'v', 'm', 'f', 'p') """ @@ -187,6 +197,30 @@ def run_query(self, query, user): class Redshift(PostgreSQL): + configuration_properties = { + "user": { + "type": "string" + }, + "password": { + "type": "string" + }, + "host": { + "type": "string" + }, + "port": { + "type": "number" + }, + "dbname": { + "type": "string", + "title": "Database Name" + }, + "sslmode": { + "type": "string", + "title": "SSL Mode", + "default": "prefer" + } + } + @classmethod def type(cls): return "redshift" @@ -210,29 +244,7 @@ def configuration_schema(cls): return { "type": "object", - "properties": { - "user": { - "type": "string" - }, - "password": { - "type": "string" - }, - "host": { - "type": "string" - }, - "port": { - "type": "number" - }, - "dbname": { - "type": "string", - "title": "Database Name" - }, - "sslmode": { - "type": "string", - "title": "SSL Mode", - "default": "prefer" - } - }, + "properties": cls.configuration_properties, "order": ['host', 'port', 'user', 'password'], "required": ["dbname", "user", "password", "host", "port"], "secret": ["password"] diff --git a/redash/query_runner/presto.py b/redash/query_runner/presto.py index e915fa9a2e..c7beb13409 100644 --- a/redash/query_runner/presto.py +++ b/redash/query_runner/presto.py @@ -1,4 +1,5 @@ import json +from markupsafe import Markup, escape from redash.utils import JSONEncoder from redash.query_runner import * @@ -33,28 +34,35 @@ class Presto(BaseQueryRunner): noop_query = 'SHOW TABLES' + configuration_properties = { + 'host': { + 'type': 'string' + }, + 'port': { + 'type': 'number' + }, + 'schema': { + 'type': 'string' + }, + 'catalog': { + 'type': 'string' + }, + 'username': { + 'type': 'string' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } @classmethod def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'host': { - 'type': 'string' - }, - 'port': { - 'type': 'number' - }, - 'schema': { - 'type': 'string' - }, - 'catalog': { - 'type': 'string' - }, - 'username': { - 'type': 'string' - } - }, + 'properties': cls.configuration_properties, 'required': ['host'] } @@ -69,9 +77,10 @@ def type(cls): def get_schema(self, get_stats=False): schema = {} query = """ - SELECT table_schema, table_name, column_name + SELECT table_schema, table_name, column_name, data_type as column_type, extra_info FROM information_schema.columns WHERE table_schema NOT IN ('pg_catalog', 'information_schema') + ORDER BY 1, 5 DESC """ results, error = self.run_query(query, None) @@ -87,7 +96,14 @@ def get_schema(self, get_stats=False): if table_name not in schema: schema[table_name] = {'name': table_name, 'columns': []} - schema[table_name]['columns'].append(row['column_name']) + if row['extra_info'] == 'partition key': + schema[table_name]['columns'].append('[P] ' + row['column_name'] + ' (' + row['column_type'] + ')') + elif row['column_type'] == 'integer' or row['column_type'] == 'varchar' or row['column_type'] == 'timestamp' or row['column_type'] == 'boolean' or row['column_type'] == 'bigint': + schema[table_name]['columns'].append(row['column_name'] + ' (' + row['column_type'] + ')') + elif row['column_type'][0:2] == 'row' or row['column_type'][0:2] == 'map' or row['column_type'][0:2] == 'arr': + schema[table_name]['columns'].append(row['column_name'] + ' (row or map or array)') + else: + schema[table_name]['columns'].append(row['column_name']) return schema.values() @@ -107,6 +123,9 @@ def run_query(self, query, user): column_tuples = [(i[0], PRESTO_TYPES_MAPPING.get(i[1], None)) for i in cursor.description] columns = self.fetch_columns(column_tuples) rows = [dict(zip(([c['name'] for c in columns]), r)) for i, r in enumerate(cursor.fetchall())] + for row in rows: + for field in row: + field = escape(field) data = {'columns': columns, 'rows': rows} json_data = json.dumps(data, cls=JSONEncoder) error = None diff --git a/redash/query_runner/python.py b/redash/query_runner/python.py index f8e69f96ce..f3d24d5064 100644 --- a/redash/query_runner/python.py +++ b/redash/query_runner/python.py @@ -46,19 +46,27 @@ class Python(BaseQueryRunner): 'tuple', 'set', 'list', 'dict', 'bool', ) + configuration_properties = { + 'allowedImportModules': { + 'type': 'string', + 'title': 'Modules to import prior to running the script' + }, + 'additionalModulesPaths': { + 'type': 'string' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } + @classmethod def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'allowedImportModules': { - 'type': 'string', - 'title': 'Modules to import prior to running the script' - }, - 'additionalModulesPaths': { - 'type': 'string' - } - }, + 'properties': cls.configuration_properties } @classmethod diff --git a/redash/query_runner/salesforce.py b/redash/query_runner/salesforce.py index 527f1e26ec..7222028fd0 100644 --- a/redash/query_runner/salesforce.py +++ b/redash/query_runner/salesforce.py @@ -81,6 +81,12 @@ def configuration_schema(cls): "type": "string", "title": "Salesforce API Version", "default": DEFAULT_API_VERSION + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, "required": ["username", "password", "token"], diff --git a/redash/query_runner/script.py b/redash/query_runner/script.py index ea54362d57..a55e54cfec 100644 --- a/redash/query_runner/script.py +++ b/redash/query_runner/script.py @@ -29,6 +29,23 @@ def run_script(script, shell): class Script(BaseQueryRunner): + configuration_properties = { + 'path': { + 'type': 'string', + 'title': 'Scripts path' + }, + 'shell': { + 'type': 'boolean', + 'title': 'Execute command through the shell' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } + @classmethod def annotate_query(cls): return False @@ -41,16 +58,7 @@ def enabled(cls): def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'path': { - 'type': 'string', - 'title': 'Scripts path' - }, - 'shell': { - 'type': 'boolean', - 'title': 'Execute command through the shell' - } - }, + 'properties': cls.configuration_properties, 'required': ['path'] } diff --git a/redash/query_runner/snowflake.py b/redash/query_runner/snowflake.py index a1a7ca447e..f223f54c9c 100644 --- a/redash/query_runner/snowflake.py +++ b/redash/query_runner/snowflake.py @@ -46,6 +46,12 @@ def configuration_schema(cls): }, "database": { "type": "string" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, "required": ["user", "password", "account", "database", "warehouse"], diff --git a/redash/query_runner/sqlite.py b/redash/query_runner/sqlite.py index 2bab1f27c4..4d2ef4bb9d 100644 --- a/redash/query_runner/sqlite.py +++ b/redash/query_runner/sqlite.py @@ -13,17 +13,24 @@ class Sqlite(BaseSQLQueryRunner): noop_query = "pragma quick_check" + configuration_properties = { + "dbpath": { + "type": "string", + "title": "Database Path" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } @classmethod def configuration_schema(cls): return { "type": "object", - "properties": { - "dbpath": { - "type": "string", - "title": "Database Path" - } - }, + "properties": cls.configuration_properties, "required": ["dbpath"], } diff --git a/redash/query_runner/treasuredata.py b/redash/query_runner/treasuredata.py index 0ecfacd25d..8aa99fc3a3 100644 --- a/redash/query_runner/treasuredata.py +++ b/redash/query_runner/treasuredata.py @@ -36,31 +36,38 @@ class TreasureData(BaseQueryRunner): noop_query = "SELECT 1" + configuration_properties = { + 'endpoint': { + 'type': 'string' + }, + 'apikey': { + 'type': 'string' + }, + 'type': { + 'type': 'string' + }, + 'db': { + 'type': 'string', + 'title': 'Database Name' + }, + 'get_schema': { + 'type': 'boolean', + 'title': 'Auto Schema Retrieval', + 'default': False + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } @classmethod def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'endpoint': { - 'type': 'string' - }, - 'apikey': { - 'type': 'string' - }, - 'type': { - 'type': 'string' - }, - 'db': { - 'type': 'string', - 'title': 'Database Name' - }, - 'get_schema': { - 'type': 'boolean', - 'title': 'Auto Schema Retrieval', - 'default': False - } - }, + 'properties': cls.configuration_properties, 'required': ['apikey','db'] } diff --git a/redash/query_runner/url.py b/redash/query_runner/url.py index 8763b63ed2..26dd2002a6 100644 --- a/redash/query_runner/url.py +++ b/redash/query_runner/url.py @@ -1,20 +1,7 @@ -import requests -from redash.query_runner import BaseQueryRunner, register +from redash.query_runner import BaseHTTPQueryRunner, register -class Url(BaseQueryRunner): - @classmethod - def configuration_schema(cls): - return { - 'type': 'object', - 'properties': { - 'url': { - 'type': 'string', - 'title': 'URL base path' - } - } - } - +class Url(BaseHTTPQueryRunner): @classmethod def annotate_query(cls): return False @@ -26,7 +13,6 @@ def run_query(self, query, user): base_url = self.configuration.get("url", None) try: - error = None query = query.strip() if base_url is not None and base_url != "": @@ -38,20 +24,17 @@ def run_query(self, query, user): url = base_url + query - response = requests.get(url) - response.raise_for_status() - json_data = response.content.strip() + response, error = self.get_response(url) + if error is not None: + return None, error - if not json_data: - error = "Got empty response from '{}'.".format(url) + json_data = response.content.strip() - return json_data, error - except requests.RequestException as e: - return None, str(e) + if json_data: + return json_data, None + else: + return None, "Got empty response from '{}'.".format(url) except KeyboardInterrupt: - error = "Query cancelled by user." - json_data = None - - return json_data, error + return None, "Query cancelled by user." register(Url) diff --git a/redash/query_runner/vertica.py b/redash/query_runner/vertica.py index 05dd3b1ea0..5d9ef73621 100644 --- a/redash/query_runner/vertica.py +++ b/redash/query_runner/vertica.py @@ -30,34 +30,41 @@ class Vertica(BaseSQLQueryRunner): noop_query = "SELECT 1" + configuration_properties = { + 'host': { + 'type': 'string' + }, + 'user': { + 'type': 'string' + }, + 'password': { + 'type': 'string', + 'title': 'Password' + }, + 'database': { + 'type': 'string', + 'title': 'Database name' + }, + "port": { + "type": "number" + }, + "read_timeout": { + "type": "number", + "title": "Read Timeout" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } + } @classmethod def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'host': { - 'type': 'string' - }, - 'user': { - 'type': 'string' - }, - 'password': { - 'type': 'string', - 'title': 'Password' - }, - 'database': { - 'type': 'string', - 'title': 'Database name' - }, - "port": { - "type": "number" - }, - "read_timeout": { - "type": "number", - "title": "Read Timeout" - }, - }, + 'properties': cls.configuration_properties, 'required': ['database'], 'secret': ['password'] } diff --git a/redash/serializers.py b/redash/serializers.py index f1e40de803..44f43ae454 100644 --- a/redash/serializers.py +++ b/redash/serializers.py @@ -9,6 +9,7 @@ from flask_login import current_user from redash import models from redash.permissions import has_access, view_only +from redash.handlers.query_results import run_query_sync def public_widget(widget): @@ -21,8 +22,15 @@ def public_widget(widget): 'created_at': widget.created_at } - if widget.visualization and widget.visualization.id: - query_data = models.QueryResult.query.get(widget.visualization.query_rel.latest_query_data_id).to_dict() + if (widget.visualization and widget.visualization.id and + widget.visualization.query_rel is not None): + q = widget.visualization.query_rel + # make sure the widget's query has a latest_query_data_id that is + # not null so public dashboards work + if (q.latest_query_data_id is None): + run_query_sync(q.data_source, {}, q.query_text) + + query_data = q.latest_query_data.to_dict() res['visualization'] = { 'type': widget.visualization.type, 'name': widget.visualization.name, @@ -31,9 +39,10 @@ def public_widget(widget): 'updated_at': widget.visualization.updated_at, 'created_at': widget.visualization.created_at, 'query': { + 'id': q.id, 'query': ' ', # workaround, as otherwise the query data won't be loaded. - 'name': widget.visualization.query_rel.name, - 'description': widget.visualization.query_rel.description, + 'name': q.name, + 'description': q.description, 'options': {}, 'latest_query_data': query_data } @@ -90,6 +99,8 @@ def serialize_query(query, with_stats=False, with_visualizations=False, with_use 'query': query.query_text, 'query_hash': query.query_hash, 'schedule': query.schedule, + 'schedule_until': query.schedule_until, + 'schedule_resultset_size': query.schedule_resultset_size, 'api_key': query.api_key, 'is_archived': query.is_archived, 'is_draft': query.is_draft, diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index 2115d8aa27..b91217173b 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -14,6 +14,7 @@ def all_settings(): return settings +SESSION_COOKIE_SECURE = True REDIS_URL = os.environ.get('REDASH_REDIS_URL', os.environ.get('REDIS_URL', "redis://localhost:6379/0")) PROXIES_COUNT = int(os.environ.get('REDASH_PROXIES_COUNT', "1")) @@ -32,8 +33,12 @@ def all_settings(): # Celery related settings CELERY_BROKER = os.environ.get("REDASH_CELERY_BROKER", REDIS_URL) -CELERY_BACKEND = os.environ.get("REDASH_CELERY_BACKEND", CELERY_BROKER) -CELERY_TASK_RESULT_EXPIRES = int(os.environ.get('REDASH_CELERY_TASK_RESULT_EXPIRES', 3600 * 4)) +CELERY_RESULT_BACKEND = os.environ.get( + "REDASH_CELERY_RESULT_BACKEND", + os.environ.get("REDASH_CELERY_BACKEND", CELERY_BROKER)) +CELERY_RESULT_EXPIRES = int(os.environ.get( + "REDASH_CELERY_RESULT_EXPIRES", + os.environ.get("REDASH_CELERY_TASK_RESULT_EXPIRES", 3600 * 4))) # The following enables periodic job (every 5 minutes) of removing unused query results. QUERY_RESULTS_CLEANUP_ENABLED = parse_boolean(os.environ.get("REDASH_QUERY_RESULTS_CLEANUP_ENABLED", "true")) @@ -78,6 +83,13 @@ def all_settings(): REMOTE_USER_LOGIN_ENABLED = parse_boolean(os.environ.get("REDASH_REMOTE_USER_LOGIN_ENABLED", "false")) REMOTE_USER_HEADER = os.environ.get("REDASH_REMOTE_USER_HEADER", "X-Forwarded-Remote-User") +# When enabled this will match the given remote groups request header with a +# configured list of allowed user groups using UNIX shell-style wildcards such +# as * and ?. +REMOTE_GROUPS_ENABLED = parse_boolean(os.environ.get("REDASH_REMOTE_GROUPS_ENABLED", "false")) +REMOTE_GROUPS_HEADER = os.environ.get("REDASH_REMOTE_GROUPS_HEADER", "X-Forwarded-Remote-Groups") +REMOTE_GROUPS_ALLOWED = set_from_string(os.environ.get("REDASH_REMOTE_GROUPS_ALLOWED", "")) + # If the organization setting auth_password_login_enabled is not false, then users will still be # able to login through Redash instead of the LDAP server LDAP_LOGIN_ENABLED = parse_boolean(os.environ.get('REDASH_LDAP_LOGIN_ENABLED', 'false')) @@ -107,8 +119,14 @@ def all_settings(): LOG_STDOUT = parse_boolean(os.environ.get('REDASH_LOG_STDOUT', 'false')) LOG_PREFIX = os.environ.get('REDASH_LOG_PREFIX', '') LOG_FORMAT = os.environ.get('REDASH_LOG_FORMAT', LOG_PREFIX + '[%(asctime)s][PID:%(process)d][%(levelname)s][%(name)s] %(message)s') -CELERYD_LOG_FORMAT = os.environ.get('REDASH_CELERYD_LOG_FORMAT', LOG_PREFIX + '[%(asctime)s][PID:%(process)d][%(levelname)s][%(processName)s] %(message)s') -CELERYD_TASK_LOG_FORMAT = os.environ.get('REDASH_CELERYD_TASK_LOG_FORMAT', LOG_PREFIX + '[%(asctime)s][PID:%(process)d][%(levelname)s][%(processName)s] task_name=%(task_name)s taks_id=%(task_id)s %(message)s') +CELERYD_WORKER_LOG_FORMAT = os.environ.get( + "REDASH_CELERYD_WORKER_LOG_FORMAT", + os.environ.get('REDASH_CELERYD_LOG_FORMAT', + LOG_PREFIX + '[%(asctime)s][PID:%(process)d][%(levelname)s][%(processName)s] %(message)s')) +CELERYD_WORKER_TASK_LOG_FORMAT = os.environ.get( + "REDASH_CELERYD_WORKER_TASK_LOG_FORMAT", + os.environ.get('REDASH_CELERYD_TASK_LOG_FORMAT', + LOG_PREFIX + '[%(asctime)s][PID:%(process)d][%(levelname)s][%(processName)s] task_name=%(task_name)s taks_id=%(task_id)s %(message)s')) # Mail settings: MAIL_SERVER = os.environ.get('REDASH_MAIL_SERVER', 'localhost') diff --git a/redash/settings/helpers.py b/redash/settings/helpers.py index aa23e7125a..e55d61001d 100644 --- a/redash/settings/helpers.py +++ b/redash/settings/helpers.py @@ -31,7 +31,7 @@ def array_from_string(s): if "" in array: array.remove("") - return array + return [item.strip() for item in array] def set_from_string(s): diff --git a/redash/tasks/queries.py b/redash/tasks/queries.py index 8d8d3cbe9b..ff4b08bb26 100644 --- a/redash/tasks/queries.py +++ b/redash/tasks/queries.py @@ -355,6 +355,7 @@ def cleanup_query_results(): deleted_count = models.QueryResult.query.filter( models.QueryResult.id.in_(unused_query_results.subquery()) ).delete(synchronize_session=False) + deleted_count += models.Query.delete_stale_resultsets() models.db.session.commit() logger.info("Deleted %d unused query results.", deleted_count) diff --git a/redash/worker.py b/redash/worker.py index ecab48ec20..d2add5a218 100644 --- a/redash/worker.py +++ b/redash/worker.py @@ -44,12 +44,12 @@ 'schedule': timedelta(minutes=5) } -celery.conf.update(CELERY_RESULT_BACKEND=settings.CELERY_BACKEND, - CELERYBEAT_SCHEDULE=celery_schedule, - CELERY_TIMEZONE='UTC', - CELERY_TASK_RESULT_EXPIRES=settings.CELERY_TASK_RESULT_EXPIRES, - CELERYD_LOG_FORMAT=settings.CELERYD_LOG_FORMAT, - CELERYD_TASK_LOG_FORMAT=settings.CELERYD_TASK_LOG_FORMAT) +celery.conf.update(result_backend=settings.CELERY_RESULT_BACKEND, + beat_schedule=celery_schedule, + timezone='UTC', + result_expires=settings.CELERY_RESULT_EXPIRES, + worker_log_format=settings.CELERYD_WORKER_LOG_FORMAT, + worker_task_log_format=settings.CELERYD_WORKER_TASK_LOG_FORMAT) if settings.SENTRY_DSN: from raven import Client @@ -78,3 +78,10 @@ def __call__(self, *args, **kwargs): def init_celery_flask_app(**kwargs): app = create_app() app.app_context().push() + +@celery.on_after_configure.connect +def add_periodic_tasks(sender, **kwargs): + app = create_app() + periodic_tasks = getattr(app, 'periodic_tasks', {}) + for params in periodic_tasks.values(): + sender.add_periodic_task(**params) diff --git a/requirements.txt b/requirements.txt index 5beae129f3..b527313cd9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ -Flask==0.11.1 +Flask==1.0.2 Werkzeug==0.11.11 Jinja2==2.8 itsdangerous==0.24 click==6.6 MarkupSafe==0.23 -pyOpenSSL==16.2.0 +pyOpenSSL==18.0.0 httplib2==0.10.3 -Flask-Admin==1.4.2 +Flask-Admin==1.5.2 Flask-RESTful==0.3.5 Flask-Login==0.4.0 Flask-OAuthLib==0.9.3 @@ -23,7 +23,7 @@ python-dateutil==2.4.2 pytz==2016.7 PyYAML==3.12 redis==2.10.5 -requests==2.11.1 +requests==2.20.0 six==1.10.0 SQLAlchemy==1.2.7 SQLAlchemy-Searchable==0.10.6 @@ -33,7 +33,7 @@ wsgiref==0.1.2 honcho==0.5.0 statsd==2.1.2 gunicorn==19.7.1 -celery==3.1.25 +celery==4.1.1 jsonschema==2.4.0 RestrictedPython==3.6.0 pysaml2==4.5.0 @@ -44,7 +44,7 @@ semver==2.2.1 xlsxwriter==0.9.3 pystache==0.5.4 parsedatetime==2.1 -cryptography==2.0.2 +cryptography==2.3.1 simplejson==3.10.0 ua-parser==0.7.3 user-agents==1.1.0 @@ -54,3 +54,4 @@ disposable-email-domains # Uncomment the requirement for ldap3 if using ldap. # It is not included by default because of the GPL license conflict. # ldap3==2.2.4 +redash-stmo>=2018.9.1 diff --git a/requirements_all_ds.txt b/requirements_all_ds.txt index 9e60dfc53d..e58418d55f 100644 --- a/requirements_all_ds.txt +++ b/requirements_all_ds.txt @@ -2,7 +2,7 @@ google-api-python-client==1.5.1 gspread==0.6.2 impyla==0.10.0 influxdb==2.7.1 -MySQL-python==1.2.5 +PyMySQL==0.7.11 oauth2client==3.0.0 pyhive==0.3.0 pymongo==3.6.1 @@ -19,7 +19,7 @@ cassandra-driver==3.11.0 memsql==2.16.0 atsd_client==2.0.12 simple_salesforce==0.72.2 -PyAthena>=1.0.0 +PyAthena>=1.2.0 pymapd>=0.2.1 qds-sdk>=1.9.6 # certifi is needed to support MongoDB and SSL: diff --git a/tests/factories.py b/tests/factories.py index 15f85e58cb..00423fdfa7 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -109,7 +109,9 @@ def __call__(self): query_hash=gen_query_hash('SELECT 1'), data_source=data_source_factory.create, org_id=1) - +query_resultset_factory = ModelFactory(redash.models.QueryResultSet, + query_rel=query_factory.create, + result=query_result_factory.create) visualization_factory = ModelFactory(redash.models.Visualization, type='CHART', query_rel=query_factory.create, @@ -295,6 +297,9 @@ def create_query_result(self, **kwargs): return query_result_factory.create(**args) + def create_query_resultset(self, **kwargs): + return query_resultset_factory.create(**kwargs) + def create_visualization(self, **kwargs): args = { 'query_rel': self.create_query() diff --git a/tests/handlers/test_dashboards.py b/tests/handlers/test_dashboards.py index 0cd38a5fea..03d6f2ba42 100644 --- a/tests/handlers/test_dashboards.py +++ b/tests/handlers/test_dashboards.py @@ -182,3 +182,31 @@ def test_requires_admin_or_owner(self): res = self.make_request('delete', '/api/dashboards/{}/share'.format(dashboard.id), user=user) self.assertEqual(res.status_code, 200) + +class TestDashboardSearchResourceGet(BaseTestCase): + def create_dashboard_sequence(self): + d1 = self.factory.create_dashboard() + new_name = 'Analytics' + rv1 = self.make_request('post', '/api/dashboards/{0}'.format(d1.id), + data={'name': new_name, 'layout': '[]', 'is_draft': False}) + d2 = self.factory.create_dashboard() + rv2 = self.make_request('post', '/api/dashboards/{0}'.format(d2.id), + data={'name': 'Metrics', 'layout': '[]', 'is_draft': True}) + user = self.factory.create_user() + return d1, d2, user + + def test_get_dashboard_search_results_does_not_contain_deleted(self): + d1, d2, user = self.create_dashboard_sequence() + res = self.make_request('delete', '/api/dashboards/{}/share'.format(d2.id)) + dash_search_list = self.make_request('get','/api/dashboards/search?q=Metrics') + dash_search_list_json = json.loads(dash_search_list.data) + self.assertNotIn(d2.id, dash_search_list_json) + + def test_get_dashboard_search_results_obeys_draft_flag(self): + d1, d2, user = self.create_dashboard_sequence() + dash_search_list = self.make_request('get','/api/dashboards/search?q=Metrics&test=True&user_id={}'.format(user.id)) + dash_search_list_json = json.loads(dash_search_list.data) + self.assertNotIn(d2.id, dash_search_list_json) + #self.assertIn(d1.id, dash_search_list_json) + + diff --git a/tests/handlers/test_data_sources.py b/tests/handlers/test_data_sources.py index f07a2b3719..4590056fd4 100644 --- a/tests/handlers/test_data_sources.py +++ b/tests/handlers/test_data_sources.py @@ -60,7 +60,8 @@ def test_updates_data_source(self): new_name = 'New Name' new_options = {"dbname": "newdb"} rv = self.make_request('post', self.path, - data={'name': new_name, 'type': 'pg', 'options': new_options}, + data={'name': new_name, 'type': 'pg', 'options': new_options, + 'doc_url': None}, user=admin) self.assertEqual(rv.status_code, 200) @@ -101,7 +102,9 @@ def test_returns_400_when_configuration_invalid(self): def test_creates_data_source(self): admin = self.factory.create_admin() rv = self.make_request('post', '/api/data_sources', - data={'name': 'DS 1', 'type': 'pg', 'options': {"dbname": "redash"}}, user=admin) + data={'name': 'DS 1', 'type': 'pg', + 'options': {"dbname": "redash"}, + 'doc_url': None}, user=admin) self.assertEqual(rv.status_code, 200) diff --git a/tests/handlers/test_embed.py b/tests/handlers/test_embed.py index 18f119d786..905a6f8672 100644 --- a/tests/handlers/test_embed.py +++ b/tests/handlers/test_embed.py @@ -1,5 +1,8 @@ +import mock + from tests import BaseTestCase from redash.models import db +from redash.query_runner.pg import PostgreSQL class TestEmbedVisualization(BaseTestCase): @@ -97,6 +100,15 @@ def test_inactive_token(self): res = self.make_request('get', '/api/dashboards/public/{}'.format(api_key.api_key), user=False, is_json=False) self.assertEqual(res.status_code, 404) + def test_dashboard_widgets(self): + dashboard = self.factory.create_dashboard() + w1 = self.factory.create_widget(dashboard=dashboard) + w2 = self.factory.create_widget(dashboard=dashboard, visualization=None, text="a text box") + api_key = self.factory.create_api_key(object=dashboard) + with mock.patch.object(PostgreSQL, "run_query") as qr: + qr.return_value = ("[1, 2]", None) + res = self.make_request('get', '/api/dashboards/public/{}'.format(api_key.api_key), user=False, is_json=False) + self.assertEqual(res.status_code, 200) # Not relevant for now, as tokens in api_keys table are only created for dashboards. Once this changes, we should # add this test. # def test_token_doesnt_belong_to_dashboard(self): diff --git a/tests/handlers/test_queries.py b/tests/handlers/test_queries.py index 8e2352553e..135d29c69a 100644 --- a/tests/handlers/test_queries.py +++ b/tests/handlers/test_queries.py @@ -1,3 +1,5 @@ +import json + from tests import BaseTestCase from redash import models from redash.models import db @@ -259,3 +261,107 @@ def test_format_sql_query(self): self.assertEqual(rv.json['query'], expected) + +class ChangeResourceTests(BaseTestCase): + def test_list(self): + query = self.factory.create_query() + query.name = 'version A' + query.record_changes(self.factory.user) + query.name = 'version B' + query.record_changes(self.factory.user) + rv = self.make_request('get', '/api/queries/{0}/version'.format(query.id)) + self.assertEquals(rv.status_code, 200) + self.assertEquals(len(rv.json), 2) + self.assertEquals(rv.json[0]['change']['name']['current'], 'version A') + self.assertEquals(rv.json[1]['change']['name']['current'], 'version B') + + def test_get(self): + query = self.factory.create_query() + query.name = 'version A' + ch1 = query.record_changes(self.factory.user) + query.name = 'version B' + ch2 = query.record_changes(self.factory.user) + rv1 = self.make_request('get', '/api/changes/' + str(ch1.id)) + self.assertEqual(rv1.status_code, 200) + self.assertEqual(rv1.json['change']['name']['current'], 'version A') + rv2 = self.make_request('get', '/api/changes/' + str(ch2.id)) + self.assertEqual(rv2.status_code, 200) + self.assertEqual(rv2.json['change']['name']['current'], 'version B') + + +class AggregateResultsTests(BaseTestCase): + def test_aggregate(self): + qtxt = "SELECT x FROM mytable;" + q = self.factory.create_query(query_text=qtxt, schedule_resultset_size=3) + qr0 = self.factory.create_query_result( + query_text=qtxt, + data=json.dumps({'columns': ['name', 'color'], + 'rows': [{'name': 'eve', 'color': 'grue'}, + {'name': 'mallory', 'color': 'bleen'}]})) + qr1 = self.factory.create_query_result( + query_text=qtxt, + data=json.dumps({'columns': ['name', 'color'], + 'rows': [{'name': 'bob', 'color': 'green'}, + {'name': 'fred', 'color': 'blue'}]})) + qr2 = self.factory.create_query_result( + query_text=qtxt, + data=json.dumps({'columns': ['name', 'color'], + 'rows': [{'name': 'alice', 'color': 'red'}, + {'name': 'eddie', 'color': 'orange'}]})) + qr3 = self.factory.create_query_result( + query_text=qtxt, + data=json.dumps({'columns': ['name', 'color'], + 'rows': [{'name': 'dave', 'color': 'yellow'}, + {'name': 'carol', 'color': 'taupe'}]})) + for qr in (qr0, qr1, qr2, qr3): + self.factory.create_query_resultset(query_rel=q, result=qr) + rv = self.make_request('get', '/api/queries/{}/resultset'.format(q.id)) + self.assertEqual(rv.status_code, 200) + self.assertEqual(rv.json['query_result']['data'], + {'columns': ['name', 'color'], + 'rows': [ + {'name': 'bob', 'color': 'green'}, + {'name': 'fred', 'color': 'blue'}, + {'name': 'alice', 'color': 'red'}, + {'name': 'eddie', 'color': 'orange'}, + {'name': 'dave', 'color': 'yellow'}, + {'name': 'carol', 'color': 'taupe'} + ]}) + + def test_underfilled_aggregate(self): + qtxt = "SELECT x FROM mytable;" + q = self.factory.create_query(query_text=qtxt, + schedule_resultset_size=3) + qr1 = self.factory.create_query_result( + query_text=qtxt, + data=json.dumps({'columns': ['name', 'color'], + 'rows': [{'name': 'bob', 'color': 'green'}, + {'name': 'fred', 'color': 'blue'}]})) + qr2 = self.factory.create_query_result( + query_text=qtxt, + data=json.dumps({'columns': ['name', 'color'], + 'rows': [{'name': 'alice', 'color': 'red'}, + {'name': 'eddie', 'color': 'orange'}]})) + for qr in (qr1, qr2): + self.factory.create_query_resultset(query_rel=q, result=qr) + rv = self.make_request('get', '/api/queries/{}/resultset'.format(q.id)) + self.assertEqual(rv.status_code, 200) + self.assertEqual(rv.json['query_result']['data'], + {'columns': ['name', 'color'], + 'rows': [ + {'name': 'bob', 'color': 'green'}, + {'name': 'fred', 'color': 'blue'}, + {'name': 'alice', 'color': 'red'}, + {'name': 'eddie', 'color': 'orange'} + ]}) + + def test_no_aggregate(self): + qtxt = "SELECT x FROM mytable;" + q = self.factory.create_query(query_text=qtxt) + self.factory.create_query_result( + query_text=qtxt, + data=json.dumps({'columns': ['name', 'color'], + 'rows': [{'name': 'eve', 'color': 'grue'}, + {'name': 'mallory', 'color': 'bleen'}]})) + rv = self.make_request('get', '/api/queries/{}/resultset'.format(q.id)) + self.assertEqual(rv.status_code, 404) diff --git a/tests/handlers/test_widgets.py b/tests/handlers/test_widgets.py index 702ef6f828..cb89caab47 100644 --- a/tests/handlers/test_widgets.py +++ b/tests/handlers/test_widgets.py @@ -64,3 +64,15 @@ def test_delete_widget(self): self.assertEquals(rv.status_code, 200) dashboard = models.Dashboard.get_by_slug_and_org(widget.dashboard.slug, widget.dashboard.org) self.assertEquals(dashboard.widgets.count(), 0) + + def test_updates_textbox_widget(self): + widget = self.factory.create_widget() + + rv = self.make_request('post', '/api/widgets/{0}'.format(widget.id), data={'width':2,'text':'sing and shine on', 'options': {}}) + + self.assertEquals(rv.status_code, 200) + dashboard = models.Dashboard.get_by_slug_and_org(widget.dashboard.slug, widget.dashboard.org) + self.assertEquals(dashboard.widgets.count(), 1) + self.assertEquals(dashboard.layout, '[]') + + diff --git a/tests/models/test_changes.py b/tests/models/test_changes.py index 124e17a30d..3d7c7496e8 100644 --- a/tests/models/test_changes.py +++ b/tests/models/test_changes.py @@ -56,23 +56,12 @@ def test_properly_log_modification(self): obj.record_changes(changed_by=self.factory.user) obj.name = 'Query 2' obj.description = 'description' - db.session.flush() obj.record_changes(changed_by=self.factory.user) change = Change.last_change(obj) self.assertIsNotNone(change) - # TODO: https://github.com/getredash/redash/issues/1550 - # self.assertEqual(change.object_version, 2) + self.assertEqual(change.object_version, 2) self.assertEqual(change.object_version, obj.version) self.assertIn('name', change.change) self.assertIn('description', change.change) - - def test_logs_create_method(self): - q = Query(name='Query', description='', query_text='', - user=self.factory.user, data_source=self.factory.data_source, - org=self.factory.org) - change = Change.last_change(q) - - self.assertIsNotNone(change) - self.assertEqual(q.user, change.user) diff --git a/tests/query_runner/test_http.py b/tests/query_runner/test_http.py new file mode 100644 index 0000000000..e4d88c24a8 --- /dev/null +++ b/tests/query_runner/test_http.py @@ -0,0 +1,136 @@ +import mock +from unittest import TestCase + +import requests +from redash.query_runner import BaseHTTPQueryRunner + + +class RequiresAuthQueryRunner(BaseHTTPQueryRunner): + requires_authentication = True + + +class TestBaseHTTPQueryRunner(TestCase): + + def test_requires_authentication_default(self): + self.assertFalse(BaseHTTPQueryRunner.requires_authentication) + schema = BaseHTTPQueryRunner.configuration_schema() + self.assertNotIn('username', schema['required']) + self.assertNotIn('password', schema['required']) + + def test_requires_authentication_true(self): + schema = RequiresAuthQueryRunner.configuration_schema() + self.assertIn('username', schema['required']) + self.assertIn('password', schema['required']) + + def test_get_auth_with_values(self): + query_runner = BaseHTTPQueryRunner({ + 'username': 'username', + 'password': 'password' + }) + self.assertEqual(query_runner.get_auth(), ('username', 'password')) + + def test_get_auth_empty(self): + query_runner = BaseHTTPQueryRunner({}) + self.assertIsNone(query_runner.get_auth()) + + def test_get_auth_empty_requires_authentication(self): + query_runner = RequiresAuthQueryRunner({}) + self.assertRaisesRegexp( + ValueError, + "Username and Password required", + query_runner.get_auth + ) + + @mock.patch('requests.get') + def test_get_response_success(self, mock_get): + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.text = "Success" + mock_get.return_value = mock_response + + url = 'https://example.com/' + query_runner = BaseHTTPQueryRunner({}) + response, error = query_runner.get_response(url) + mock_get.assert_called_once_with(url, auth=None) + self.assertEqual(response.status_code, 200) + self.assertIsNone(error) + + @mock.patch('requests.get') + def test_get_response_success_custom_auth(self, mock_get): + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.text = "Success" + mock_get.return_value = mock_response + + url = 'https://example.com/' + query_runner = BaseHTTPQueryRunner({}) + auth = ('username', 'password') + response, error = query_runner.get_response(url, auth=auth) + mock_get.assert_called_once_with(url, auth=auth) + self.assertEqual(response.status_code, 200) + self.assertIsNone(error) + + @mock.patch('requests.get') + def test_get_response_failure(self, mock_get): + mock_response = mock.Mock() + mock_response.status_code = 301 + mock_response.text = "Redirect" + mock_get.return_value = mock_response + + url = 'https://example.com/' + query_runner = BaseHTTPQueryRunner({}) + response, error = query_runner.get_response(url) + mock_get.assert_called_once_with(url, auth=None) + self.assertIn(query_runner.response_error, error) + + @mock.patch('requests.get') + def test_get_response_httperror_exception(self, mock_get): + mock_response = mock.Mock() + mock_response.status_code = 500 + mock_response.text = "Server Error" + http_error = requests.HTTPError() + mock_response.raise_for_status.side_effect = http_error + mock_get.return_value = mock_response + + url = 'https://example.com/' + query_runner = BaseHTTPQueryRunner({}) + response, error = query_runner.get_response(url) + mock_get.assert_called_once_with(url, auth=None) + self.assertIsNotNone(error) + self.assertIn("Failed to execute query", error) + + @mock.patch('requests.get') + def test_get_response_requests_exception(self, mock_get): + mock_response = mock.Mock() + mock_response.status_code = 500 + mock_response.text = "Server Error" + exception_message = "Some requests exception" + requests_exception = requests.RequestException(exception_message) + mock_response.raise_for_status.side_effect = requests_exception + mock_get.return_value = mock_response + + url = 'https://example.com/' + query_runner = BaseHTTPQueryRunner({}) + response, error = query_runner.get_response(url) + mock_get.assert_called_once_with(url, auth=None) + self.assertIsNotNone(error) + self.assertEqual(exception_message, error) + + @mock.patch('requests.get') + def test_get_response_generic_exception(self, mock_get): + mock_response = mock.Mock() + mock_response.status_code = 500 + mock_response.text = "Server Error" + exception_message = "Some generic exception" + exception = ValueError(exception_message) + mock_response.raise_for_status.side_effect = exception + mock_get.return_value = mock_response + + url = 'https://example.com/' + query_runner = BaseHTTPQueryRunner({}) + self.assertRaisesRegexp( + ValueError, + exception_message, + query_runner.get_response, + url + ) diff --git a/tests/tasks/test_refresh_queries.py b/tests/tasks/test_refresh_queries.py index 90641ed1a3..5202559671 100644 --- a/tests/tasks/test_refresh_queries.py +++ b/tests/tasks/test_refresh_queries.py @@ -45,3 +45,23 @@ def test_doesnt_enqueue_outdated_queries_for_paused_data_source(self): add_job_mock.assert_called_with( query.query_text, query.data_source, query.user_id, scheduled_query=query, metadata=ANY) + + def test_enqueues_parameterized_queries(self): + """ + Scheduled queries with parameters use saved values. + """ + query = self.factory.create_query( + query_text="select {{n}}", + options={"parameters": [{ + "global": False, + "type": "text", + "name": "n", + "value": "42", + "title": "n"}]}) + oq = staticmethod(lambda: [query]) + with patch('redash.tasks.queries.enqueue_query') as add_job_mock, \ + patch.object(Query, 'outdated_queries', oq): + refresh_queries() + add_job_mock.assert_called_with( + "select 42", query.data_source, query.user_id, + scheduled_query=query, metadata=ANY) diff --git a/tests/test_cli.py b/tests/test_cli.py index fc6d707025..77e1a772d6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -16,7 +16,7 @@ def test_interactive_new(self): result = runner.invoke( manager, ['ds', 'new'], - input="test\n%s\n\n\nexample.com\n\n\ntestdb\n" % (pg_i,)) + input="test\n%s\n\n\n\n\nexample.com\n\n\ntestdb\n" % (pg_i,)) self.assertFalse(result.exception) self.assertEqual(result.exit_code, 0) self.assertEqual(DataSource.query.count(), 1) diff --git a/tests/test_models.py b/tests/test_models.py index f08e28ec53..8767b67fcf 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -180,7 +180,8 @@ def test_failure_extends_schedule(self): Execution failures recorded for a query result in exponential backoff for scheduling future execution. """ - query = self.factory.create_query(schedule="60", schedule_failures=4) + query = self.factory.create_query(schedule="60") + query.schedule_failures = 4 retrieved_at = utcnow() - datetime.timedelta(minutes=16) query_result = self.factory.create_query_result( retrieved_at=retrieved_at, query_text=query.query_text, @@ -192,6 +193,34 @@ def test_failure_extends_schedule(self): query_result.retrieved_at = utcnow() - datetime.timedelta(minutes=17) self.assertEqual(list(models.Query.outdated_queries()), [query]) + def test_schedule_until_after(self): + """ + Queries with non-null ``schedule_until`` are not reported by + Query.outdated_queries() after the given time is past. + """ + three_hours_ago = utcnow() - datetime.timedelta(hours=3) + two_hours_ago = utcnow() - datetime.timedelta(hours=2) + query = self.factory.create_query(schedule="3600", schedule_until=three_hours_ago) + query_result = self.factory.create_query_result(query=query.query_text, retrieved_at=two_hours_ago) + query.latest_query_data = query_result + + queries = models.Query.outdated_queries() + self.assertNotIn(query, queries) + + def test_schedule_until_before(self): + """ + Queries with non-null ``schedule_until`` are reported by + Query.outdated_queries() before the given time is past. + """ + one_hour_from_now = utcnow() + datetime.timedelta(hours=1) + two_hours_ago = utcnow() - datetime.timedelta(hours=2) + query = self.factory.create_query(schedule="3600", schedule_until=one_hour_from_now) + query_result = self.factory.create_query_result(query=query.query_text, retrieved_at=two_hours_ago) + query.latest_query_data = query_result + + queries = models.Query.outdated_queries() + self.assertIn(query, queries) + class QueryArchiveTest(BaseTestCase): def setUp(self): @@ -306,22 +335,74 @@ def test_get_latest_returns_the_last_cached_result_for_negative_ttl(self): class TestUnusedQueryResults(BaseTestCase): def test_returns_only_unused_query_results(self): two_weeks_ago = utcnow() - datetime.timedelta(days=14) - qr = self.factory.create_query_result() - query = self.factory.create_query(latest_query_data=qr) + qt = "SELECT 1" + qr = self.factory.create_query_result(query_text=qt, retrieved_at=two_weeks_ago) + query = self.factory.create_query(query_text=qt, latest_query_data=qr) + unused_qr = self.factory.create_query_result(query_text=qt, retrieved_at=two_weeks_ago) db.session.flush() - unused_qr = self.factory.create_query_result(retrieved_at=two_weeks_ago) self.assertIn((unused_qr.id,), models.QueryResult.unused()) self.assertNotIn((qr.id,), list(models.QueryResult.unused())) def test_returns_only_over_a_week_old_results(self): two_weeks_ago = utcnow() - datetime.timedelta(days=14) - unused_qr = self.factory.create_query_result(retrieved_at=two_weeks_ago) + qt = "SELECT 1" + unused_qr = self.factory.create_query_result(query_text=qt, retrieved_at=two_weeks_ago) db.session.flush() - new_unused_qr = self.factory.create_query_result() - + new_unused_qr = self.factory.create_query_result(query_text=qt) self.assertIn((unused_qr.id,), models.QueryResult.unused()) self.assertNotIn((new_unused_qr.id,), models.QueryResult.unused()) + def test_doesnt_return_live_incremental_results(self): + two_weeks_ago = utcnow() - datetime.timedelta(days=14) + qt = "SELECT 1" + qrs = [self.factory.create_query_result(query_text=qt, retrieved_at=two_weeks_ago) + for _ in range(5)] + q = self.factory.create_query(query_text=qt, latest_query_data=qrs[0], + schedule_resultset_size=3) + for qr in qrs: + self.factory.create_query_resultset(query_rel=q, result=qr) + db.session.flush() + self.assertEqual([], list(models.QueryResult.unused())) + + def test_deletes_stale_resultsets(self): + qt = "SELECT 17" + query = self.factory.create_query(query_text=qt, + schedule_resultset_size=5) + for _ in range(10): + r = self.factory.create_query_result(query_text=qt) + self.factory.create_query_resultset(query_rel=query, result=r) + qt2 = "SELECT 100" + query2 = self.factory.create_query(query_text=qt2, schedule_resultset_size=5) + for _ in range(10): + r = self.factory.create_query_result(query_text=qt2) + self.factory.create_query_resultset(query_rel=query2, result=r) + db.session.flush() + self.assertEqual(models.QueryResultSet.query.count(), 20) + self.assertEqual(models.Query.delete_stale_resultsets(), 10) + self.assertEqual(models.QueryResultSet.query.count(), 10) + + def test_deletes_stale_resultsets_with_dupe_queries(self): + qt = "SELECT 17" + query = self.factory.create_query(query_text=qt, + schedule_resultset_size=5) + for _ in range(10): + r = self.factory.create_query_result(query_text=qt) + self.factory.create_query_resultset(query_rel=query, result=r) + query2 = self.factory.create_query(query_text=qt, + schedule_resultset_size=3) + for _ in range(10): + self.factory.create_query_result(query_text=qt) + self.factory.create_query_resultset(query_rel=query2) + qt2 = "SELECT 100" + query3 = self.factory.create_query(query_text=qt2, schedule_resultset_size=5) + for _ in range(10): + r = self.factory.create_query_result(query_text=qt2) + self.factory.create_query_resultset(query_rel=query3, result=r) + db.session.flush() + self.assertEqual(models.QueryResultSet.query.count(), 30) + self.assertEqual(models.Query.delete_stale_resultsets(), 10) + self.assertEqual(models.QueryResultSet.query.count(), 13) + class TestQueryAll(BaseTestCase): def test_returns_only_queries_in_given_groups(self): diff --git a/webpack.config.js b/webpack.config.js index 3d9200a513..50eea339c0 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -17,6 +17,10 @@ const redashBackend = process.env.REDASH_BACKEND || "http://localhost:5000"; const basePath = fs.realpathSync(path.join(__dirname, "client")); const appPath = fs.realpathSync(path.join(__dirname, "client", "app")); +const extensionsRelativePath = process.env.EXTENSIONS_DIRECTORY || + path.join("client", "app", "extensions"); +const extensionPath = fs.realpathSync(path.join(__dirname, extensionsRelativePath)); + const config = { entry: { app: ["./client/app/index.js", "./client/app/assets/less/main.less"], @@ -29,7 +33,8 @@ const config = { }, resolve: { alias: { - "@": appPath + "@": appPath, + "%": extensionPath } }, plugins: [