Only load fields that are needed from Google API for Big Query schema.

.circleci/config.yml

@@ -39,7 +39,7 @@ jobs:
           name: Copy Test Results
           command: |
             mkdir -p /tmp/test-results/unit-tests
-            docker cp tests:/app/coverage.xml ./coverage.xml 
+            docker cp tests:/app/coverage.xml ./coverage.xml
             docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml
       - store_test_results:
           path: /tmp/test-results
@@ -76,29 +76,41 @@ jobs:
       - run:
           name: Execute Cypress tests
           command: npm run cypress run-ci
-  build-tarball:
+  deploy-master:
     docker:
-      - image: circleci/node:8
+      - image: circleci/buildpack-deps:xenial
     steps:
+      - setup_remote_docker
       - checkout
-      - run: sudo apt install python-pip
-      - run: npm install
-      - run: .circleci/update_version
-      - run: npm run bundle
-      - run: npm run build
-      - run: .circleci/pack
-      - store_artifacts:
-          path: /tmp/artifacts/
-  build-docker-image:
+      - run: ./bin/dockerflow-version "master"
+      - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
+      - run: docker build -t $DOCKERHUB_REPO:master .
+      - run: docker push $DOCKERHUB_REPO:master
+  deploy-rc:
+    docker:
+      - image: circleci/buildpack-deps:xenial
+    steps:
+      - setup_remote_docker
+      - checkout
+      - run: ./bin/dockerflow-version "rc"
+      - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
+      - run: docker build -t $DOCKERHUB_REPO:rc .
+      - run: docker push $DOCKERHUB_REPO:rc
+  deploy-milestone:
     docker:
       - image: circleci/buildpack-deps:xenial
     steps:
       - setup_remote_docker
       - checkout
-      - run: .circleci/update_version
+      - run: ./bin/dockerflow-version "$CIRCLE_TAG"
       - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
-      - run: docker build -t redash/redash:$(.circleci/docker_tag) .
-      - run: docker push redash/redash:$(.circleci/docker_tag)
+      - run: docker build -t $DOCKERHUB_REPO:$CIRCLE_TAG .
+      - run: docker push $DOCKERHUB_REPO:$CIRCLE_TAG
+      # Create alias from tag to "latest"
+      - run: docker tag $DOCKERHUB_REPO:$CIRCLE_TAG $DOCKERHUB_REPO:latest
+      - run: docker push $DOCKERHUB_REPO:latest
+
+
 workflows:
   version: 2
   build:
@@ -108,22 +120,27 @@ workflows:
       - backend-unit-tests
       - frontend-unit-tests
       - frontend-e2e-tests
-      - build-tarball:
-           requires:
-             - backend-unit-tests
-           filters:
-             tags:
-               only: /v[0-9]+(\.[0-9\-a-z]+)*/
-             branches:
-               only:
-                 - master
-                 - /release\/.*/
-      - build-docker-image:
-           requires:
-             - backend-unit-tests
-           filters:
-             branches:
-               only:
-                 - master
-                 - preview-build
-                 - /release\/.*/
+      - deploy-master:
+          requires:
+            - backend-unit-tests
+          filters:
+            branches:
+              only:
+                - master
+
+      - deploy-rc:
+          requires:
+            - backend-unit-tests
+          filters:
+            branches:
+              only:
+                - release
+
+      - deploy-milestone:
+          requires:
+            - backend-unit-tests
+          filters:
+            tags:
+              only: /^m[0-9]+(\.[0-9]+)?$/
+            branches:
+              ignore: /.*/

Dockerfile

@@ -10,9 +10,19 @@ RUN pip install -r requirements.txt -r requirements_dev.txt
 RUN if [ "x$skip_ds_deps" = "x" ] ; then pip install -r requirements_all_ds.txt ; else echo "Skipping pip install -r requirements_all_ds.txt" ; fi
 
 COPY . ./
+RUN npm install && npm run bundle && npm run build && rm -rf node_modules
+
+# Upgrade node to LTS 6.11.2
+RUN cd ~
+RUN wget https://nodejs.org/download/release/v6.11.2/node-v6.11.2-linux-x64.tar.gz
+RUN sudo tar --strip-components 1 -xzvf node-v* -C /usr/local
+
+# Upgrade npm
+RUN npm upgrade npm
+
 RUN npm install && npm run bundle && npm run build && rm -rf node_modules
 RUN chown -R redash /app
 USER redash
 
 ENTRYPOINT ["/app/bin/docker-entrypoint"]
-CMD ["server"]
+CMD ["server"]

bin/bundle-extensions

@@ -22,9 +22,12 @@ os.environ["EXTENSIONS_DIRECTORY"] = EXTENSIONS_RELATIVE_PATH
 for entry_point in iter_entry_points('redash.extensions'):
     # This is where the frontend code for an extension lives
     # inside of its package.
-    content_folder_relative = os.path.join(
-        entry_point.name, 'bundle')
-    (root_module, _) = os.path.splitext(entry_point.module_name)
+
+    split_module_path = entry_point.module_name.split(os.extsep)
+    root_module = split_module_path.pop(0)
+
+    content_folder_relative = os.path.join(os.path.join(
+        *split_module_path), 'bundle')
 
     if not resource_isdir(root_module, content_folder_relative):
         continue

bin/docker-entrypoint

@@ -2,23 +2,32 @@
 set -e
 
 worker() {
+  /app/manage.py db upgrade
   WORKERS_COUNT=${WORKERS_COUNT:-2}
   QUEUES=${QUEUES:-queries,scheduled_queries,celery}
+  MAX_MEMORY=$(($(/usr/bin/awk '/MemTotal/ {print $2}' /proc/meminfo)/4))
 
   echo "Starting $WORKERS_COUNT workers for queues: $QUEUES..."
-  exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair
+  exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo \
+      --max-tasks-per-child=10 \
+      --max-memory-per-child=$MAX_MEMORY \
+      -Ofair
 }
 
 scheduler() {
+  /app/manage.py db upgrade
   WORKERS_COUNT=${WORKERS_COUNT:-1}
   QUEUES=${QUEUES:-celery}
 
   echo "Starting scheduler and $WORKERS_COUNT workers for queues: $QUEUES..."
 
-  exec /usr/local/bin/celery worker --app=redash.worker --beat -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair
+  exec /usr/local/bin/celery worker --app=redash.worker --beat -c$WORKERS_COUNT -Q$QUEUES -linfo \
+      --max-tasks-per-child=10 \
+      -Ofair
 }
 
 server() {
+  /app/manage.py db upgrade
   exec /usr/local/bin/gunicorn -b 0.0.0.0:5000 --name redash -w${REDASH_WEB_WORKERS:-4} redash.wsgi:app
 }
 

bin/dockerflow-version

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -eo pipefail
+
+VERSION="$1"
+
+printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
+  "$CIRCLE_SHA1" \
+  "$VERSION" \
+  "$CIRCLE_PROJECT_USERNAME" \
+  "$CIRCLE_PROJECT_REPONAME" \
+  "$CIRCLE_BUILD_URL" \
+> version.json

client/app/assets/less/redash/query.less

@@ -457,6 +457,7 @@ a.label-tag {
 
 .datasource-small {
   visibility: hidden;
+  display: none !important;
 }
 
 .query-fullscreen .query-metadata__mobile {
@@ -579,6 +580,11 @@ nav .rg-bottom {
     display: none;
   }
 
+  .datasource-small {
+    visibility: visible;
+    display: inline-block !important;
+  }
+
   .query-fullscreen {
     flex-direction: column;
     overflow: hidden;

client/app/components/queries/schedule-dialog.html

@@ -15,4 +15,11 @@ <h4 class="modal-title">Refresh Schedule</h4>
             <query-time-picker refresh-type="$ctrl.refreshType" query="$ctrl.query" save-query="$ctrl.saveQuery"></query-time-picker>
         </label>
     </div>
+    <label>
+      Stop scheduling at date/time (format yyyy-MM-ddTHH:mm:ss, like 2016-12-28T14:57:00):
+      <schedule-until query="$ctrl.query" save-query="$ctrl.saveQuery"></schedule-until>
+    </label>
+    <label>
+      Number of query results to keep <schedule-keep-results query="$ctrl.query" save-query="$ctrl.saveQuery"></schedule-keep-results>
+    </label>
 </div>

client/app/components/queries/schedule-dialog.js

@@ -103,11 +103,32 @@ function queryRefreshSelect(clientConfig, Policy) {
   };
 }
 
+function scheduleUntil() {
+  return {
+    restrict: 'E',
+    scope: {
+      query: '=',
+      saveQuery: '=',
+    },
+    template: '<input type="datetime-local" step="1" class="form-control" ng-model="query.scheduleUntil" ng-change="saveQuery()">',
+  };
+}
+
+function scheduleKeepResults() {
+  return {
+    restrict: 'E',
+    scope: {
+      query: '=',
+      saveQuery: '=',
+    },
+    template: '<input type="number" class="form-control" ng-model="query.schedule_resultset_size" ng-change="saveQuery()" ng-disabled="!query.schedule">',
+  };
+}
+
 const ScheduleForm = {
   controller() {
     this.query = this.resolve.query;
     this.saveQuery = this.resolve.saveQuery;
-
     if (this.query.hasDailySchedule()) {
       this.refreshType = 'daily';
     } else {
@@ -125,6 +146,8 @@ const ScheduleForm = {
 export default function init(ngModule) {
   ngModule.directive('queryTimePicker', queryTimePicker);
   ngModule.directive('queryRefreshSelect', queryRefreshSelect);
+  ngModule.directive('scheduleUntil', scheduleUntil);
+  ngModule.directive('scheduleKeepResults', scheduleKeepResults);
   ngModule.component('scheduleDialog', ScheduleForm);
 }
 

client/app/components/queries/schema-browser.html

@@ -6,10 +6,20 @@
             ng-click="$ctrl.onRefresh()">
       <span class="zmdi zmdi-refresh"></span>
     </button>
+
+    <button class="btn btn-default"
+          title="Toggle Versioned Tables"
+          ng-click="$ctrl.flipToggleVersionedTables($ctrl.versionToggle, $ctrl.tabletogglestring)"
+          ng-if="$ctrl.tabletogglestring && $ctrl.tabletogglestring != ''"
+          >
+    <span class="fa " ng-class="{'fa-toggle-on': $ctrl.versionToggle == true, 'fa-toggle-off': !$ctrl.versionToggle}">
+      <input type="checkbox" id="versioned-tables-toggle" ng-model="$ctrl.versionToggle" hidden/>
+    </span>
+    </button>
   </div>
 
   <div class="schema-browser" vs-repeat vs-size="$ctrl.getSize(table)">
-    <div ng-repeat="table in $ctrl.schema | filter:$ctrl.schemaFilterObject track by table.name">
+    <div ng-repeat="table in $ctrl.schema | filter:$ctrl.schemaFilterObject | filter: {name: '!'+$ctrl.versionFilter}">
       <div class="table-name" ng-click="$ctrl.showTable(table)">
         <i class="fa fa-table"></i>
         <strong>

client/app/components/queries/schema-browser.js

@@ -3,6 +3,9 @@ import template from './schema-browser.html';
 function SchemaBrowserCtrl($rootScope, $scope) {
   'ngInject';
 
+  this.versionToggle = false;
+  this.versionFilter = 'abcdefghijklmnop';
+
   this.showTable = (table) => {
     table.collapsed = !table.collapsed;
     $scope.$broadcast('vsRepeatTrigger');
@@ -21,6 +24,15 @@ function SchemaBrowserCtrl($rootScope, $scope) {
   this.isEmpty = function isEmpty() {
     return this.schema === undefined || this.schema.length === 0;
   };
+  this.flipToggleVersionedTables = (versionToggle, toggleString) => {
+    if (versionToggle === false) {
+      this.versionToggle = true;
+      this.versionFilter = toggleString;
+    } else {
+      this.versionToggle = false;
+      this.versionFilter = 'abcdefghijklmnop';
+    }
+  };
 
   this.itemSelected = ($event, hierarchy) => {
     $rootScope.$broadcast('query-editor.command', 'paste', hierarchy.join('.'));
@@ -44,7 +56,9 @@ function SchemaBrowserCtrl($rootScope, $scope) {
 const SchemaBrowser = {
   bindings: {
     schema: '<',
+    tabletogglestring: '<',
     onRefresh: '&',
+    flipToggleVersionedTables: '&',
   },
   controller: SchemaBrowserCtrl,
   template,

client/app/pages/data-sources/list.html

@@ -9,7 +9,7 @@
       <div class="database-source">
         <a class="visual-card" ng-href="data_sources/{{dataSource.id}}" ng-repeat="dataSource in $ctrl.dataSources" title="{{dataSource.name}}">
           <img ng-src="/static/images/db-logos/{{dataSource.type}}.png" alt="{{dataSource.name}}">
-          <h3>{{dataSource.name}}</h3>
+          <h3>{{dataSource.name}}</h3> - {{dataSource.type_name}}
         </a>
       </div>
     </div>

client/app/pages/queries/compare-query-dialog.css

@@ -0,0 +1,54 @@
+/* Compare Query Version container */
+/* Offers slight visual improvement (alignment) to modern UAs */
+.compare-query-version {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+}
+
+.diff-removed {
+  background-color: rgba(208, 2, 27, 0.3);
+}
+
+.diff-added {
+  background-color: rgba(65, 117, 5, 0.3);
+}
+
+.query-diff-container span {
+  display: inline-block;
+  border-radius: 3px;
+  line-height: 20px;
+  vertical-align: middle;
+  margin: 0 5px 0 0;
+}
+
+.query-diff-container > div:not(.compare-query-version-controls) {
+  float: left;
+  width: calc(50% - 5px);
+  margin: 0 10px 0 0;
+}
+
+.compare-query-version {
+  background-color: #f5f5f5;
+  padding: 5px;
+  border: 1px solid #ccc;
+  margin-right: 15px;
+  border-radius: 3px;
+}
+
+.diff-content {
+  border: 1px solid #ccc;
+  background-color: #f5f5f5;
+  border-radius: 3px;
+  padding: 15px;
+}
+
+.query-diff-container > div:last-child {
+  margin: 0;
+}
+
+.compare-query-version-controls {
+  display: flex;
+  align-items: center;
+  margin-bottom: 25px;
+}