Scheduled daily dependency update on thursday

.circleci/config.yml

@@ -1,124 +1,131 @@
-version: 2.0
+# These environment variables must be set in CircleCI UI
+#
+# DOCKERHUB_REPO - docker hub repo, format: <username>/<repo>
+# DOCKER_USER
+# DOCKER_PASS
+#
+
+version: 2
 jobs:
-  unit-tests:
-    environment:
-      COMPOSE_FILE: .circleci/docker-compose.circle.yml
-      COMPOSE_PROJECT_NAME: redash
+  build:
     docker:
-      - image: circleci/buildpack-deps:xenial
+      - image: docker:18.02.0-ce
+    working_directory: ~/mozilla/redash
     steps:
-      - setup_remote_docker
       - checkout
+      - setup_remote_docker
       - run:
-          name: Build Docker Images
-          command: |
-            set -x
-            docker-compose up -d
-            sleep 10
-      - run:
-          name: Create Test Database
-          command: docker-compose run --rm postgres psql -h postgres -U postgres -c "create database tests;"
-      - run:
-          name: Run Tests
-          command: docker-compose run --name tests redash tests --junitxml=junit.xml tests/
-      - run:
-          name: Copy Test Results
-          command: |
-            mkdir -p /tmp/test-results/unit-tests
-            docker cp tests:/app/coverage.xml ./coverage.xml 
-            docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml
-      - store_test_results:
-          path: /tmp/test-results
-      - store_artifacts:
-          path: coverage.xml
-  build-tarball:
+          command: docker build -t app:build .
+          no_output_timeout: 20m
+
+  test:
     docker:
-      - image: circleci/node:8
+      - image: circleci/node:6.14.3-stretch
+      - image: redis
+      - image: circleci/postgres:10-alpine-postgis
+    working_directory: ~/mozilla/redash
     steps:
       - checkout
-      - run: npm install
-      - run: npm run build
-      - run: .circleci/update_version
-      - run: .circleci/pack
+      - run: mkdir -p /tmp/test-reports/pytest
+      - run: sudo apt-get update
+      - run: sudo apt-get install -y python-pip python-dev
+      - run: sudo apt-get install -y redis-tools redis-server
+      - run: sudo pip install --upgrade setuptools
+      - run: sudo pip install -r requirements_dev.txt
+      - run: sudo pip install -r requirements.txt
+      - run: sudo npm install
+      - run: sudo npm run bundle
+      - run: sudo npm run build
+      - run:
+          command: pytest --junitxml=/tmp/test-reports/pytest/junit.xml tests/
+          environment:
+            REDASH_REDIS_URL: redis://localhost:6379/0
+            REDASH_DATABASE_URL: "postgresql://postgres@localhost/postgres"
       - store_artifacts:
-          path: /tmp/artifacts/
-  build-docker-image:
-    docker:
-      - image: circleci/buildpack-deps:xenial
-    steps:
-      - setup_remote_docker
-      - checkout
-      - run: .circleci/update_version
-      - run: docker login -u $DOCKER_USER -p $DOCKER_PASS
-      - run: docker build -t redash/redash:$(.circleci/docker_tag) .
-      - run: docker push redash/redash:$(.circleci/docker_tag)
-  integration-tests:
-    working_directory: ~/redash
-    machine: true
-    environment:
-      REDASH_SERVER_URL : "http://127.0.0.1:5000/"
-      DOCKER_IMAGE: mozilla/redash-ui-tests
+          path: /tmp/test-reports/
+          destination: tr1
+      - store_test_results:
+          path: /tmp/test-reports/
+
+  deploy-master:
+    machine:
+      enable: true
+    working_directory: ~/mozilla/redash
     steps:
       - checkout
       - run:
-          name: Install Docker Compose
-          command: |
-            set -x
-            pip install --upgrade pip
-            pip install docker-compose>=1.18
-            docker-compose --version
-      - run:
-          name: Pull redash images
+          name: Deploy to Dockerhub
+          no_output_timeout: 20m
           command: |
-            set -x
-            docker-compose -f docker-compose.yml up --no-start
-            sleep 10
-      - run:
-          name: Pull redash-ui-tests
-          command: docker pull "${DOCKER_IMAGE}":latest
+            ./bin/deploy "master"
+
+  deploy-rc:
+    machine:
+      enable: true
+    working_directory: ~/mozilla/redash
+    steps:
+      - checkout
       - run:
-          name: Setup redash instance
+          name: Deploy to Dockerhub
+          no_output_timeout: 20m
           command: |
-            set -x
-            docker-compose run --rm --user root server create_db
-            docker-compose run --rm postgres psql -h postgres -U postgres -c "create database tests"
-            docker-compose run --rm --user root server /app/manage.py users create_root root@example.com "rootuser" --password "IAMROOT" --org default
-            docker-compose run --rm --user root server /app/manage.py ds new "ui-tests" --type "url" --options '{"title": "uitests"}'
-            docker-compose run -d -p 5000:5000 --user root server
-            docker-compose start postgres
+            ./bin/deploy "rc"
+
+  deploy-milestone:
+    machine:
+      enable: true
+    working_directory: ~/mozilla/redash
+    steps:
+      - checkout
       - run:
-          name: Run tests
+          name: Deploy milestone to Dockerhub
+          no_output_timeout: 20m
           command: |
-            set -x
-            docker run --net="host" --env REDASH_SERVER_URL="${REDASH_SERVER_URL}" "${DOCKER_IMAGE}"
-      - store_artifacts:
-          path: report.html
+            ./bin/deploy "$CIRCLE_TAG"
+            ./bin/alias "$CIRCLE_TAG" "latest"
+
+
 workflows:
   version: 2
-  integration_tests:
+  build-test-deploy:
     jobs:
-      - integration-tests:
+      - build:
           filters:
-             branches:
-               only: master
-  build:
-    jobs:
-      - unit-tests
-      - build-tarball:
-           requires:
-             - unit-tests
-           filters:
-             tags:
-               only: /v[0-9]+(\.[0-9\-a-z]+)*/
-             branches:
-               only:
-                 - master
-                 - release
-      - build-docker-image:
-           requires:
-             - unit-tests
-           filters:
-             branches:
-               ignore: /.*/
-             tags:
-               only: /v[0-9]+(\.[0-9\-a-z]+)*/
+            tags:
+              only: /.*/
+            branches:
+              ignore:
+                - gh-pages
+
+      - test:
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              ignore:
+                - gh-pages
+
+      - deploy-master:
+          requires:
+            - test
+          filters:
+            branches:
+              only:
+                - master
+
+      - deploy-rc:
+          requires:
+            - test
+          filters:
+            branches:
+              only:
+                - release
+
+      - deploy-milestone:
+          requires:
+            - test
+          filters:
+            tags:
+              only: /^m[0-9]+(\.[0-9]+)?$/
+            branches:
+              ignore: /.*/

.pyup.yml

@@ -0,0 +1,6 @@
+schedule: "every day"
+search: False
+update: insecure
+requirements:
+  - requirements.txt:
+      update: insecure

Dockerfile

@@ -6,7 +6,16 @@ COPY requirements.txt requirements_dev.txt requirements_all_ds.txt ./
 RUN pip install -r requirements.txt -r requirements_dev.txt -r requirements_all_ds.txt
 
 COPY . ./
-RUN npm install && npm run build && rm -rf node_modules
+
+# Upgrade node to LTS 6.11.2
+RUN cd ~
+RUN wget https://nodejs.org/download/release/v6.11.2/node-v6.11.2-linux-x64.tar.gz
+RUN sudo tar --strip-components 1 -xzvf node-v* -C /usr/local
+
+# Upgrade npm
+RUN npm upgrade npm
+
+RUN npm install && npm run bundle && npm run build && rm -rf node_modules
 RUN chown -R redash /app
 USER redash
 

Makefile

@@ -0,0 +1,22 @@
+.PHONY: build bundle compose_build create_database tests test_db clean
+
+compose_build:
+	docker-compose build
+
+test_db:
+	docker-compose run --rm postgres psql -h postgres -U postgres -c "create database tests"
+
+create_database:
+	docker-compose run server create_db
+
+clean:
+	docker ps -a -q | xargs docker kill;docker ps -a -q | xargs docker rm
+
+bundle:
+	docker-compose run server bin/bundle-extensions
+
+tests:
+	docker-compose run server tests
+
+build: bundle
+	npm run build

bin/alias

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -eo pipefail
+
+[ ! -z $DOCKERHUB_REPO ] && [ $# -eq 2 ]
+
+VERSION="$1"
+ALIAS="$2"
+
+docker login -u $DOCKER_USER -p $DOCKER_PASS
+docker tag $DOCKERHUB_REPO:$VERSION $DOCKERHUB_REPO:$ALIAS
+docker push $DOCKERHUB_REPO:$ALIAS

bin/bundle-extensions

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+import os
+from subprocess import call
+from distutils.dir_util import copy_tree
+
+from pkg_resources import iter_entry_points, resource_filename, resource_isdir
+
+
+
+# Make a directory for extensions and set it as an environment variable
+# to be picked up by webpack.
+EXTENSIONS_RELATIVE_PATH = os.path.join('client', 'app', 'extensions')
+EXTENSIONS_DIRECTORY = os.path.join(
+    os.path.dirname(os.path.dirname(__file__)),
+    EXTENSIONS_RELATIVE_PATH)
+
+if not os.path.exists(EXTENSIONS_DIRECTORY):
+    os.makedirs(EXTENSIONS_DIRECTORY)
+os.environ["EXTENSIONS_DIRECTORY"] = EXTENSIONS_RELATIVE_PATH
+
+for entry_point in iter_entry_points('redash.extensions'):
+    # This is where the frontend code for an extension lives
+    # inside of its package.
+    content_folder_relative = os.path.join(
+        entry_point.name, 'bundle')
+    (root_module, _) = os.path.splitext(entry_point.module_name)
+
+    if not resource_isdir(root_module, content_folder_relative):
+        continue
+
+    content_folder = resource_filename(root_module, content_folder_relative)
+
+    # This is where we place our extensions folder.
+    destination = os.path.join(
+        EXTENSIONS_DIRECTORY,
+        entry_point.name)
+
+    copy_tree(content_folder, destination)

bin/deploy

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eo pipefail
+
+[ ! -z $DOCKERHUB_REPO ] && [ $# -eq 1 ]
+
+VERSION="$1"
+
+printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
+  "$CIRCLE_SHA1" \
+  "$VERSION" \
+  "$CIRCLE_PROJECT_USERNAME" \
+  "$CIRCLE_PROJECT_REPONAME" \
+  "$CIRCLE_BUILD_URL" \
+> version.json
+
+docker login -u $DOCKER_USER -p $DOCKER_PASS
+docker build -t $DOCKERHUB_REPO:$VERSION .
+docker push $DOCKERHUB_REPO:$VERSION

bin/docker-entrypoint

@@ -2,23 +2,32 @@
 set -e
 
 worker() {
+  /app/manage.py db upgrade
   WORKERS_COUNT=${WORKERS_COUNT:-2}
   QUEUES=${QUEUES:-queries,scheduled_queries,celery}
+  MAX_MEMORY=$(($(/usr/bin/awk '/MemTotal/ {print $2}' /proc/meminfo)/4))
 
   echo "Starting $WORKERS_COUNT workers for queues: $QUEUES..."
-  exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair
+  exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo \
+      --max-tasks-per-child=10 \
+      --max-memory-per-child=$MAX_MEMORY \
+      -Ofair
 }
 
 scheduler() {
+  /app/manage.py db upgrade
   WORKERS_COUNT=${WORKERS_COUNT:-1}
   QUEUES=${QUEUES:-celery}
 
   echo "Starting scheduler and $WORKERS_COUNT workers for queues: $QUEUES..."
 
-  exec /usr/local/bin/celery worker --app=redash.worker --beat -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair
+  exec /usr/local/bin/celery worker --app=redash.worker --beat -c$WORKERS_COUNT -Q$QUEUES -linfo \
+      --max-tasks-per-child=10 \
+      -Ofair
 }
 
 server() {
+  /app/manage.py db upgrade
   exec /usr/local/bin/gunicorn -b 0.0.0.0:5000 --name redash -w${REDASH_WEB_WORKERS:-4} redash.wsgi:app
 }