From 783aa315d758fb887366dea916081868b309641a Mon Sep 17 00:00:00 2001
From: Daniel Gaspar <danielvazgaspar@gmail.com>
Date: Wed, 15 Apr 2020 17:10:15 +0100
Subject: [PATCH 1/6] [tests] Helper script to run single tests

---
 scripts/tests/README.md          |  32 +++++++++
 scripts/tests/docker-compose.yml |  34 +++++++++
 scripts/tests/run.sh             | 117 +++++++++++++++++++++++++++++++
 3 files changed, 183 insertions(+)
 create mode 100644 scripts/tests/README.md
 create mode 100644 scripts/tests/docker-compose.yml
 create mode 100755 scripts/tests/run.sh

diff --git a/scripts/tests/README.md b/scripts/tests/README.md
new file mode 100644
index 000000000000..faf82452386a
--- /dev/null
+++ b/scripts/tests/README.md
@@ -0,0 +1,32 @@
+# Utility script to run tests faster
+
+## Use:
+
+From the superset repo root directory:
+
+- Example run a single module, will launch (or clean relaunch) Redis and Postgres:
+```$bash
+scripts/tests/run.sh tests.charts.api_tests
+```
+
+- Example run a single test, will launch (or clean relaunch) Redis and Postgres:
+```$bash
+scripts/tests/run.sh tests.charts.api_tests:ChartApiTests.test_get_charts
+```
+
+- Example run a single test, without any init procedures. Init procedures include
+  relaunching containers, db upgrade, superset init, loading example data. If your tests
+  are idempotent, after the first run, subsequent runs are really fast
+```$bash
+scripts/tests/run.sh tests.charts.api_tests:ChartApiTests.test_get_charts --no-init
+```
+
+- Same has above but using MySQL backend
+```$bash
+scripts/tests/run.sh tests.charts.api_tests:ChartApiTests.test_get_charts --no-init --mysql
+```
+
+- Example for not using the "provided" containers, you have your own probably
+```$bash
+scripts/tests/run.sh tests.charts.api_tests:ChartApiTests.test_get_charts --no-docker
+```
diff --git a/scripts/tests/docker-compose.yml b/scripts/tests/docker-compose.yml
new file mode 100644
index 000000000000..e14284d089a6
--- /dev/null
+++ b/scripts/tests/docker-compose.yml
@@ -0,0 +1,34 @@
+version: '2'
+services:
+  redis:
+    image: redis:3.2
+    ports:
+      - 6379:6379
+    volumes:
+      - redis:/data
+
+  postgres:
+    image: postgres:9
+    environment:
+      POSTGRES_DB: superset
+      POSTGRES_PASSWORD: pguserpassword
+      POSTGRES_USER: postgresuser
+    ports:
+      - 5432:5432
+  mysql:
+    image: mysql:5.7
+    environment:
+      MYSQL_DATABASE: superset
+      MYSQL_USER: mysqluser
+      MYSQL_PASSWORD: mysqluserpassword
+      MYSQL_ROOT_PASSWORD: root-password
+    ports:
+      - 3306:3306
+    networks:
+      - default
+    volumes:
+      - ./mysqlconf:/etc/mysql/conf.d
+
+volumes:
+  redis:
+    external: false
diff --git a/scripts/tests/run.sh b/scripts/tests/run.sh
new file mode 100755
index 000000000000..78e1aaac3a4b
--- /dev/null
+++ b/scripts/tests/run.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+function restart_docker() {
+  docker-compose -f "${SCRIPT_DIR}"/docker-compose.yml down
+  docker-compose -f "${SCRIPT_DIR}"/docker-compose.yml up -d --force-recreate
+  # Bad way to wait for the db's to start
+  sleep 5
+}
+
+function test_init() {
+  echo --------------------
+  echo Upgrading
+  echo --------------------
+  superset db upgrade
+  echo --------------------
+  echo Superset init
+  echo --------------------
+  superset init
+  echo --------------------
+  echo Load examples
+  echo --------------------
+  nosetests tests/load_examples_test.py
+}
+
+
+SCRIPT_DIR=$(dirname "$0")
+
+if [[ "$#" -eq  "0" ]]
+then
+  echo "No argument suplied"
+  echo ------------------------
+  echo use:
+  echo "run.sh <test module name> [options]"
+  echo "[options]:"
+  echo "--mysql: Use MySQL container on tests"
+  echo "--no-init: Dont restart docker and no db migrations, superset init and test data"
+  echo "--no-docker: Dont restart docker"
+  exit 1
+fi
+
+export SUPERSET__SQLALCHEMY_DATABASE_URI=${SUPERSET__SQLALCHEMY_DATABASE_URI:-postgresql+psycopg2://postgresuser:pguserpassword@localhost/superset}
+export SUPERSET_CONFIG=${SUPERSET_CONFIG:-tests.superset_test_config}
+RUN_INIT=1
+RUN_DOCKER=1
+TEST_MODULE="${1}"
+shift 1
+
+PARAMS=""
+while (( "$#" )); do
+  case "$1" in
+    --mysql)
+      export SUPERSET__SQLALCHEMY_DATABASE_URI="mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8"
+      shift 1
+      ;;
+    --no-init)
+      RUN_INIT=0
+      RUN_DOCKER=0
+      shift 1
+      ;;
+    --no-docker)
+      RUN_DOCKER=0
+      shift 1
+      ;;
+    --) # end argument parsing
+      shift
+      break
+      ;;
+    --*) # unsupported flags
+      echo "Error: Unsupported flag $1" >&2
+      exit 1
+      ;;
+    *) # preserve positional arguments
+      PARAMS="$PARAMS $1"
+      shift
+      ;;
+  esac
+done
+
+echo ------------------------------------
+echo DB_URI="${SUPERSET__SQLALCHEMY_DATABASE_URI}"
+echo Superset config module="${SUPERSET_CONFIG}"
+echo Run init procedures=$RUN_INIT
+echo Run Docker=$RUN_DOCKER
+echo Test to run:"${TEST_MODULE}"
+echo ------------------------------------
+
+
+if [ $RUN_DOCKER -eq 1 ]
+then
+  restart_docker
+fi
+
+if [ $RUN_INIT -eq 1 ]
+then
+  test_init
+fi
+
+nosetests --exclude=load_examples_test "${TEST_MODULE}"

From 663cca1c6cd7b6f6e659d7f82bbff0ca635ad6b2 Mon Sep 17 00:00:00 2001
From: Daniel Gaspar <danielvazgaspar@gmail.com>
Date: Wed, 15 Apr 2020 17:44:04 +0100
Subject: [PATCH 2/6] [tests] add missing license

---
 scripts/tests/docker-compose.yml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/scripts/tests/docker-compose.yml b/scripts/tests/docker-compose.yml
index e14284d089a6..32055fb23523 100644
--- a/scripts/tests/docker-compose.yml
+++ b/scripts/tests/docker-compose.yml
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 version: '2'
 services:
   redis:

From 9e2d93d078ec4e5b8958dadbebca96538247232c Mon Sep 17 00:00:00 2001
From: Daniel Gaspar <danielvazgaspar@gmail.com>
Date: Wed, 15 Apr 2020 17:47:59 +0100
Subject: [PATCH 3/6] [tests] add missing license

---
 scripts/tests/README.md | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/scripts/tests/README.md b/scripts/tests/README.md
index faf82452386a..e147056d3e2e 100644
--- a/scripts/tests/README.md
+++ b/scripts/tests/README.md
@@ -1,3 +1,22 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
 # Utility script to run tests faster
 
 ## Use:

From 843be12d78468630326764953c697e3bf288e033 Mon Sep 17 00:00:00 2001
From: Daniel Gaspar <danielvazgaspar@gmail.com>
Date: Mon, 20 Apr 2020 11:38:46 +0100
Subject: [PATCH 4/6] [pypi] use postgres defined at the root docker-compose
 file

---
 scripts/tests/README.md          | 14 ++++-----
 scripts/tests/docker-compose.yml | 51 --------------------------------
 scripts/tests/run.sh             | 50 ++++++++++++++++++-------------
 3 files changed, 36 insertions(+), 79 deletions(-)
 delete mode 100644 scripts/tests/docker-compose.yml

diff --git a/scripts/tests/README.md b/scripts/tests/README.md
index e147056d3e2e..d3e34ce1cb94 100644
--- a/scripts/tests/README.md
+++ b/scripts/tests/README.md
@@ -19,6 +19,11 @@ under the License.
 
 # Utility script to run tests faster
 
+By default tests will be run using the Postgres container defined at the `docker-compose` file on the root of the repo,
+so prior to using this script make sure to launch the dev containers.
+
+You can use a different DB backend by defining `SUPERSET__SQLALCHEMY_DATABASE_URI` env var.
+
 ## Use:
 
 From the superset repo root directory:
@@ -40,12 +45,7 @@ scripts/tests/run.sh tests.charts.api_tests:ChartApiTests.test_get_charts
 scripts/tests/run.sh tests.charts.api_tests:ChartApiTests.test_get_charts --no-init
 ```
 
-- Same has above but using MySQL backend
-```$bash
-scripts/tests/run.sh tests.charts.api_tests:ChartApiTests.test_get_charts --no-init --mysql
-```
-
-- Example for not using the "provided" containers, you have your own probably
+- Example for not recreating the test DB (will still run all the tests init procedures)
 ```$bash
-scripts/tests/run.sh tests.charts.api_tests:ChartApiTests.test_get_charts --no-docker
+scripts/tests/run.sh tests.charts.api_tests:ChartApiTests.test_get_charts --no-reset-db
 ```
diff --git a/scripts/tests/docker-compose.yml b/scripts/tests/docker-compose.yml
deleted file mode 100644
index 32055fb23523..000000000000
--- a/scripts/tests/docker-compose.yml
+++ /dev/null
@@ -1,51 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-version: '2'
-services:
-  redis:
-    image: redis:3.2
-    ports:
-      - 6379:6379
-    volumes:
-      - redis:/data
-
-  postgres:
-    image: postgres:9
-    environment:
-      POSTGRES_DB: superset
-      POSTGRES_PASSWORD: pguserpassword
-      POSTGRES_USER: postgresuser
-    ports:
-      - 5432:5432
-  mysql:
-    image: mysql:5.7
-    environment:
-      MYSQL_DATABASE: superset
-      MYSQL_USER: mysqluser
-      MYSQL_PASSWORD: mysqluserpassword
-      MYSQL_ROOT_PASSWORD: root-password
-    ports:
-      - 3306:3306
-    networks:
-      - default
-    volumes:
-      - ./mysqlconf:/etc/mysql/conf.d
-
-volumes:
-  redis:
-    external: false
diff --git a/scripts/tests/run.sh b/scripts/tests/run.sh
index 78e1aaac3a4b..a8c085cc2eb9 100755
--- a/scripts/tests/run.sh
+++ b/scripts/tests/run.sh
@@ -19,13 +19,20 @@
 
 set -e
 
-function restart_docker() {
-  docker-compose -f "${SCRIPT_DIR}"/docker-compose.yml down
-  docker-compose -f "${SCRIPT_DIR}"/docker-compose.yml up -d --force-recreate
-  # Bad way to wait for the db's to start
-  sleep 5
+#
+# Reset test DATABASE
+#
+function reset_db() {
+  echo --------------------
+  echo Reseting test DB
+  echo --------------------
+  docker exec -i preset_fork_superset_db_1 bash -c "/usr/bin/psql -h 127.0.0.1 -U ${DB_USER} -w -c 'DROP DATABASE ${DB_NAME};'"
+  docker exec -i preset_fork_superset_db_1 bash -c "/usr/bin/psql -h 127.0.0.1 -U ${DB_USER} -w -c 'CREATE DATABASE ${DB_NAME};'"
 }
 
+#
+# Run init test procedures
+#
 function test_init() {
   echo --------------------
   echo Upgrading
@@ -42,8 +49,6 @@ function test_init() {
 }
 
 
-SCRIPT_DIR=$(dirname "$0")
-
 if [[ "$#" -eq  "0" ]]
 then
   echo "No argument suplied"
@@ -51,33 +56,36 @@ then
   echo use:
   echo "run.sh <test module name> [options]"
   echo "[options]:"
-  echo "--mysql: Use MySQL container on tests"
   echo "--no-init: Dont restart docker and no db migrations, superset init and test data"
-  echo "--no-docker: Dont restart docker"
+  echo "--no-reset-db: Recreates test database (DROP, CREATE)"
   exit 1
 fi
 
-export SUPERSET__SQLALCHEMY_DATABASE_URI=${SUPERSET__SQLALCHEMY_DATABASE_URI:-postgresql+psycopg2://postgresuser:pguserpassword@localhost/superset}
+#
+# Init global vars
+#
+DB_NAME="test"
+DB_USER="superset"
+DB_PASSWORD="superset"
+export SUPERSET__SQLALCHEMY_DATABASE_URI=${SUPERSET__SQLALCHEMY_DATABASE_URI:-postgresql+psycopg2://"${DB_USER}":"${DB_PASSWORD}"@localhost/"${DB_NAME}"}
 export SUPERSET_CONFIG=${SUPERSET_CONFIG:-tests.superset_test_config}
 RUN_INIT=1
-RUN_DOCKER=1
+RUN_RESET_DB=1
 TEST_MODULE="${1}"
+
+# Shift to pass the first cmd parameter for the test module
 shift 1
 
 PARAMS=""
 while (( "$#" )); do
   case "$1" in
-    --mysql)
-      export SUPERSET__SQLALCHEMY_DATABASE_URI="mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8"
-      shift 1
-      ;;
     --no-init)
       RUN_INIT=0
-      RUN_DOCKER=0
+      RUN_RESET_DB=0
       shift 1
       ;;
-    --no-docker)
-      RUN_DOCKER=0
+    --no-reset-db)
+      RUN_RESET_DB=0
       shift 1
       ;;
     --) # end argument parsing
@@ -99,14 +107,14 @@ echo ------------------------------------
 echo DB_URI="${SUPERSET__SQLALCHEMY_DATABASE_URI}"
 echo Superset config module="${SUPERSET_CONFIG}"
 echo Run init procedures=$RUN_INIT
-echo Run Docker=$RUN_DOCKER
+echo Run reset DB=$RUN_RESET_DB
 echo Test to run:"${TEST_MODULE}"
 echo ------------------------------------
 
 
-if [ $RUN_DOCKER -eq 1 ]
+if [ $RUN_RESET_DB -eq 1 ]
 then
-  restart_docker
+  reset_db
 fi
 
 if [ $RUN_INIT -eq 1 ]

From 963eb170e4bbdbea2717b781768cc90d662b2f04 Mon Sep 17 00:00:00 2001
From: Daniel Gaspar <danielvazgaspar@gmail.com>
Date: Mon, 20 Apr 2020 11:42:13 +0100
Subject: [PATCH 5/6] [pypi] improve README remove previous container
 references

---
 scripts/tests/README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/tests/README.md b/scripts/tests/README.md
index d3e34ce1cb94..814e54235fad 100644
--- a/scripts/tests/README.md
+++ b/scripts/tests/README.md
@@ -28,18 +28,18 @@ You can use a different DB backend by defining `SUPERSET__SQLALCHEMY_DATABASE_UR
 
 From the superset repo root directory:
 
-- Example run a single module, will launch (or clean relaunch) Redis and Postgres:
+- Example run a single test module:
 ```$bash
 scripts/tests/run.sh tests.charts.api_tests
 ```
 
-- Example run a single test, will launch (or clean relaunch) Redis and Postgres:
+- Example run a single test:
 ```$bash
 scripts/tests/run.sh tests.charts.api_tests:ChartApiTests.test_get_charts
 ```
 
-- Example run a single test, without any init procedures. Init procedures include
-  relaunching containers, db upgrade, superset init, loading example data. If your tests
+- Example run a single test, without any init procedures. Init procedures include:
+  resetting test database, db upgrade, superset init, loading example data. If your tests
   are idempotent, after the first run, subsequent runs are really fast
 ```$bash
 scripts/tests/run.sh tests.charts.api_tests:ChartApiTests.test_get_charts --no-init

From 4a11281112dab22b194eca8bc15cc8e10e61c9b7 Mon Sep 17 00:00:00 2001
From: Daniel Gaspar <danielvazgaspar@gmail.com>
Date: Tue, 21 Apr 2020 09:51:23 +0100
Subject: [PATCH 6/6] [tests] name the containers

---
 docker-compose.yml   | 8 +++++++-
 scripts/tests/run.sh | 4 ++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index f8df1a36304c..cb45e9c2b433 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -36,6 +36,7 @@ version: "3.7"
 services:
   redis:
     image: redis:3.2
+    container_name: superset_cache
     restart: unless-stopped
     ports:
       - "127.0.0.1:6379:6379"
@@ -45,6 +46,7 @@ services:
   db:
     env_file: docker/.env
     image: postgres:10
+    container_name: superset_db
     restart: unless-stopped
     ports:
       - "127.0.0.1:5432:5432"
@@ -52,9 +54,10 @@ services:
       - db_home:/var/lib/postgresql/data
 
   superset:
+    env_file: docker/.env
     build: *superset-build
+    container_name: superset_app
     command: ["flask", "run", "-p", "8088", "--with-threads", "--reload", "--debugger", "--host=0.0.0.0"]
-    env_file: docker/.env
     restart: unless-stopped
     ports:
       - 8088:8088
@@ -63,6 +66,7 @@ services:
 
   superset-init:
     build: *superset-build
+    container_name: superset_init
     command: ["/app/docker-init.sh"]
     env_file: docker/.env
     depends_on: *superset-depends-on
@@ -70,6 +74,7 @@ services:
 
   superset-node:
     image: node:10-jessie
+    container_name: superset_node
     command: ["bash", "-c", "cd /app/superset-frontend && npm install --global webpack webpack-cli && npm install && npm run dev"]
     env_file: docker/.env
     depends_on: *superset-depends-on
@@ -77,6 +82,7 @@ services:
 
   superset-worker:
     build: *superset-build
+    container_name: superset_worker
     command: ["celery", "worker", "--app=superset.tasks.celery_app:app", "-Ofair"]
     env_file: docker/.env
     restart: unless-stopped
diff --git a/scripts/tests/run.sh b/scripts/tests/run.sh
index a8c085cc2eb9..a4be3ec55625 100755
--- a/scripts/tests/run.sh
+++ b/scripts/tests/run.sh
@@ -26,8 +26,8 @@ function reset_db() {
   echo --------------------
   echo Reseting test DB
   echo --------------------
-  docker exec -i preset_fork_superset_db_1 bash -c "/usr/bin/psql -h 127.0.0.1 -U ${DB_USER} -w -c 'DROP DATABASE ${DB_NAME};'"
-  docker exec -i preset_fork_superset_db_1 bash -c "/usr/bin/psql -h 127.0.0.1 -U ${DB_USER} -w -c 'CREATE DATABASE ${DB_NAME};'"
+  docker exec -i superset_db bash -c "/usr/bin/psql -h 127.0.0.1 -U ${DB_USER} -w -c 'DROP DATABASE ${DB_NAME};'"
+  docker exec -i superset_db bash -c "/usr/bin/psql -h 127.0.0.1 -U ${DB_USER} -w -c 'CREATE DATABASE ${DB_NAME};'"
 }
 
 #