[WIP] Try adding back MongoDB changes to see if it helps with the tests performance

.travis.yml

@@ -12,6 +12,7 @@ branches:
 env:
   global:
     - CACHE_NAME=JOB1
+    - MONGODB_VERSION=3.4.16
     - ENABLE_COVERAGE=$([ "${TRAVIS_PULL_REQUEST}" = "false" ] && echo "yes" || echo "no")
 matrix:
   include:
@@ -36,17 +37,16 @@ matrix:
 addons:
   apt:
     sources:
-      - mongodb-upstart
-      - sourceline: 'deb [arch=amd64] http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.4 multiverse'
-        key_url: 'https://www.mongodb.org/static/pgp/server-3.4.asc'
+     #- mongodb-upstart
+     # - sourceline: 'deb [arch=amd64] http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.4 multiverse'
+     #  key_url: 'https://www.mongodb.org/static/pgp/server-3.4.asc'
       - sourceline: 'ppa:git-core/ppa'
     packages:
-      - mongodb-org-server
-      - mongodb-org-shell
+      #- mongodb-org-server
+      #- mongodb-org-shell
       - git
 
 services:
-  - mongodb
   - rabbitmq
 
 cache:
@@ -68,10 +68,14 @@ install:
   - if [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then pip install "tox==3.0.0"; else make requirements; fi
   - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] && [ "${ENABLE_COVERAGE}" = 'yes' ]; then pip install codecov; fi
   - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi
+  # Note: We use ramdisk for better performance
+  - sudo -E ./scripts/travis/create-and-mount-ramdisk.sh
+  - ./scripts/travis/install-and-run-mongodb.sh
 
 # Let's enable rabbitmqadmin
 # See https://github.com/messagebus/lapine/wiki/Testing-on-Travis.
 before_script:
+  - mongo --version
   - git --version
   - pip --version
   - virtualenv --version

scripts/travis/install-and-run-mongodb.sh

@@ -1,33 +1,56 @@
 #!/usr/bin/env bash
 
 # Script which installs versions of MongoDB specified using an environment variable
-
-if [ ! "${MONGODB}" ]; then
-    echo "MONGODB environment variable not set"
+if [ ! "${MONGODB_VERSION}" ]; then
+    echo "MONGODB_VERSION environment variable not set."
     exit 2
 fi
 
 # Note: MongoDB 2.4 and 2.6 don't work with ramdisk since they don't work with
 # small files and require at least 3 GB of space
-if [ ${MONGODB} = '2.4.9' ] || [ ${MONGODB} = '2.6.12' ]; then
+if [ "${MONGODB_VERSION}" = '2.4.9' ] || [ "${MONGODB_VERSION}" = '2.6.12' ]; then
     DATA_DIR=/tmp/mongodbdata
 else
     DATA_DIR=/mnt/ramdisk/mongodb
 fi
 
-wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-${MONGODB}.tgz -O /tmp/mongodb.tgz
-tar -xvf /tmp/mongodb.tgz
+DATA_DIR=/tmp/mongodbdata
+MONGODB_DIR=/tmp/mongodb
+
 mkdir -p ${DATA_DIR}
-echo "Starting MongoDB v${MONGODB}"
-${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --nojournal --journalCommitInterval 500 \
-    --syncdelay 0 --dbpath ${DATA_DIR} --bind_ip 127.0.0.1 &> /tmp/mongodb.log &
-EXIT_CODE=$?
+mkdir -p ${MONGODB_DIR}
+
+DOWNLOAD_URL="http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-${MONGODB_VERSION}.tgz"
+
+echo "Downloading MongoDB ${MONGODB_VERSION} from ${DOWNLOAD_URL}"
+wget -q ${DOWNLOAD_URL} -O /tmp/mongodb.tgz
+tar -xvf /tmp/mongodb.tgz -C ${MONGODB_DIR} --strip=1
+
+echo "Symlinking mongo shell binary to /usr/local/bin"
+
+# Symlink latest mongodb shell
+sudo ln -sf ${MONGODB_DIR}/bin/mongo /usr/local/bin/mongo
+sudo ln -sf ${MONGODB_DIR}/bin/mongo /usr/bin/mongo
+
+echo "Starting MongoDB v${MONGODB_VERSION}"
+# Note: We use --notablescan option to detected missing indexes early. When this
+# option is enabled, queries which result in table scan (which usually means a
+# missing index or a bad query) are not allowed and result in a failed test.
+#--wiredTigerStatisticsLogDelaySecs 0 --noIndexBuildRetry --noscripting --notablescan \
+${MONGODB_DIR}/bin/mongod --nojournal --journalCommitInterval 500 --syncdelay 0 \
+    --wiredTigerStatisticsLogDelaySecs 0 --noIndexBuildRetry --noscripting \
+    --dbpath ${DATA_DIR} --bind_ip 127.0.0.1 &> /tmp/mongodb.log &
+MONGODB_PID=$!
+
+# Give process some time to start up
 sleep 5
 
-if [ ${EXIT_CODE} -ne 0 ]; then
+if ps -p ${MONGODB_PID} > /dev/null; then
+    echo "MongoDB successfuly started"
+    tail -30 /tmp/mongodb.log
+    exit 0
+else
     echo "Failed to start MongoDB"
     tail -30 /tmp/mongodb.log
     exit 1
 fi
-
-tail -30 /tmp/mongodb.log

st2tests/st2tests/base.py

@@ -219,8 +219,9 @@ def _establish_connection_and_re_create_db(cls):
             db_ensure_indexes()
 
     @classmethod
-    def _drop_db(cls):
-        cls._drop_collections()
+    def _drop_db(cls):        # NOTE: In older MongoDB versions you needed to drop all the collections prior to dropping
+        # the database - that's not needed anymore with the wiredtiger engine
+        # cls._drop_collections()
 
         if cls.db_connection is not None:
             cls.db_connection.drop_database(cfg.CONF.database.db_name)