Various fixes so our code also works with Travis Xenial VM image

.circle/add-itest-user.sh

@@ -1,6 +1,12 @@
-#!/bin/bash
+#!/usr/bin/env bash
+
 set -e
 
+if [ "$(whoami)" != 'root' ]; then
+    echo 'Please run with sudo'
+    exit 2
+fi
+
 # Create an SSH system user (default `stanley` user may be already created)
 if (! id stanley 2>/dev/null); then
   useradd stanley

.travis.yml

@@ -1,6 +1,7 @@
 # Used old infrastructure, needed for integration tests:
 # http://docs.travis-ci.com/user/workers/standard-infrastructure/
 sudo: required
+# NOTE: We use precise because tests finish faster than on Xenial
 dist: precise
 language: python
 
@@ -60,6 +61,22 @@ matrix:
       python: 3.6
       name: "Integration Tests (Python 3.6)"
 
+addons:
+  apt:
+    sources:
+      - mongodb-upstart
+      - sourceline: 'deb [arch=amd64] http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.4 multiverse'
+        key_url: 'https://www.mongodb.org/static/pgp/server-3.4.asc'
+      - sourceline: 'ppa:git-core/ppa'
+    packages:
+      - mongodb-org-server
+      - mongodb-org-shell
+      # NOTE: Uncomment below for Xenial
+      # -rabbitmq-server
+      - git
+      - libffi-dev
+
+# NOTE: Remove / comment services section below for Xenial
 services:
   - rabbitmq
 
@@ -76,13 +93,8 @@ cache:
     #- .tox/
 
 before_install:
-  # 1. Install MongoDB 3.4 and latest version of git
-  - sudo add-apt-repository -y ppa:git-core/ppa
-  - curl https://www.mongodb.org/static/pgp/server-3.4.asc | sudo apt-key add -
-  - echo "deb [arch=amd64] http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.4 multiverse" | sudo tee -a /etc/apt/sources.list
-  # Work around for Travis timeout issues, see https://github.com/travis-ci/travis-ci/issues/9112
-  - sudo apt-get update --option Acquire::Retries=100 --option Acquire::http::Timeout="60"
-  - sudo apt-get install mongodb-org-server mongodb-org-shell git libffi-dev -y
+  # Work around for apt Travis timeout issues, see https://github.com/travis-ci/travis-ci/issues/9112
+  #- sudo apt-get update --option Acquire::Retries=100 --option Acquire::http::Timeout="60"
   - pip install --upgrade "pip>=9.0,<9.1"
   - sudo pip install --upgrade "virtualenv==15.1.0"
 
@@ -100,7 +112,7 @@ before_script:
   - sudo rm -rf /var/lib/mongodb ; sudo mkdir /var/lib/mongodb ; sudo chown -R mongodb:mongodb /var/lib/mongodb
   - sudo service mongod restart ; sleep 5
   - sudo service mongod status
-  - tail -30 /var/log/mongodb/mongod.log
+  - sudo tail -30 /var/log/mongodb/mongod.log
   - mongod --version
   - git --version
   - pip --version
@@ -113,6 +125,7 @@ before_script:
   - make play
 
 script:
+  # Clean up egg-info directories which get created when installing runners
   # NOTE: We enable code coverage and per test timing information on master so build can take twice
   # as long as PR builds
   - if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then COMMAND_THRESHOLD=$(expr ${COMMAND_THRESHOLD} \* 2); fi; ./scripts/travis/time-command.sh "make ${TASK}" ${COMMAND_THRESHOLD}

Makefile

@@ -134,16 +134,20 @@ play:
 .PHONY: check
 check: requirements flake8 checklogs
 
+# NOTE: We pass --no-deps to the script so we don't install all the
+# package dependencies which are already installed as part of "requirements"
+# make targets. This speeds up the build
 .PHONY: install-runners
 install-runners:
+
 	@echo ""
 	@echo "================== INSTALL RUNNERS ===================="
 	@echo ""
 	@for component in $(COMPONENTS_RUNNERS); do \
 		echo "==========================================================="; \
 		echo "Installing runner:" $$component; \
 		echo "==========================================================="; \
-        (. $(VIRTUALENV_DIR)/bin/activate; cd $$component; python setup.py develop); \
+		(. $(VIRTUALENV_DIR)/bin/activate; cd $$component; python setup.py develop --no-deps); \
 	done
 
 .PHONY: checklogs
@@ -392,16 +396,21 @@ requirements: virtualenv .sdist-requirements install-runners
 	done
 
 	# Install st2common package to load drivers defined in st2common setup.py
-	(cd st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop)
-
+	# NOTE: We pass --no-deps to the script so we don't install all the
+	# package dependencies which are already installed as part of "requirements"
+	# make targets. This speeds up the build
+	(cd st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps)
 
 	# Note: We install prance here and not as part of any component
 	# requirements.txt because it has a conflict with our dependency (requires
 	# new version of requests) which we cant resolve at this moment
 	$(VIRTUALENV_DIR)/bin/pip install "prance==0.6.1"
 
 	# Install st2common to register metrics drivers
-	(cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop)
+	# NOTE: We pass --no-deps to the script so we don't install all the
+	# package dependencies which are already installed as part of "requirements"
+	# make targets. This speeds up the build
+	(cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps)
 
 	# Some of the tests rely on submodule so we need to make sure submodules are check out
 	git submodule update --init --recursive
@@ -581,7 +590,7 @@ endif
 		echo "-----------------------------------------------------------"; \
 		. $(VIRTUALENV_DIR)/bin/activate; \
 		    COVERAGE_FILE=.coverage.integration.$$(echo $$component | tr '/' '.') \
-		    nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \
+		    nosetests $(NOSE_OPTS) -s -v --exe $(NOSE_COVERAGE_FLAGS) \
 		    $(NOSE_COVERAGE_PACKAGES) \
 		    $$component/tests/integration || exit 1; \
 		echo "-----------------------------------------------------------"; \

contrib/runners/local_runner/tests/integration/test_localrunner.py

@@ -433,7 +433,6 @@ def test_script_with_parameters_parameter_serialization(self):
         runner.pre_run()
         status, result, _ = runner.run(action_parameters=action_parameters)
         runner.post_run(status, result)
-
         self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED)
         self.assertTrue('PARAM_STRING=test string' in result['stdout'])
         self.assertTrue('PARAM_INTEGER=1' in result['stdout'])

requirements.txt

@@ -31,7 +31,6 @@ oslo.utils<=3.37.0,>=3.36.2
 paramiko==2.4.2
 passlib==1.7.1
 prettytable
-prometheus_client==0.1.1
 prompt-toolkit==1.0.15
 psutil==5.4.8
 pyinotify==0.9.6

scripts/travis/install-requirements.sh

@@ -1,18 +1,18 @@
-#!/usr/bin/env bash  
+#!/usr/bin/env bash
 
 if [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then
     pip install "tox==3.5.2"
 
-	# Install runners
+    # Install runners
     . virtualenv/bin/activate
 
-	CURRENT_DIR=`pwd`
-	for RUNNER in `ls -d $CURRENT_DIR/contrib/runners/*`
-	do
-	  echo "Installing runner: $RUNNER..."
+    CURRENT_DIR=`pwd`
+    for RUNNER in `ls -d $CURRENT_DIR/contrib/runners/*`
+    do
+      echo "Installing runner: $RUNNER..."
       cd $RUNNER
-      python setup.py develop
-	done
+      python setup.py develop --no-deps
+    done
 
     # NOTE: We create the environment and install the dependencies first. This
     # means that the subsequent tox build / test command has a stable run time

scripts/travis/prepare-integration.sh

@@ -6,46 +6,7 @@ if [ "$(whoami)" != 'root' ]; then
     exit 2
 fi
 
-# create and configure user
-# proudly stolen from `./tools/st2_deploy.sh`
-TYPE='debs'
-SYSTEMUSER='stanley'
-STAN="/home/${SYSTEMUSER}/${TYPE}"
-mkdir -p ${STAN}
-
-VIRTUALENV_DIR=virtualenv
-
-create_user() {
-  if [ $(id -u ${SYSTEMUSER} &> /devnull; echo $?) != 0 ]
-  then
-    echo "###########################################################################################"
-    echo "# Creating system user: ${SYSTEMUSER}"
-    useradd ${SYSTEMUSER}
-    mkdir -p /home/${SYSTEMUSER}/.ssh
-    rm -Rf ${STAN}/*
-    chmod 0700 /home/${SYSTEMUSER}/.ssh
-    mkdir -p /home/${SYSTEMUSER}/${TYPE}
-    echo "###########################################################################################"
-    echo "# Generating system user ssh keys"
-    ssh-keygen -f /home/${SYSTEMUSER}/.ssh/stanley_rsa -P ""
-    cat /home/${SYSTEMUSER}/.ssh/stanley_rsa.pub >> /home/${SYSTEMUSER}/.ssh/authorized_keys
-    chmod 0600 /home/${SYSTEMUSER}/.ssh/authorized_keys
-    chown -R ${SYSTEMUSER}:${SYSTEMUSER} /home/${SYSTEMUSER}
-    if [ $(grep 'stanley' /etc/sudoers.d/* &> /dev/null; echo $?) != 0 ]
-    then
-      echo "${SYSTEMUSER}    ALL=(ALL)       NOPASSWD: SETENV: ALL" >> /etc/sudoers.d/st2
-      chmod 0440 /etc/sudoers.d/st2
-    fi
-
-    # make sure requiretty is disabled.
-    sed -i "s/^Defaults\s\+requiretty/# Defaults requiretty/g" /etc/sudoers
-  fi
-}
-
-create_user
-
-# install screen
-apt-get install -y screen
+UBUNTU_VERSION=`lsb_release -a 2>&1 | grep Codename | grep -v "LSB" | awk '{print $2}'`
 
 # Activate the virtualenv created during make requirements phase
 source ./virtualenv/bin/activate
@@ -61,3 +22,12 @@ st2 --version
 # as root can't write to logs/ directory and tests fail
 chmod 777 logs/
 chmod 777 logs/*
+
+# Workaround for Travis on Ubuntu Xenial so local runner integration tests work
+# when executing them under user "stanley" (by default Travis checks out the
+# code and runs tests under a different system user).
+# NOTE: We need to pass "--exe" flag to nosetests when using this workaround.
+if [ "${UBUNTU_VERSION}" == "xenial" ]; then
+  echo "Applying workaround for stanley user permissions issue to /home/travis on Xenial"
+  chmod 777 -R /home/travis
+fi

st2common/in-requirements.txt

@@ -29,7 +29,6 @@ flex
 webob
 jsonpath-rw
 python-statsd
-prometheus_client
 ujson
 # Note: amqp is used by kombu, this needs to be added here to be picked up by
 # requirements fixate script.

st2common/requirements.txt

@@ -15,7 +15,6 @@ mongoengine==0.16.3
 networkx==1.11
 oslo.config<1.13,>=1.12.1
 paramiko==2.4.2
-prometheus_client==0.1.1
 pymongo==3.7.2
 python-dateutil==2.7.5
 python-statsd==2.1.0

st2reactor/tests/integration/test_garbage_collector.py

@@ -188,7 +188,7 @@ def test_garbage_collection(self):
         process = self._start_garbage_collector()
 
         # Give it some time to perform garbage collection and kill it
-        eventlet.sleep(10)
+        eventlet.sleep(15)
         process.send_signal(signal.SIGKILL)
         self.remove_process(process=process)
 
@@ -236,7 +236,7 @@ def test_inquiry_garbage_collection(self):
         process = self._start_garbage_collector()
 
         # Give it some time to perform garbage collection and kill it
-        eventlet.sleep(10)
+        eventlet.sleep(15)
         process.send_signal(signal.SIGKILL)
         self.remove_process(process=process)
 

st2tests/st2tests/fixtures/localrunner_pack/actions/text_gen.py

@@ -1,4 +1,4 @@
-#! /usr/bin/python
+#!/usr/bin/env python
 
 from __future__ import absolute_import
 

test-requirements.txt

@@ -21,8 +21,8 @@ nose-parallel==0.3.1
 # Required by st2client tests
 pyyaml<4.0,>=3.12
 RandomWords
-gunicorn==19.7.1
-psutil==5.4.5
+gunicorn==19.9.0
+psutil==5.4.8
 webtest==2.0.25
 rstcheck>=3.3.1,<3.4
 pyrabbit

tox.ini

@@ -82,15 +82,15 @@ deps = virtualenv
        -e{toxinidir}/st2client
        -e{toxinidir}/st2common
 commands =
-    nosetests --rednose --immediate -sv st2actions/tests/integration/
-    nosetests --rednose --immediate -sv st2api/tests/integration/
-    nosetests --rednose --immediate -sv st2common/tests/integration/
-    nosetests --rednose --immediate -sv st2debug/tests/integration/
-    nosetests --rednose --immediate -sv st2exporter/tests/integration/
-    nosetests --rednose --immediate -sv st2reactor/tests/integration/
-    nosetests --rednose --immediate -sv contrib/runners/action_chain_runner/tests/integration/
-    nosetests --rednose --immediate -sv contrib/runners/local_runner/tests/integration/
-    nosetests --rednose --immediate -sv contrib/runners/mistral_v2/tests/integration/
-    nosetests --rednose --immediate -sv contrib/runners/orquesta_runner/tests/integration/
-    nosetests --rednose --immediate -sv st2tests/integration/orquesta/
-    nosetests --rednose --immediate -sv contrib/runners/python_runner/tests/integration/
+    nosetests --rednose --immediate -sv --exe st2actions/tests/integration/
+    nosetests --rednose --immediate -sv --exe st2api/tests/integration/
+    nosetests --rednose --immediate -sv --exe st2common/tests/integration/
+    nosetests --rednose --immediate -sv --exe st2debug/tests/integration/
+    nosetests --rednose --immediate -sv --exe st2exporter/tests/integration/
+    nosetests --rednose --immediate -sv --exe st2reactor/tests/integration/
+    nosetests --rednose --immediate -sv --exe contrib/runners/action_chain_runner/tests/integration/
+    nosetests --rednose --immediate -sv --exe contrib/runners/local_runner/tests/integration/
+    nosetests --rednose --immediate -sv --exe contrib/runners/mistral_v2/tests/integration/
+    nosetests --rednose --immediate -sv --exe contrib/runners/orquesta_runner/tests/integration/
+    nosetests --rednose --immediate -sv --exe st2tests/integration/orquesta/
+    nosetests --rednose --immediate -sv --exe contrib/runners/python_runner/tests/integration/