From c0586b2ad176dc5a3079433b2e4fc79994a9b8ee Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 12:34:18 -0800 Subject: [PATCH 01/52] Try using xenial base image on travis. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e4912a9e2a..261dce9c52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ # Used old infrastructure, needed for integration tests: # http://docs.travis-ci.com/user/workers/standard-infrastructure/ sudo: required -dist: precise +dist: xenial language: python branches: From e6a3e35617d72630d94289e7e8014f45ba219757 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 12:51:46 -0800 Subject: [PATCH 02/52] Use sudo. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 261dce9c52..66220f6014 100644 --- a/.travis.yml +++ b/.travis.yml @@ -99,8 +99,8 @@ before_script: # Clean up any old MongoDB 3.4 data files laying around and make sure mongodb user can write to it - sudo rm -rf /var/lib/mongodb ; sudo mkdir /var/lib/mongodb ; sudo chown -R mongodb:mongodb /var/lib/mongodb - sudo service mongod restart ; sleep 5 - - sudo service mongod status - - tail -30 /var/log/mongodb/mongod.log + - sudo service mongod status -l + - sudo tail -30 /var/log/mongodb/mongod.log - mongod --version - git --version - pip --version From fb77c9206581cf3a9f1a7a7dbcef6f62bc636081 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 13:03:40 -0800 Subject: [PATCH 03/52] rabbitmq service is not available on xenial so we need to install the package. --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66220f6014..0601b97fc5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,9 +60,6 @@ matrix: python: 3.6 name: "Integration Tests (Python 3.6)" -services: - - rabbitmq - cache: pip: true directories: @@ -80,9 +77,10 @@ before_install: - sudo add-apt-repository -y ppa:git-core/ppa - curl https://www.mongodb.org/static/pgp/server-3.4.asc | sudo apt-key add - - echo "deb [arch=amd64] http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.4 multiverse" | sudo tee -a /etc/apt/sources.list + # 2. Install RabbitMQ server # Work around for Travis timeout issues, see https://github.com/travis-ci/travis-ci/issues/9112 - sudo apt-get update --option Acquire::Retries=100 --option Acquire::http::Timeout="60" - - sudo apt-get install mongodb-org-server mongodb-org-shell git libffi-dev -y + - sudo apt-get install mongodb-org-server mongodb-org-shell rabbitmq-server git libffi-dev -y - pip install --upgrade "pip>=9.0,<9.1" - sudo pip install --upgrade "virtualenv==15.1.0" From ca332e103df6ca5d171887722b9fe558acfc6ce9 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 13:18:11 -0800 Subject: [PATCH 04/52] Remove bandit checks since they are extremly slow. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9a769aca53..d6a5a4ce47 100644 --- a/Makefile +++ b/Makefile @@ -827,7 +827,7 @@ debs: ci: ci-checks ci-unit ci-integration ci-mistral ci-packs-tests .PHONY: ci-checks -ci-checks: compile .generated-files-check .pylint .flake8 .bandit .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .st2client-install-check +ci-checks: compile .generated-files-check .pylint .flake8 .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .st2client-install-check .PHONY: ci-py3-unit ci-py3-unit: From 3b0a31854487af2f513fbe68ca1a308894c130cd Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 13:33:21 -0800 Subject: [PATCH 05/52] Tag logs on before_cache. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 0601b97fc5..569c805f91 100644 --- a/.travis.yml +++ b/.travis.yml @@ -125,3 +125,4 @@ script: # Alternative: use strict pip pinning, including git-based pip packages before_cache: - if [ ${TRAVIS_PULL_REQUEST} = 'false' ]; then rm -rf virtualenv/; fi + - cat logs/*.log From 94bfb2f99c1e04118aa6d42e8e182a42c292b441 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 13:42:22 -0800 Subject: [PATCH 06/52] Only cat actionrunner logs. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 569c805f91..eb0366473a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -125,4 +125,4 @@ script: # Alternative: use strict pip pinning, including git-based pip packages before_cache: - if [ ${TRAVIS_PULL_REQUEST} = 'false' ]; then rm -rf virtualenv/; fi - - cat logs/*.log + - cat logs/st2actionrunner*.log From dc7babdc51dc8c91292728a6a84c84d4976bf993 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 13:52:34 -0800 Subject: [PATCH 07/52] Add debug log statements. --- .../local_runner/tests/integration/test_localrunner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/contrib/runners/local_runner/tests/integration/test_localrunner.py b/contrib/runners/local_runner/tests/integration/test_localrunner.py index 31e684b722..5cf59f3f95 100644 --- a/contrib/runners/local_runner/tests/integration/test_localrunner.py +++ b/contrib/runners/local_runner/tests/integration/test_localrunner.py @@ -431,9 +431,11 @@ def test_script_with_parameters_parameter_serialization(self): runner = self._get_runner(action_db=action_db, entry_point=entry_point) runner.pre_run() - status, result, _ = runner.run(action_parameters=action_parameters) + status, result, output = runner.run(action_parameters=action_parameters) runner.post_run(status, result) + print(output) + self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue('PARAM_STRING=test string' in result['stdout']) self.assertTrue('PARAM_INTEGER=1' in result['stdout']) @@ -606,8 +608,9 @@ def test_large_stdout(self): runner = self._get_runner(action_db, entry_point=entry_point) runner.pre_run() char_count = 10 ** 6 # Note 10^7 succeeds but ends up being slow. - status, result, _ = runner.run({'chars': char_count}) + status, result, output = runner.run({'chars': char_count}) runner.post_run(status, result) + print(output) self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED) self.assertEquals(len(result['stdout']), char_count) From 36a8266b6b64c01589b02d0a0f184550bdca9f4e Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 14:07:35 -0800 Subject: [PATCH 08/52] Add more debug statements. --- .../runners/local_runner/tests/integration/test_localrunner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/runners/local_runner/tests/integration/test_localrunner.py b/contrib/runners/local_runner/tests/integration/test_localrunner.py index 5cf59f3f95..e3384a5807 100644 --- a/contrib/runners/local_runner/tests/integration/test_localrunner.py +++ b/contrib/runners/local_runner/tests/integration/test_localrunner.py @@ -434,6 +434,7 @@ def test_script_with_parameters_parameter_serialization(self): status, result, output = runner.run(action_parameters=action_parameters) runner.post_run(status, result) + print('done debug') print(output) self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED) @@ -610,6 +611,7 @@ def test_large_stdout(self): char_count = 10 ** 6 # Note 10^7 succeeds but ends up being slow. status, result, output = runner.run({'chars': char_count}) runner.post_run(status, result) + print('done debug') print(output) self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED) self.assertEquals(len(result['stdout']), char_count) From b3479421e00de596b9d36b102fce70e8979746a7 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 14:19:31 -0800 Subject: [PATCH 09/52] Also log debug result output. --- .../runners/local_runner/tests/integration/test_localrunner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/runners/local_runner/tests/integration/test_localrunner.py b/contrib/runners/local_runner/tests/integration/test_localrunner.py index e3384a5807..d3d790d6dd 100644 --- a/contrib/runners/local_runner/tests/integration/test_localrunner.py +++ b/contrib/runners/local_runner/tests/integration/test_localrunner.py @@ -435,6 +435,7 @@ def test_script_with_parameters_parameter_serialization(self): runner.post_run(status, result) print('done debug') + print(result) print(output) self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED) @@ -612,6 +613,7 @@ def test_large_stdout(self): status, result, output = runner.run({'chars': char_count}) runner.post_run(status, result) print('done debug') + print(result) print(output) self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED) self.assertEquals(len(result['stdout']), char_count) From bb50a39470078752e7429fd26ff8f3b5c6175c38 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 14:22:58 -0800 Subject: [PATCH 10/52] test a change. --- st2tests/st2tests/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py index f96aa774f3..a96a8fa986 100644 --- a/st2tests/st2tests/config.py +++ b/st2tests/st2tests/config.py @@ -28,7 +28,7 @@ LOG = logging.getLogger(__name__) -def parse_args(coordinator_noop=False): +def parse_args(coordinator_noop=True): _setup_config_opts(coordinator_noop=coordinator_noop) CONF(args=[]) From 6fa2fdcabfdcef76e2bfcfd71894f6379745279d Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 14:35:44 -0800 Subject: [PATCH 11/52] Add workaround for xenial build. --- scripts/travis/prepare-integration.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh index c9569f54af..97d5fd9738 100755 --- a/scripts/travis/prepare-integration.sh +++ b/scripts/travis/prepare-integration.sh @@ -61,3 +61,6 @@ st2 --version # as root can't write to logs/ directory and tests fail chmod 777 logs/ chmod 777 logs/* + +# Work around for travis +chmod -R 777 st2tests/st2tests/fixtures From 12da04d3f5f3e4283eaab170834072b1f4e128d8 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 14:52:59 -0800 Subject: [PATCH 12/52] Test a change. --- scripts/travis/prepare-integration.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh index 97d5fd9738..384bed5041 100755 --- a/scripts/travis/prepare-integration.sh +++ b/scripts/travis/prepare-integration.sh @@ -38,7 +38,9 @@ create_user() { fi # make sure requiretty is disabled. - sed -i "s/^Defaults\s\+requiretty/# Defaults requiretty/g" /etc/sudoers + sed -i -r "s/^Defaults\s+\+?requiretty/# Defaults requiretty/g" /etc/sudoers + else + echo "Stanley system user already exists, skipping addition." fi } From 695a8bb1def20b6f4f635446bd83aad24baf2b6c Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 14:57:09 -0800 Subject: [PATCH 13/52] Fix typo. --- scripts/travis/prepare-integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh index 384bed5041..48466b760e 100755 --- a/scripts/travis/prepare-integration.sh +++ b/scripts/travis/prepare-integration.sh @@ -16,7 +16,7 @@ mkdir -p ${STAN} VIRTUALENV_DIR=virtualenv create_user() { - if [ $(id -u ${SYSTEMUSER} &> /devnull; echo $?) != 0 ] + if [ $(id -u ${SYSTEMUSER} &> /dev/null; echo $?) != 0 ] then echo "###########################################################################################" echo "# Creating system user: ${SYSTEMUSER}" From 6d9a4d01e24367abe105969d3f27ec3edadce6dd Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 14:59:58 -0800 Subject: [PATCH 14/52] Make sure sudoers entry is always added, even if the system user already exists. --- .travis.yml | 1 + scripts/travis/prepare-integration.sh | 17 +++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index eb0366473a..159d62a94d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -111,6 +111,7 @@ before_script: - make play script: + # Clean up egg-info directories which get created when installing runners # NOTE: We enable code coverage and per test timing information on master so build can take twice # as long as PR builds - if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then COMMAND_THRESHOLD=$(expr ${COMMAND_THRESHOLD} \* 2); fi; ./scripts/travis/time-command.sh "make ${TASK}" ${COMMAND_THRESHOLD} diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh index 48466b760e..45a4f04c64 100755 --- a/scripts/travis/prepare-integration.sh +++ b/scripts/travis/prepare-integration.sh @@ -31,17 +31,18 @@ create_user() { cat /home/${SYSTEMUSER}/.ssh/stanley_rsa.pub >> /home/${SYSTEMUSER}/.ssh/authorized_keys chmod 0600 /home/${SYSTEMUSER}/.ssh/authorized_keys chown -R ${SYSTEMUSER}:${SYSTEMUSER} /home/${SYSTEMUSER} - if [ $(grep 'stanley' /etc/sudoers.d/* &> /dev/null; echo $?) != 0 ] - then - echo "${SYSTEMUSER} ALL=(ALL) NOPASSWD: SETENV: ALL" >> /etc/sudoers.d/st2 - chmod 0440 /etc/sudoers.d/st2 - fi - - # make sure requiretty is disabled. - sed -i -r "s/^Defaults\s+\+?requiretty/# Defaults requiretty/g" /etc/sudoers else echo "Stanley system user already exists, skipping addition." fi + + if [ $(grep 'stanley' /etc/sudoers.d/* &> /dev/null; echo $?) != 0 ] + then + echo "${SYSTEMUSER} ALL=(ALL) NOPASSWD: SETENV: ALL" >> /etc/sudoers.d/st2 + chmod 0440 /etc/sudoers.d/st2 + fi + + # make sure requiretty is disabled. + sed -i -r "s/^Defaults\s+\+?requiretty/# Defaults requiretty/g" /etc/sudoers } create_user From 873c57f8331e29832e17904fd9dbb9494d53c53b Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 12:57:57 -0800 Subject: [PATCH 15/52] Get rif of prometheus_client dependency which is not used by anything anymore. --- requirements.txt | 1 - st2common/in-requirements.txt | 1 - st2common/requirements.txt | 1 - 3 files changed, 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index a12aecb64e..3f50400f75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,6 @@ oslo.utils<=3.37.0,>=3.36.2 paramiko==2.4.2 passlib==1.7.1 prettytable -prometheus_client==0.1.1 prompt-toolkit==1.0.15 psutil==5.4.8 pyinotify==0.9.6 diff --git a/st2common/in-requirements.txt b/st2common/in-requirements.txt index 93fd72847e..1ac8820920 100644 --- a/st2common/in-requirements.txt +++ b/st2common/in-requirements.txt @@ -29,5 +29,4 @@ flex webob jsonpath-rw python-statsd -prometheus_client ujson diff --git a/st2common/requirements.txt b/st2common/requirements.txt index a5c9cbe9e7..e86c0688b7 100644 --- a/st2common/requirements.txt +++ b/st2common/requirements.txt @@ -14,7 +14,6 @@ mongoengine==0.16.3 networkx==1.11 oslo.config<1.13,>=1.12.1 paramiko==2.4.2 -prometheus_client==0.1.1 pymongo==3.7.2 python-dateutil==2.7.5 python-statsd==2.1.0 From c9c2abe8c09a508a87d43f5f641d5fda9b4c72d1 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 15:14:31 -0800 Subject: [PATCH 16/52] Test a change. --- st2tests/st2tests/fixtures/localrunner_pack/actions/text_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2tests/st2tests/fixtures/localrunner_pack/actions/text_gen.py b/st2tests/st2tests/fixtures/localrunner_pack/actions/text_gen.py index 8f8c925ea1..69782d8f5b 100755 --- a/st2tests/st2tests/fixtures/localrunner_pack/actions/text_gen.py +++ b/st2tests/st2tests/fixtures/localrunner_pack/actions/text_gen.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#!/usr/bin/env python from __future__ import absolute_import From 9028c1113dcfb5e8f8904862e5d9a433598834c2 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 15:19:49 -0800 Subject: [PATCH 17/52] Update prepare-integration script from bootstrap-deb script. --- scripts/travis/prepare-integration.sh | 59 +++++++++++++-------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh index 45a4f04c64..5304a1dc3c 100755 --- a/scripts/travis/prepare-integration.sh +++ b/scripts/travis/prepare-integration.sh @@ -7,41 +7,43 @@ if [ "$(whoami)" != 'root' ]; then fi # create and configure user -# proudly stolen from `./tools/st2_deploy.sh` -TYPE='debs' -SYSTEMUSER='stanley' -STAN="/home/${SYSTEMUSER}/${TYPE}" -mkdir -p ${STAN} - VIRTUALENV_DIR=virtualenv create_user() { - if [ $(id -u ${SYSTEMUSER} &> /dev/null; echo $?) != 0 ] - then - echo "###########################################################################################" - echo "# Creating system user: ${SYSTEMUSER}" - useradd ${SYSTEMUSER} - mkdir -p /home/${SYSTEMUSER}/.ssh - rm -Rf ${STAN}/* - chmod 0700 /home/${SYSTEMUSER}/.ssh - mkdir -p /home/${SYSTEMUSER}/${TYPE} - echo "###########################################################################################" - echo "# Generating system user ssh keys" - ssh-keygen -f /home/${SYSTEMUSER}/.ssh/stanley_rsa -P "" - cat /home/${SYSTEMUSER}/.ssh/stanley_rsa.pub >> /home/${SYSTEMUSER}/.ssh/authorized_keys - chmod 0600 /home/${SYSTEMUSER}/.ssh/authorized_keys - chown -R ${SYSTEMUSER}:${SYSTEMUSER} /home/${SYSTEMUSER} - else - echo "Stanley system user already exists, skipping addition." + echo "###########################################################################################" + echo "# Creating system user: stanley" + + if (! id stanley 2>/dev/null); then + useradd stanley + fi + + SYSTEM_HOME=$(echo ~stanley) + + mkdir -p ${SYSTEM_HOME}/.ssh + + if ! test -s ${SYSTEM_HOME}/.ssh/stanley_rsa; then + ssh-keygen -f ${SYSTEM_HOME}/.ssh/stanley_rsa -P "" fi - if [ $(grep 'stanley' /etc/sudoers.d/* &> /dev/null; echo $?) != 0 ] + if ! grep -s -q -f ${SYSTEM_HOME}/.ssh/stanley_rsa.pub ${SYSTEM_HOME}/.ssh/authorized_keys; then - echo "${SYSTEMUSER} ALL=(ALL) NOPASSWD: SETENV: ALL" >> /etc/sudoers.d/st2 - chmod 0440 /etc/sudoers.d/st2 + # Authorize key-base access + cat ${SYSTEM_HOME}/.ssh/stanley_rsa.pub >> ${SYSTEM_HOME}/.ssh/authorized_keys + fi + + chmod 0600 ${SYSTEM_HOME}/.ssh/authorized_keys + chmod 0700 ${SYSTEM_HOME}/.ssh + chown -R stanley:stanley ${SYSTEM_HOME} + + # Enable passwordless sudo + local STANLEY_SUDOERS="stanley ALL=(ALL) NOPASSWD: SETENV: ALL" + if ! grep -s -q ^"${STANLEY_SUDOERS}" /etc/sudoers.d/st2; then + echo '${STANLEY_SUDOERS}' >> /etc/sudoers.d/st2 fi - # make sure requiretty is disabled. + chmod 0440 /etc/sudoers.d/st2 + + # Disable requiretty for all users sed -i -r "s/^Defaults\s+\+?requiretty/# Defaults requiretty/g" /etc/sudoers } @@ -64,6 +66,3 @@ st2 --version # as root can't write to logs/ directory and tests fail chmod 777 logs/ chmod 777 logs/* - -# Work around for travis -chmod -R 777 st2tests/st2tests/fixtures From 7dc3c86bc02187a49b2a4e506a1c496eed153b18 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 19 Jan 2019 15:22:35 -0800 Subject: [PATCH 18/52] Fix syntax. --- scripts/travis/prepare-integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh index 5304a1dc3c..80564f8921 100755 --- a/scripts/travis/prepare-integration.sh +++ b/scripts/travis/prepare-integration.sh @@ -38,7 +38,7 @@ create_user() { # Enable passwordless sudo local STANLEY_SUDOERS="stanley ALL=(ALL) NOPASSWD: SETENV: ALL" if ! grep -s -q ^"${STANLEY_SUDOERS}" /etc/sudoers.d/st2; then - echo '${STANLEY_SUDOERS}' >> /etc/sudoers.d/st2 + echo "${STANLEY_SUDOERS}" >> /etc/sudoers.d/st2 fi chmod 0440 /etc/sudoers.d/st2 From ce24ead515b1016b45f8f33103392d942d93af40 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 12:32:44 +0100 Subject: [PATCH 19/52] Add a work around for tests on Travis being up to 10 times slower. It turns out that on AMQP publish on Travis, "Channel.open: (504) CHANNEL_ERROR - second 'channel.open' seen" error is returned almost on every publish which causes our connection retry wrapper to sleep for 10 seconds and slowing down the tests. I haven't been able to track down the root cause if this issue yet, but this workaround should work fine and cause no issues on production. --- .../st2common/transport/connection_retry_wrapper.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/st2common/st2common/transport/connection_retry_wrapper.py b/st2common/st2common/transport/connection_retry_wrapper.py index a657ed2afd..281e299eeb 100644 --- a/st2common/st2common/transport/connection_retry_wrapper.py +++ b/st2common/st2common/transport/connection_retry_wrapper.py @@ -35,7 +35,13 @@ def __init__(self, cluster_size): # No of nodes attempted. Starts at 1 since the self._nodes_attempted = 1 - def test_should_stop(self): + def test_should_stop(self, e=None): + # Special workaround for "(504) CHANNEL_ERROR - second 'channel.open' seen" which happens + # during tests on Travis and block and slown down the tests + # NOTE: This error is not fatal during tests and we can simply switch to a next connection + if "CHANNEL_ERROR - second 'channel.open' seen" in e: + return False, 0 + should_stop = True if self._nodes_attempted > self.cluster_size * self.cluster_retry: return should_stop, -1 @@ -117,7 +123,7 @@ def run(self, connection, wrapped_callback): wrapped_callback(connection=connection, channel=channel) should_stop = True except connection.connection_errors + connection.channel_errors as e: - should_stop, wait = self._retry_context.test_should_stop() + should_stop, wait = self._retry_context.test_should_stop(e) # reset channel to None to avoid any channel closing errors. At this point # in case of an exception there should be no channel but that is better to # guarantee. From 56b216bb822cf11cabeef41e0e9672d67bcc50b3 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 12:37:00 +0100 Subject: [PATCH 20/52] Log a message when we are sleeping due to the rabbitmq connection error. --- st2common/st2common/transport/connection_retry_wrapper.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/st2common/st2common/transport/connection_retry_wrapper.py b/st2common/st2common/transport/connection_retry_wrapper.py index 281e299eeb..e7d5d66986 100644 --- a/st2common/st2common/transport/connection_retry_wrapper.py +++ b/st2common/st2common/transport/connection_retry_wrapper.py @@ -14,6 +14,7 @@ # limitations under the License. from __future__ import absolute_import + import eventlet __all__ = ['ConnectionRetryWrapper', 'ClusterRetryContext'] @@ -39,8 +40,9 @@ def test_should_stop(self, e=None): # Special workaround for "(504) CHANNEL_ERROR - second 'channel.open' seen" which happens # during tests on Travis and block and slown down the tests # NOTE: This error is not fatal during tests and we can simply switch to a next connection + # without sleeping. if "CHANNEL_ERROR - second 'channel.open' seen" in e: - return False, 0 + return False, -1 should_stop = True if self._nodes_attempted > self.cluster_size * self.cluster_retry: @@ -132,7 +134,10 @@ def run(self, connection, wrapped_callback): # be notified so raise. if should_stop: raise + # -1, 0 and 1+ are handled properly by eventlet.sleep + self._logger.debug('Received RabbitMQ server error, sleeping for %s seconds ' + 'before retrying: %s' % (wait, str(e))) eventlet.sleep(wait) connection.close() From da9177a1cc105d7c98f2e7957bbec890faef1f99 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 12:55:02 +0100 Subject: [PATCH 21/52] Fix the if check. --- st2common/st2common/transport/connection_retry_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/transport/connection_retry_wrapper.py b/st2common/st2common/transport/connection_retry_wrapper.py index e7d5d66986..05eb667e1b 100644 --- a/st2common/st2common/transport/connection_retry_wrapper.py +++ b/st2common/st2common/transport/connection_retry_wrapper.py @@ -41,7 +41,7 @@ def test_should_stop(self, e=None): # during tests on Travis and block and slown down the tests # NOTE: This error is not fatal during tests and we can simply switch to a next connection # without sleeping. - if "CHANNEL_ERROR - second 'channel.open' seen" in e: + if "second 'channel.open' seen" in str(e): return False, -1 should_stop = True From 1849751658770985d79f26fb64bf8c3569f499ab Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 13:43:49 +0100 Subject: [PATCH 22/52] Pass -x flag to bandit. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d6a5a4ce47..aa2cb91b3e 100644 --- a/Makefile +++ b/Makefile @@ -267,7 +267,7 @@ bandit: requirements .bandit @echo @echo "==================== bandit ====================" @echo - . $(VIRTUALENV_DIR)/bin/activate; bandit -r $(COMPONENTS_WITH_RUNNERS) -lll + . $(VIRTUALENV_DIR)/bin/activate; bandit -r $(COMPONENTS_WITH_RUNNERS) -lll -x build,dist .PHONY: lint lint: requirements .lint From be1146fb659eeb62fef8030c3a42819df1113d29 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 14:15:49 +0100 Subject: [PATCH 23/52] Add a workaround for Travis xenial permission issue. --- scripts/travis/prepare-integration.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh index 80564f8921..b90db82a66 100755 --- a/scripts/travis/prepare-integration.sh +++ b/scripts/travis/prepare-integration.sh @@ -66,3 +66,6 @@ st2 --version # as root can't write to logs/ directory and tests fail chmod 777 logs/ chmod 777 logs/* + +# Workaround for Travis on Ubuntu Xenial +chown -R stanley:stanley /home/travis From 6ec174253144237651b5e67bf263d29f4955c6e8 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 14:35:24 +0100 Subject: [PATCH 24/52] Remove debug changes. --- .../tests/integration/test_localrunner.py | 12 ++---------- st2tests/st2tests/config.py | 2 +- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/contrib/runners/local_runner/tests/integration/test_localrunner.py b/contrib/runners/local_runner/tests/integration/test_localrunner.py index d3d790d6dd..d57a2f95ad 100644 --- a/contrib/runners/local_runner/tests/integration/test_localrunner.py +++ b/contrib/runners/local_runner/tests/integration/test_localrunner.py @@ -431,13 +431,8 @@ def test_script_with_parameters_parameter_serialization(self): runner = self._get_runner(action_db=action_db, entry_point=entry_point) runner.pre_run() - status, result, output = runner.run(action_parameters=action_parameters) + status, result, _ = runner.run(action_parameters=action_parameters) runner.post_run(status, result) - - print('done debug') - print(result) - print(output) - self.assertEqual(status, action_constants.LIVEACTION_STATUS_SUCCEEDED) self.assertTrue('PARAM_STRING=test string' in result['stdout']) self.assertTrue('PARAM_INTEGER=1' in result['stdout']) @@ -610,11 +605,8 @@ def test_large_stdout(self): runner = self._get_runner(action_db, entry_point=entry_point) runner.pre_run() char_count = 10 ** 6 # Note 10^7 succeeds but ends up being slow. - status, result, output = runner.run({'chars': char_count}) + status, result, _ = runner.run({'chars': char_count}) runner.post_run(status, result) - print('done debug') - print(result) - print(output) self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED) self.assertEquals(len(result['stdout']), char_count) diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py index a96a8fa986..f96aa774f3 100644 --- a/st2tests/st2tests/config.py +++ b/st2tests/st2tests/config.py @@ -28,7 +28,7 @@ LOG = logging.getLogger(__name__) -def parse_args(coordinator_noop=True): +def parse_args(coordinator_noop=False): _setup_config_opts(coordinator_noop=coordinator_noop) CONF(args=[]) From 553485091e3829c7bf2794325b644ea761792477 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 14:42:15 +0100 Subject: [PATCH 25/52] Use chmod instead of chown. --- scripts/travis/prepare-integration.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh index b90db82a66..ea6abf237b 100755 --- a/scripts/travis/prepare-integration.sh +++ b/scripts/travis/prepare-integration.sh @@ -67,5 +67,6 @@ st2 --version chmod 777 logs/ chmod 777 logs/* -# Workaround for Travis on Ubuntu Xenial -chown -R stanley:stanley /home/travis +# Workaround for Travis on Ubuntu Xenial so local runner integration tests work +# when executing them under user "stanley" +chmod 777 -R stanley:stanley /home/travis From f8b62b24850bb31a842bc1e82b603f8598ca641e Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 14:50:38 +0100 Subject: [PATCH 26/52] Fix typo. --- scripts/travis/prepare-integration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh index ea6abf237b..9cb143aaed 100755 --- a/scripts/travis/prepare-integration.sh +++ b/scripts/travis/prepare-integration.sh @@ -69,4 +69,4 @@ chmod 777 logs/* # Workaround for Travis on Ubuntu Xenial so local runner integration tests work # when executing them under user "stanley" -chmod 777 -R stanley:stanley /home/travis +chmod 777 -R /home/travis From a9fe3bff56bb116d568331827b73136e627551da Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 17:03:35 +0100 Subject: [PATCH 27/52] We need to pass --exe flag to nosetests for Travis CI workaround to work. --- Makefile | 2 +- scripts/travis/prepare-integration.sh | 1 + tox.ini | 24 ++++++++++++------------ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index a25c4a5eb6..8541ec1c24 100644 --- a/Makefile +++ b/Makefile @@ -581,7 +581,7 @@ endif echo "-----------------------------------------------------------"; \ . $(VIRTUALENV_DIR)/bin/activate; \ COVERAGE_FILE=.coverage.integration.$$(echo $$component | tr '/' '.') \ - nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \ + nosetests $(NOSE_OPTS) -s -v --exe $(NOSE_COVERAGE_FLAGS) \ $(NOSE_COVERAGE_PACKAGES) \ $$component/tests/integration || exit 1; \ echo "-----------------------------------------------------------"; \ diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh index 9cb143aaed..d952f1a086 100755 --- a/scripts/travis/prepare-integration.sh +++ b/scripts/travis/prepare-integration.sh @@ -69,4 +69,5 @@ chmod 777 logs/* # Workaround for Travis on Ubuntu Xenial so local runner integration tests work # when executing them under user "stanley" +# NOTE: We need to pass "--exe" flag to nosetests when using this workaround. chmod 777 -R /home/travis diff --git a/tox.ini b/tox.ini index 84acbf239d..f265ffc7f3 100644 --- a/tox.ini +++ b/tox.ini @@ -82,15 +82,15 @@ deps = virtualenv -e{toxinidir}/st2client -e{toxinidir}/st2common commands = - nosetests --rednose --immediate -sv st2actions/tests/integration/ - nosetests --rednose --immediate -sv st2api/tests/integration/ - nosetests --rednose --immediate -sv st2common/tests/integration/ - nosetests --rednose --immediate -sv st2debug/tests/integration/ - nosetests --rednose --immediate -sv st2exporter/tests/integration/ - nosetests --rednose --immediate -sv st2reactor/tests/integration/ - nosetests --rednose --immediate -sv contrib/runners/action_chain_runner/tests/integration/ - nosetests --rednose --immediate -sv contrib/runners/local_runner/tests/integration/ - nosetests --rednose --immediate -sv contrib/runners/mistral_v2/tests/integration/ - nosetests --rednose --immediate -sv contrib/runners/orquesta_runner/tests/integration/ - nosetests --rednose --immediate -sv st2tests/integration/orquesta/ - nosetests --rednose --immediate -sv contrib/runners/python_runner/tests/integration/ + nosetests --rednose --immediate -sv --exe st2actions/tests/integration/ + nosetests --rednose --immediate -sv --exe st2api/tests/integration/ + nosetests --rednose --immediate -sv --exe st2common/tests/integration/ + nosetests --rednose --immediate -sv --exe st2debug/tests/integration/ + nosetests --rednose --immediate -sv --exe st2exporter/tests/integration/ + nosetests --rednose --immediate -sv --exe st2reactor/tests/integration/ + nosetests --rednose --immediate -sv --exe contrib/runners/action_chain_runner/tests/integration/ + nosetests --rednose --immediate -sv --exe contrib/runners/local_runner/tests/integration/ + nosetests --rednose --immediate -sv --exe contrib/runners/mistral_v2/tests/integration/ + nosetests --rednose --immediate -sv --exe contrib/runners/orquesta_runner/tests/integration/ + nosetests --rednose --immediate -sv --exe st2tests/integration/orquesta/ + nosetests --rednose --immediate -sv --exe contrib/runners/python_runner/tests/integration/ From dcdc0e849ab2572362c6959441232ec8501d6140 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 17:30:05 +0100 Subject: [PATCH 28/52] Update comment, add some debug code. --- scripts/travis/prepare-integration.sh | 3 ++- st2reactor/tests/integration/test_garbage_collector.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh index d952f1a086..416e094c79 100755 --- a/scripts/travis/prepare-integration.sh +++ b/scripts/travis/prepare-integration.sh @@ -68,6 +68,7 @@ chmod 777 logs/ chmod 777 logs/* # Workaround for Travis on Ubuntu Xenial so local runner integration tests work -# when executing them under user "stanley" +# when executing them under user "stanley" (by default Travis checks out the +# code and runs tests under a different system user). # NOTE: We need to pass "--exe" flag to nosetests when using this workaround. chmod 777 -R /home/travis diff --git a/st2reactor/tests/integration/test_garbage_collector.py b/st2reactor/tests/integration/test_garbage_collector.py index 89dfbb619c..5407a70266 100644 --- a/st2reactor/tests/integration/test_garbage_collector.py +++ b/st2reactor/tests/integration/test_garbage_collector.py @@ -188,8 +188,10 @@ def test_garbage_collection(self): process = self._start_garbage_collector() # Give it some time to perform garbage collection and kill it - eventlet.sleep(10) + eventlet.sleep(12) process.send_signal(signal.SIGKILL) + print(process.stdout.read()) + print(process.stderr.read()) self.remove_process(process=process) # Old executions and corresponding objects should have been garbage collected @@ -236,8 +238,10 @@ def test_inquiry_garbage_collection(self): process = self._start_garbage_collector() # Give it some time to perform garbage collection and kill it - eventlet.sleep(10) + eventlet.sleep(12) process.send_signal(signal.SIGKILL) + print(process.stdout.read()) + print(process.stderr.read()) self.remove_process(process=process) # Expired Inquiries should have been garbage collected From d008d1ec6bba983bd9ecd5904013559e930275c2 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 19:10:12 +0100 Subject: [PATCH 29/52] Update test_garbage_collector.py --- st2reactor/tests/integration/test_garbage_collector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/st2reactor/tests/integration/test_garbage_collector.py b/st2reactor/tests/integration/test_garbage_collector.py index 5407a70266..540af24b6c 100644 --- a/st2reactor/tests/integration/test_garbage_collector.py +++ b/st2reactor/tests/integration/test_garbage_collector.py @@ -188,7 +188,7 @@ def test_garbage_collection(self): process = self._start_garbage_collector() # Give it some time to perform garbage collection and kill it - eventlet.sleep(12) + eventlet.sleep(15) process.send_signal(signal.SIGKILL) print(process.stdout.read()) print(process.stderr.read()) @@ -238,7 +238,7 @@ def test_inquiry_garbage_collection(self): process = self._start_garbage_collector() # Give it some time to perform garbage collection and kill it - eventlet.sleep(12) + eventlet.sleep(15) process.send_signal(signal.SIGKILL) print(process.stdout.read()) print(process.stderr.read()) From 84321c0913ed04b8b55364b79ba6e03d8ecb99a3 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 21:44:45 +0100 Subject: [PATCH 30/52] Add log statement. --- st2reactor/tests/integration/test_garbage_collector.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/st2reactor/tests/integration/test_garbage_collector.py b/st2reactor/tests/integration/test_garbage_collector.py index 5407a70266..60a87ad1ee 100644 --- a/st2reactor/tests/integration/test_garbage_collector.py +++ b/st2reactor/tests/integration/test_garbage_collector.py @@ -239,7 +239,11 @@ def test_inquiry_garbage_collection(self): # Give it some time to perform garbage collection and kill it eventlet.sleep(12) + print('======== log1') + print(process.stdout.read()) + print(process.stderr.read()) process.send_signal(signal.SIGKILL) + print('======== log') print(process.stdout.read()) print(process.stderr.read()) self.remove_process(process=process) From 06ba44cbb7c3d8c2a2c2c9a2bb7d56f61291ec13 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 21:45:01 +0100 Subject: [PATCH 31/52] Remove extra whitespace. --- tox.ini | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tox.ini b/tox.ini index f265ffc7f3..fb193b3449 100644 --- a/tox.ini +++ b/tox.ini @@ -83,14 +83,14 @@ deps = virtualenv -e{toxinidir}/st2common commands = nosetests --rednose --immediate -sv --exe st2actions/tests/integration/ - nosetests --rednose --immediate -sv --exe st2api/tests/integration/ - nosetests --rednose --immediate -sv --exe st2common/tests/integration/ - nosetests --rednose --immediate -sv --exe st2debug/tests/integration/ - nosetests --rednose --immediate -sv --exe st2exporter/tests/integration/ - nosetests --rednose --immediate -sv --exe st2reactor/tests/integration/ - nosetests --rednose --immediate -sv --exe contrib/runners/action_chain_runner/tests/integration/ - nosetests --rednose --immediate -sv --exe contrib/runners/local_runner/tests/integration/ - nosetests --rednose --immediate -sv --exe contrib/runners/mistral_v2/tests/integration/ - nosetests --rednose --immediate -sv --exe contrib/runners/orquesta_runner/tests/integration/ - nosetests --rednose --immediate -sv --exe st2tests/integration/orquesta/ - nosetests --rednose --immediate -sv --exe contrib/runners/python_runner/tests/integration/ + nosetests --rednose --immediate -sv --exe st2api/tests/integration/ + nosetests --rednose --immediate -sv --exe st2common/tests/integration/ + nosetests --rednose --immediate -sv --exe st2debug/tests/integration/ + nosetests --rednose --immediate -sv --exe st2exporter/tests/integration/ + nosetests --rednose --immediate -sv --exe st2reactor/tests/integration/ + nosetests --rednose --immediate -sv --exe contrib/runners/action_chain_runner/tests/integration/ + nosetests --rednose --immediate -sv --exe contrib/runners/local_runner/tests/integration/ + nosetests --rednose --immediate -sv --exe contrib/runners/mistral_v2/tests/integration/ + nosetests --rednose --immediate -sv --exe contrib/runners/orquesta_runner/tests/integration/ + nosetests --rednose --immediate -sv --exe st2tests/integration/orquesta/ + nosetests --rednose --immediate -sv --exe contrib/runners/python_runner/tests/integration/ From b7745ff83f8abb6f3d282d06447fe467e11cfafc Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 21:45:27 +0100 Subject: [PATCH 32/52] Use larger timeout. --- st2reactor/tests/integration/test_garbage_collector.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/st2reactor/tests/integration/test_garbage_collector.py b/st2reactor/tests/integration/test_garbage_collector.py index 60a87ad1ee..9cc86debec 100644 --- a/st2reactor/tests/integration/test_garbage_collector.py +++ b/st2reactor/tests/integration/test_garbage_collector.py @@ -238,10 +238,8 @@ def test_inquiry_garbage_collection(self): process = self._start_garbage_collector() # Give it some time to perform garbage collection and kill it - eventlet.sleep(12) + eventlet.sleep(15) print('======== log1') - print(process.stdout.read()) - print(process.stderr.read()) process.send_signal(signal.SIGKILL) print('======== log') print(process.stdout.read()) From 593048da29a87b3a32c5968271c691b3d35903ed Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 23:35:06 +0100 Subject: [PATCH 33/52] Don't use echo driver in st2.dev.conf since it causes too much noise during Travis integration builds. --- conf/st2.dev.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conf/st2.dev.conf b/conf/st2.dev.conf index c266534710..13b5bce84f 100644 --- a/conf/st2.dev.conf +++ b/conf/st2.dev.conf @@ -123,6 +123,8 @@ jitter_interval = 0 enable_common_libs = True [metrics] -driver = echo +# uncomment when debugging +#driver = echo +driver = noop host = 127.0.0.1 port = 8125 From bac279971066d30d7256d6f58a354e953393e035 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 21 Jan 2019 23:46:18 +0100 Subject: [PATCH 34/52] Pass --no-deps argument to python setup.py develop line in Makefile. This way we don't try to install dependencies twice which speeds up a build a bit (dependencies are already installed as part of "requirements" make target). --- Makefile | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8541ec1c24..f3c2dabe04 100644 --- a/Makefile +++ b/Makefile @@ -134,8 +134,12 @@ play: .PHONY: check check: requirements flake8 checklogs +# NOTE: We pass --no-deps to the script so we don't install all the +# package dependencies which are already installed as part of "requirements" +# make targets. This speeds up the build .PHONY: install-runners install-runners: + @echo "" @echo "================== INSTALL RUNNERS ====================" @echo "" @@ -143,7 +147,7 @@ install-runners: echo "==========================================================="; \ echo "Installing runner:" $$component; \ echo "==========================================================="; \ - (. $(VIRTUALENV_DIR)/bin/activate; cd $$component; python setup.py develop); \ + (. $(VIRTUALENV_DIR)/bin/activate; cd $$component; python setup.py develop --no-deps); \ done .PHONY: checklogs @@ -392,7 +396,10 @@ requirements: virtualenv .sdist-requirements install-runners done # Install st2common package to load drivers defined in st2common setup.py - (cd st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop) + # NOTE: We pass --no-deps to the script so we don't install all the + # package dependencies which are already installed as part of "requirements" + # make targets. This speeds up the build + (cd st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps) # Note: We install prance here and not as part of any component @@ -401,7 +408,10 @@ requirements: virtualenv .sdist-requirements install-runners $(VIRTUALENV_DIR)/bin/pip install "prance==0.6.1" # Install st2common to register metrics drivers - (cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop) + # NOTE: We pass --no-deps to the script so we don't install all the + # package dependencies which are already installed as part of "requirements" + # make targets. This speeds up the build + (cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps) # Some of the tests rely on submodule so we need to make sure submodules are check out git submodule update --init --recursive From e522e41f30c5237b9b7210dd0c813f89ff4b199b Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 00:02:38 +0100 Subject: [PATCH 35/52] Remove workaround we don't need. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 159d62a94d..60ab25992e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -126,4 +126,3 @@ script: # Alternative: use strict pip pinning, including git-based pip packages before_cache: - if [ ${TRAVIS_PULL_REQUEST} = 'false' ]; then rm -rf virtualenv/; fi - - cat logs/st2actionrunner*.log From 9bd0dafe7fd14dc0b7b8928f9cfabd6ab814b308 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 00:02:55 +0100 Subject: [PATCH 36/52] Revert "Don't use echo driver in st2.dev.conf since it causes too much noise" This reverts commit 593048da29a87b3a32c5968271c691b3d35903ed. --- conf/st2.dev.conf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/conf/st2.dev.conf b/conf/st2.dev.conf index 13b5bce84f..c266534710 100644 --- a/conf/st2.dev.conf +++ b/conf/st2.dev.conf @@ -123,8 +123,6 @@ jitter_interval = 0 enable_common_libs = True [metrics] -# uncomment when debugging -#driver = echo -driver = noop +driver = echo host = 127.0.0.1 port = 8125 From 3db50f15c9d26644165f4bce5154ac23f5054145 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 00:28:00 +0100 Subject: [PATCH 37/52] Test a change. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 60ab25992e..fc75b39ca5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,7 +70,7 @@ cache: - virtualenv/ # NOTE: Caching .tox speeds up py3 build for 30-60 seconds, but causes issues when dependencies # are updated so it's disabled - #- .tox/ + - .tox/ before_install: # 1. Install MongoDB 3.4 and latest version of git From ce1b77a8ef54ec5670078ef3df60a68eeb22bf07 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 00:29:17 +0100 Subject: [PATCH 38/52] Remove .tox for master builds. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fc75b39ca5..47dcddf61a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -125,4 +125,4 @@ script: # See: https://docs.travis-ci.com/user/caching/#Pull-request-builds-and-caches # Alternative: use strict pip pinning, including git-based pip packages before_cache: - - if [ ${TRAVIS_PULL_REQUEST} = 'false' ]; then rm -rf virtualenv/; fi + - if [ ${TRAVIS_PULL_REQUEST} = 'false' ]; then rm -rf virtualenv/ ; rm -rf .tox/; fi From 657f4c8582629607906a22eb328805ad7f2ae77d Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 00:36:30 +0100 Subject: [PATCH 39/52] Remove debug log. --- st2reactor/tests/integration/test_garbage_collector.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/st2reactor/tests/integration/test_garbage_collector.py b/st2reactor/tests/integration/test_garbage_collector.py index a208e8a278..351e356210 100644 --- a/st2reactor/tests/integration/test_garbage_collector.py +++ b/st2reactor/tests/integration/test_garbage_collector.py @@ -190,8 +190,6 @@ def test_garbage_collection(self): # Give it some time to perform garbage collection and kill it eventlet.sleep(15) process.send_signal(signal.SIGKILL) - print(process.stdout.read()) - print(process.stderr.read()) self.remove_process(process=process) # Old executions and corresponding objects should have been garbage collected @@ -240,9 +238,6 @@ def test_inquiry_garbage_collection(self): # Give it some time to perform garbage collection and kill it eventlet.sleep(15) process.send_signal(signal.SIGKILL) - print('======== log') - print(process.stdout.read()) - print(process.stderr.read()) self.remove_process(process=process) # Expired Inquiries should have been garbage collected From 67bc810245fdeb554fc9f74897ac86d88670d41f Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 12:45:15 +0100 Subject: [PATCH 40/52] Remove -l flag, comment out .tox caching. --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 47dcddf61a..c5871dbdeb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,7 +70,7 @@ cache: - virtualenv/ # NOTE: Caching .tox speeds up py3 build for 30-60 seconds, but causes issues when dependencies # are updated so it's disabled - - .tox/ + #- .tox/ before_install: # 1. Install MongoDB 3.4 and latest version of git @@ -97,7 +97,7 @@ before_script: # Clean up any old MongoDB 3.4 data files laying around and make sure mongodb user can write to it - sudo rm -rf /var/lib/mongodb ; sudo mkdir /var/lib/mongodb ; sudo chown -R mongodb:mongodb /var/lib/mongodb - sudo service mongod restart ; sleep 5 - - sudo service mongod status -l + - sudo service mongod status - sudo tail -30 /var/log/mongodb/mongod.log - mongod --version - git --version @@ -125,4 +125,4 @@ script: # See: https://docs.travis-ci.com/user/caching/#Pull-request-builds-and-caches # Alternative: use strict pip pinning, including git-based pip packages before_cache: - - if [ ${TRAVIS_PULL_REQUEST} = 'false' ]; then rm -rf virtualenv/ ; rm -rf .tox/; fi + - if [ ${TRAVIS_PULL_REQUEST} = 'false' ]; then rm -rf virtualenv/; fi From 234e667a4b851c0c58acab697e68c4e9c8fdf857 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 12:48:47 +0100 Subject: [PATCH 41/52] Switch back to apt addon for installing apt packages. Container infrastructure has been removed and apt addon works fine with new and old infrastructure. --- .travis.yml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index c5871dbdeb..f6b438253f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,6 +60,20 @@ matrix: python: 3.6 name: "Integration Tests (Python 3.6)" +addons: + apt: + sources: + - mongodb-upstart + - sourceline: 'deb [arch=amd64] http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.4 multiverse' + key_url: 'https://www.mongodb.org/static/pgp/server-3.4.asc' + - sourceline: 'ppa:git-core/ppa' + packages: + - mongodb-org-server + - mongodb-org-shell + - rabbitmq-server + - git + - libffi-dev + cache: pip: true directories: @@ -73,14 +87,8 @@ cache: #- .tox/ before_install: - # 1. Install MongoDB 3.4 and latest version of git - - sudo add-apt-repository -y ppa:git-core/ppa - - curl https://www.mongodb.org/static/pgp/server-3.4.asc | sudo apt-key add - - - echo "deb [arch=amd64] http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.4 multiverse" | sudo tee -a /etc/apt/sources.list - # 2. Install RabbitMQ server - # Work around for Travis timeout issues, see https://github.com/travis-ci/travis-ci/issues/9112 - - sudo apt-get update --option Acquire::Retries=100 --option Acquire::http::Timeout="60" - - sudo apt-get install mongodb-org-server mongodb-org-shell rabbitmq-server git libffi-dev -y + # Work around for apt Travis timeout issues, see https://github.com/travis-ci/travis-ci/issues/9112 + #- sudo apt-get update --option Acquire::Retries=100 --option Acquire::http::Timeout="60" - pip install --upgrade "pip>=9.0,<9.1" - sudo pip install --upgrade "virtualenv==15.1.0" From af09c7326724ea901665c9d6cf821c6a7397464a Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 14:13:10 +0100 Subject: [PATCH 42/52] Pass --no-dep to python setup.py develop to speed up the build and avoid duplicate dependency installation. --- scripts/travis/install-requirements.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/travis/install-requirements.sh b/scripts/travis/install-requirements.sh index 4b5d6d2f9e..5e34660712 100755 --- a/scripts/travis/install-requirements.sh +++ b/scripts/travis/install-requirements.sh @@ -3,16 +3,16 @@ if [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then pip install "tox==3.5.2" - # Install runners + # Install runners . virtualenv/bin/activate - CURRENT_DIR=`pwd` - for RUNNER in `ls -d $CURRENT_DIR/contrib/runners/*` - do - echo "Installing runner: $RUNNER..." + CURRENT_DIR=`pwd` + for RUNNER in `ls -d $CURRENT_DIR/contrib/runners/*` + do + echo "Installing runner: $RUNNER..." cd $RUNNER - python setup.py develop - done + python setup.py develop --no-deps + done # NOTE: We create the environment and install the dependencies first. This # means that the subsequent tox build / test command has a stable run time From 8fee73bb37c33671aa076376c3e4b6f72574023e Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 14:17:27 +0100 Subject: [PATCH 43/52] No need to perform step which already happens as part of Make target. --- .travis.yml | 3 ++- Makefile | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f6b438253f..09ba93e812 100644 --- a/.travis.yml +++ b/.travis.yml @@ -94,7 +94,8 @@ before_install: install: - ./scripts/travis/install-requirements.sh - - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi + # NOTE: This happes inside make targer + #- if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi # Let's enable rabbitmqadmin # See https://github.com/messagebus/lapine/wiki/Testing-on-Travis. diff --git a/Makefile b/Makefile index f3c2dabe04..cb55347027 100644 --- a/Makefile +++ b/Makefile @@ -401,7 +401,6 @@ requirements: virtualenv .sdist-requirements install-runners # make targets. This speeds up the build (cd st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps) - # Note: We install prance here and not as part of any component # requirements.txt because it has a conflict with our dependency (requires # new version of requests) which we cant resolve at this moment From 4769fcb505f238fa057feade6564568ee610f1ea Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 14:21:07 +0100 Subject: [PATCH 44/52] Make sure we don't have conflicting dependency versions in test-requirements.txt and requirements.txt. --- scripts/travis/install-requirements.sh | 2 +- test-requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/travis/install-requirements.sh b/scripts/travis/install-requirements.sh index 5e34660712..57faa43a31 100755 --- a/scripts/travis/install-requirements.sh +++ b/scripts/travis/install-requirements.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash if [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then pip install "tox==3.5.2" diff --git a/test-requirements.txt b/test-requirements.txt index 287c6a9500..35b350ef48 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -20,8 +20,8 @@ nose-parallel==0.3.1 # Required by st2client tests pyyaml<4.0,>=3.12 RandomWords -gunicorn==19.7.1 -psutil==5.4.5 +gunicorn==19.9.0 +psutil==5.4.8 webtest==2.0.25 rstcheck>=3.3.1,<3.4 pyrabbit From 9cec88fb37ec72b11ab1691a76b617e29eb8396d Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 14:45:28 +0100 Subject: [PATCH 45/52] User is already added as part of add-itest-user.sh script, no need to add it twice. --- .circle/add-itest-user.sh | 8 ++++- .travis.yml | 3 +- scripts/travis/prepare-integration.sh | 46 --------------------------- 3 files changed, 8 insertions(+), 49 deletions(-) diff --git a/.circle/add-itest-user.sh b/.circle/add-itest-user.sh index 3a8c94f2cc..b78a62e5b9 100755 --- a/.circle/add-itest-user.sh +++ b/.circle/add-itest-user.sh @@ -1,6 +1,12 @@ -#!/bin/bash +#!/usr/bin/env bash + set -e +if [ "$(whoami)" != 'root' ]; then + echo 'Please run with sudo' + exit 2 +fi + # Create an SSH system user (default `stanley` user may be already created) if (! id stanley 2>/dev/null); then useradd stanley diff --git a/.travis.yml b/.travis.yml index 09ba93e812..f6b438253f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -94,8 +94,7 @@ before_install: install: - ./scripts/travis/install-requirements.sh - # NOTE: This happes inside make targer - #- if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi + - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi # Let's enable rabbitmqadmin # See https://github.com/messagebus/lapine/wiki/Testing-on-Travis. diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh index 416e094c79..53d5fdd163 100755 --- a/scripts/travis/prepare-integration.sh +++ b/scripts/travis/prepare-integration.sh @@ -6,52 +6,6 @@ if [ "$(whoami)" != 'root' ]; then exit 2 fi -# create and configure user -VIRTUALENV_DIR=virtualenv - -create_user() { - echo "###########################################################################################" - echo "# Creating system user: stanley" - - if (! id stanley 2>/dev/null); then - useradd stanley - fi - - SYSTEM_HOME=$(echo ~stanley) - - mkdir -p ${SYSTEM_HOME}/.ssh - - if ! test -s ${SYSTEM_HOME}/.ssh/stanley_rsa; then - ssh-keygen -f ${SYSTEM_HOME}/.ssh/stanley_rsa -P "" - fi - - if ! grep -s -q -f ${SYSTEM_HOME}/.ssh/stanley_rsa.pub ${SYSTEM_HOME}/.ssh/authorized_keys; - then - # Authorize key-base access - cat ${SYSTEM_HOME}/.ssh/stanley_rsa.pub >> ${SYSTEM_HOME}/.ssh/authorized_keys - fi - - chmod 0600 ${SYSTEM_HOME}/.ssh/authorized_keys - chmod 0700 ${SYSTEM_HOME}/.ssh - chown -R stanley:stanley ${SYSTEM_HOME} - - # Enable passwordless sudo - local STANLEY_SUDOERS="stanley ALL=(ALL) NOPASSWD: SETENV: ALL" - if ! grep -s -q ^"${STANLEY_SUDOERS}" /etc/sudoers.d/st2; then - echo "${STANLEY_SUDOERS}" >> /etc/sudoers.d/st2 - fi - - chmod 0440 /etc/sudoers.d/st2 - - # Disable requiretty for all users - sed -i -r "s/^Defaults\s+\+?requiretty/# Defaults requiretty/g" /etc/sudoers -} - -create_user - -# install screen -apt-get install -y screen - # Activate the virtualenv created during make requirements phase source ./virtualenv/bin/activate From 48705ff6fafa4ca63b8f8e275bd73c031b4acb65 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 15:28:31 +0100 Subject: [PATCH 46/52] Try using price to see if the build finishes faster. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f6b438253f..7acaa19c27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ # Used old infrastructure, needed for integration tests: # http://docs.travis-ci.com/user/workers/standard-infrastructure/ sudo: required -dist: xenial +dist: precise language: python branches: From 17a3880c4ab66949124edce67c2de75f246ae759 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 15:32:26 +0100 Subject: [PATCH 47/52] Try using trusty. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7acaa19c27..9522d4dcd0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ # Used old infrastructure, needed for integration tests: # http://docs.travis-ci.com/user/workers/standard-infrastructure/ sudo: required -dist: precise +dist: trusty language: python branches: From 1fcb75ec0128f06b0847513cfd64d640f2093c09 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 15:34:49 +0100 Subject: [PATCH 48/52] Use xenial. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9522d4dcd0..f6b438253f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ # Used old infrastructure, needed for integration tests: # http://docs.travis-ci.com/user/workers/standard-infrastructure/ sudo: required -dist: trusty +dist: xenial language: python branches: From 5d804f0939c6e2516da91d5970b25bfe6f43c445 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 15:38:04 +0100 Subject: [PATCH 49/52] Try using trusty. --- .travis.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f6b438253f..bbbd8566f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ # Used old infrastructure, needed for integration tests: # http://docs.travis-ci.com/user/workers/standard-infrastructure/ sudo: required -dist: xenial +dist: trusty language: python branches: @@ -67,10 +67,15 @@ addons: - sourceline: 'deb [arch=amd64] http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.4 multiverse' key_url: 'https://www.mongodb.org/static/pgp/server-3.4.asc' - sourceline: 'ppa:git-core/ppa' + - sourceline: deb https://packages.erlang-solutions.com/ubuntu trusty contrib + key_url: https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc + - sourceline: deb https://dl.bintray.com/rabbitmq/debian trusty main + key_url: https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc packages: - mongodb-org-server - mongodb-org-shell - - rabbitmq-server + - esl-erlang=1:20.1 + - rabbitmq-server=3.7.0-1 - git - libffi-dev From 5d37fa9b81915d514fc157c293e023227bdbf59f Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 15:41:39 +0100 Subject: [PATCH 50/52] Switch back to precise. --- .travis.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index bbbd8566f3..e0a4ddee06 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ # Used old infrastructure, needed for integration tests: # http://docs.travis-ci.com/user/workers/standard-infrastructure/ sudo: required -dist: trusty +dist: precise language: python branches: @@ -67,18 +67,18 @@ addons: - sourceline: 'deb [arch=amd64] http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.4 multiverse' key_url: 'https://www.mongodb.org/static/pgp/server-3.4.asc' - sourceline: 'ppa:git-core/ppa' - - sourceline: deb https://packages.erlang-solutions.com/ubuntu trusty contrib - key_url: https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc - - sourceline: deb https://dl.bintray.com/rabbitmq/debian trusty main - key_url: https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc packages: - mongodb-org-server - mongodb-org-shell - - esl-erlang=1:20.1 - - rabbitmq-server=3.7.0-1 + # NOTE: Uncomment below for Xenial + # -rabbitmq-server - git - libffi-dev +# NOTE: Remove / comment services section below for Xenial +services: + - rabbitmq + cache: pip: true directories: From ade1986ce08843cb14b67c5394d6c6bb141f6af2 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 15:55:13 +0100 Subject: [PATCH 51/52] Only apply workaround for permissions issue to /home/travis when running on Xenial. --- scripts/travis/prepare-integration.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh index 53d5fdd163..deb77d620f 100755 --- a/scripts/travis/prepare-integration.sh +++ b/scripts/travis/prepare-integration.sh @@ -6,6 +6,8 @@ if [ "$(whoami)" != 'root' ]; then exit 2 fi +UBUNTU_VERSION=`lsb_release -a 2>&1 | grep Codename | grep -v "LSB" | awk '{print $2}'` + # Activate the virtualenv created during make requirements phase source ./virtualenv/bin/activate @@ -25,4 +27,7 @@ chmod 777 logs/* # when executing them under user "stanley" (by default Travis checks out the # code and runs tests under a different system user). # NOTE: We need to pass "--exe" flag to nosetests when using this workaround. -chmod 777 -R /home/travis +if [ "${UBUNTU_VERSION}" == "xenial" ]; then + echo "Applying workaround for stanley user permissions issue to /home/travis on Xenial" + chmod 777 -R /home/travis +fi From 3838a53c77b1336170a7cb43815314325f82f62f Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 22 Jan 2019 16:00:50 +0100 Subject: [PATCH 52/52] Add comment to travis config. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e0a4ddee06..78d703392a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ # Used old infrastructure, needed for integration tests: # http://docs.travis-ci.com/user/workers/standard-infrastructure/ sudo: required +# NOTE: We use precise because tests finish faster than on Xenial dist: precise language: python