Changes for v2.10.2 release

.travis.yml

@@ -1,6 +1,7 @@
 # Used old infrastructure, needed for integration tests:
 # http://docs.travis-ci.com/user/workers/standard-infrastructure/
 sudo: required
+# NOTE: We use precise because tests finish faster than on Xenial
 dist: precise
 language: python
 
@@ -55,13 +56,30 @@ matrix:
       name: "Lint Checks, Packs Tests (Python 2.7)"
     - env: TASK="compilepy3 ci-py3-unit" CACHE_NAME=py3 COMMAND_THRESHOLD=680
       python: 3.6
-      name: "Unit Tests (Python 3.6)"
+      name: "Unit Tests, Pack Tests (Python 3.6)"
     - env: TASK="ci-py3-integration" CACHE_NAME=py3 COMMAND_THRESHOLD=310
       python: 3.6
       name: "Integration Tests (Python 3.6)"
 
-services:
-  - rabbitmq
+addons:
+  apt:
+    sources:
+      - mongodb-upstart
+      - sourceline: 'deb [arch=amd64] http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.4 multiverse'
+        key_url: 'https://www.mongodb.org/static/pgp/server-3.4.asc'
+      # NOTE: Precise repo doesn't contain Erlang 20.x, latest version is 19.x so we need to use RabbitMQ 3.7.6
+      #- sourceline: 'deb [arch=amd64] http://packages.erlang-solutions.com/ubuntu precise contrib'
+      #  key_url: 'https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc'
+      #- sourceline: 'deb [arch=amd64] https://dl.bintray.com/rabbitmq/debian precise rabbitmq-server-v3.6.x'
+      #  key_url: 'https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc'
+      - sourceline: 'ppa:git-core/ppa'
+    packages:
+      - mongodb-org-server
+      - mongodb-org-shell
+      - erlang
+      - rabbitmq-server
+      - git
+      - libffi-dev
 
 cache:
   pip: true
@@ -76,39 +94,39 @@ cache:
     #- .tox/
 
 before_install:
-  # 1. Install MongoDB 3.4 and latest version of git
-  - sudo add-apt-repository -y ppa:git-core/ppa
-  - curl https://www.mongodb.org/static/pgp/server-3.4.asc | sudo apt-key add -
-  - echo "deb [arch=amd64] http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.4 multiverse" | sudo tee -a /etc/apt/sources.list
-  # Work around for Travis timeout issues, see https://github.com/travis-ci/travis-ci/issues/9112
-  - sudo apt-get update --option Acquire::Retries=100 --option Acquire::http::Timeout="60"
-  - sudo apt-get install mongodb-org-server mongodb-org-shell git libffi-dev -y
+  # Work around for apt Travis timeout issues, see https://github.com/travis-ci/travis-ci/issues/9112
+  #- sudo apt-get update --option Acquire::Retries=100 --option Acquire::http::Timeout="60"
   - pip install --upgrade "pip>=9.0,<9.1"
   - sudo pip install --upgrade "virtualenv==15.1.0"
 
 install:
   - ./scripts/travis/install-requirements.sh
-  - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi
+  - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'ci-checks ci-packs-tests' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi
 
 # Let's enable rabbitmqadmin
 # See https://github.com/messagebus/lapine/wiki/Testing-on-Travis.
 before_script:
-  # key_url no longer works for APT addon
   # Use a custom mongod.conf which uses various speed optimizations
   - sudo cp scripts/travis/mongod.conf /etc/mongod.conf
   # Clean up any old MongoDB 3.4 data files laying around and make sure mongodb user can write to it
   - sudo rm -rf /var/lib/mongodb ; sudo mkdir /var/lib/mongodb ; sudo chown -R mongodb:mongodb /var/lib/mongodb
   - sudo service mongod restart ; sleep 5
   - sudo service mongod status
-  - tail -30 /var/log/mongodb/mongod.log
-  - mongod --version
-  - git --version
-  - pip --version
-  - virtualenv --version
+  - sudo tail -n 30 /var/log/mongodb/mongod.log
+  # Use custom RabbitMQ config which enables SSL / TLS listener on port 5671 with test certs
+  - sudo cp scripts/travis/rabbitmq.config /etc/rabbitmq/rabbitmq.config
+  # Install rabbitmq_management RabbitMQ plugin
+  - sudo service rabbitmq-server restart ; sleep 5
   - sudo rabbitmq-plugins enable rabbitmq_management
   - sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
   - sudo chmod +x /usr/local/bin/rabbitmqadmin
   - sudo service rabbitmq-server restart
+  - sudo tail -n 30 /var/log/rabbitmq/*
+  # Print various binary versions
+  - mongod --version
+  - git --version
+  - pip --version
+  - virtualenv --version
   # Print out various environment variables info
   - make play
 

CHANGELOG.rst

@@ -1,6 +1,78 @@
 Changelog
 =========
 
+in development
+--------------
+
+Added
+~~~~~
+
+* Add support for various new SSL / TLS related config options (``ssl_keyfile``, ``ssl_certfile``,
+  ``ssl_ca_certs``, ``ssl_certfile``, ``authentication_mechanism``) to the ``messaging`` section in
+  ``st2.conf`` config file.
+
+  With those config options, user can configure things such as client based certificate
+  authentication, client side verification of a server certificate against a specific CA bundle, etc.
+
+  NOTE: Those options are only supported when using a default and officially supported AMQP backend
+  with RabbitMQ server. (new feature) #4541
+* Add metrics instrumentation to the ``st2notifier`` service. For the available / exposed metrics,
+  please refer to https://docs.stackstorm.com/reference/metrics.html. (improvement) #4536
+
+Changed
+~~~~~~~
+
+* Update logging code so we exclude log messages with log level ``AUDIT`` from a default service
+  log file (e.g. ``st2api.log``). Log messages with level ``AUDIT`` are already logged in a
+  dedicated service audit log file (e.g. ``st2api.audit.log``) so there is no need for them to also
+  be duplicated and included in regular service log file.
+
+  NOTE: To aid with debugging, audit log messages are also included in a regular log file when log
+  level is set to ``DEBUG`` or ``system.debug`` config option is set to ``True``.
+
+  Reported by Nick Maludy. (improvement) #4538 #4502
+* Update ``pyyaml`` dependency to the latest version. This latest version fixes an issue which
+  could result in a code execution vulnerability if code uses ``yaml.load`` in an unsafe manner
+  on untrusted input.
+
+  NOTE: StackStorm platform itself is not affected, because we already used ``yaml.safe_load``
+  everywhere.
+
+  Only custom packs which use ``yaml.load`` with non trusted user input could potentially be
+  affected. (improvement) #4510 #4552 #4554
+* Update Orquesta to ``v0.4``. #4551
+
+Fixed
+~~~~~
+
+* Fixed the ``packs.pack_install`` / ``!pack install {{ packs }}`` action-alias to not have
+  redundant patterns. Previously this prevented it from being executed via
+  ``st2 action-alias execute 'pack install xxx'``. #4511
+
+  Contributed by Nick Maludy (Encore Technologies)
+* Fix datastore value encryption and make sure it also works correctly for unicode (non-ascii)
+  values.
+
+  Reported by @dswebbthg, @nickbaum. (bug fix) #4513 #4527 #4528
+* Fix a bug with action positional parameter serialization used in local and remote script runner
+  not working correctly with non-ascii (unicode) values.
+
+  This would prevent actions such as ``core.sendmail`` which utilize positional parameters from
+  working correctly when a unicode value was provided.
+
+  Reported by @johandahlberg (bug fix) #4533
+* Fix ``core.sendmail`` action so it specifies ``charset=UTF-8`` in the ``Content-Type`` email
+  header. This way it works correctly when an email subject and / or body contains unicode data.
+
+  Reported by @johandahlberg (bug fix) #4533 4534
+* Fix CLI ``st2 apikey load`` not being idempotent and API endpoint ``/api/v1/apikeys`` not
+  honoring desired ``ID`` for the new record creation. #4542
+* Moved the lock from concurrency policies into the scheduler to fix a race condition when there
+  are multiple scheduler instances scheduling execution for action with concurrency policies.
+  #4481 (bug fix)
+* Add retries to scheduler to handle temporary hiccup in DB connection. Refactor scheduler
+  service to return proper exit code when there is a failure. #4539 (bug fix)
+
 2.10.1 - December 19, 2018
 --------------------------
 

Makefile

@@ -132,8 +132,11 @@ play:
 	@echo
 
 .PHONY: check
-check: requirements flake8 checklogs
+check: check-requirements flake8 checklogs
 
+# NOTE: We pass --no-deps to the script so we don't install all the
+# package dependencies which are already installed as part of "requirements"
+# make targets. This speeds up the build
 .PHONY: install-runners
 install-runners:
 	@echo ""
@@ -143,9 +146,18 @@ install-runners:
 		echo "==========================================================="; \
 		echo "Installing runner:" $$component; \
 		echo "==========================================================="; \
-        (. $(VIRTUALENV_DIR)/bin/activate; cd $$component; python setup.py develop); \
+		(. $(VIRTUALENV_DIR)/bin/activate; cd $$component; python setup.py develop --no-deps); \
 	done
 
+.PHONY: check-requirements
+check-requirements: requirements
+	@echo
+	@echo "============== CHECKING REQUIREMENTS =============="
+	@echo
+	# Update requirements and then make sure no files were changed
+	git status -- *requirements.txt */*requirements.txt | grep -q "nothing to commit"
+	@echo "All requirements files up-to-date!"
+
 .PHONY: checklogs
 checklogs:
 	@echo
@@ -175,6 +187,19 @@ configgen: requirements .configgen
 	@echo "================== pylint ===================="
 	@echo
 	# Lint st2 components
+	@for component in $(COMPONENTS); do\
+		echo "==========================================================="; \
+		echo "Running pylint on" $$component; \
+		echo "==========================================================="; \
+		. $(VIRTUALENV_DIR)/bin/activate ; pylint -j $(PYLINT_CONCURRENCY) -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models --load-plugins=pylint_plugins.db_models $$component/$$component || exit 1; \
+	done
+	# Lint runner modules and packages
+	@for component in $(COMPONENTS_RUNNERS); do\
+		echo "==========================================================="; \
+		echo "Running pylint on" $$component; \
+		echo "==========================================================="; \
+		. $(VIRTUALENV_DIR)/bin/activate ; pylint -j $(PYLINT_CONCURRENCY) -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models --load-plugins=pylint_plugins.db_models $$component/*.py || exit 1; \
+	done
 	# Lint Python pack management actions
 	. $(VIRTUALENV_DIR)/bin/activate; pylint -j $(PYLINT_CONCURRENCY) -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/packs/actions/*.py || exit 1;
 	. $(VIRTUALENV_DIR)/bin/activate; pylint -j $(PYLINT_CONCURRENCY) -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/packs/actions/*/*.py || exit 1;
@@ -827,7 +852,7 @@ debs:
 ci: ci-checks ci-unit ci-integration ci-mistral ci-packs-tests
 
 .PHONY: ci-checks
-ci-checks: compile .generated-files-check .pylint .flake8 .bandit .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .st2client-install-check
+ci-checks: compile .generated-files-check .pylint .flake8 check-requirements .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .st2client-install-check
 
 .PHONY: ci-py3-unit
 ci-py3-unit:

conf/st2.conf.sample

@@ -175,14 +175,26 @@ mask_secrets_blacklist =  # comma separated list allowed here.
 mask_secrets = True
 
 [messaging]
-# URL of the messaging server.
-url = amqp://guest:guest@127.0.0.1:5672//
-# How long should we wait between connection retries.
-connection_retry_wait = 10000
+# Certificate file used to identify the local connection (client).
+ssl_certfile = None
 # How many times should we retry connection before failing.
 connection_retries = 10
+# Use SSL / TLS to connect to the messaging server. Same as appending "?ssl=true" at the end of the connection URL string.
+ssl = False
+# URL of the messaging server.
+url = amqp://guest:guest@127.0.0.1:5672//
+# Specifies whether a certificate is required from the other side of the connection, and whether it will be validated if provided.
+ssl_cert_reqs = None
 # URL of all the nodes in a messaging service cluster.
 cluster_urls =  # comma separated list allowed here.
+# How long should we wait between connection retries.
+connection_retry_wait = 10000
+# Private keyfile used to identify the local connection against RabbitMQ.
+ssl_keyfile = None
+# ca_certs file contains a set of concatenated CA certificates, which are used to validate certificates passed from RabbitMQ.
+ssl_ca_certs = None
+# Login method to use (AMQPLAIN, PLAIN, EXTERNAL, etc.).
+login_method = None
 
 [metrics]
 # Randomly sample and only send metrics for X% of metric operations to the backend. Default value of 1 means no sampling is done and all the metrics are sent to the backend. E.g. 0.1 would mean 10% of operations are sampled.
@@ -249,14 +261,18 @@ thread_pool_size = 10
 logging = /etc/st2/logging.rulesengine.conf
 
 [scheduler]
-# How long (in seconds) to sleep between each action scheduler main loop run interval.
-sleep_interval = 0.1
-# How often (in seconds) to look for zombie execution requests before rescheduling them.
-gc_interval = 10
+# The maximum number of attempts that the scheduler retries on error.
+retry_max_attempt = 10
 # Location of the logging configuration file.
 logging = /etc/st2/logging.scheduler.conf
+# How long (in seconds) to sleep between each action scheduler main loop run interval.
+sleep_interval = 0.1
 # The size of the pool used by the scheduler for scheduling executions.
 pool_size = 10
+# The number of milliseconds to wait in between retries.
+retry_wait_msec = 3000
+# How often (in seconds) to look for zombie execution requests before rescheduling them.
+gc_interval = 10
 
 [schema]
 # Version of JSON schema to use.

conf/st2.dev.conf

@@ -91,7 +91,14 @@ ssh_key_file = /home/vagrant/.ssh/stanley_rsa
 
 [messaging]
 url = amqp://guest:guest@127.0.0.1:5672/
-#url = redis://localhost:6379/0
+# Uncomment to test SSL options
+#url = amqp://guest:guest@127.0.0.1:5671/
+#ssl = True
+#ssl_keyfile = /data/stanley/st2tests/st2tests/fixtures/ssl_certs/client/private_key.pem
+#ssl_certfile = /data/stanley/st2tests/st2tests/fixtures/ssl_certs/client/client_certificate.pem
+#ssl_ca_certs = /data/stanley/st2tests/st2tests/fixtures/ssl_certs/ca/ca_certificate_bundle.pem
+#ssl_cert_reqs = required
+#ssl_cert_reqs = required
 
 [ssh_runner]
 remote_dir = /tmp

contrib/core/actions/send_mail/send_mail

@@ -3,14 +3,27 @@
 HOSTNAME=$(hostname -f)
 LINE_BREAK=""
 
-SENDMAIL=`which sendmail`
-if [ $? -ne 0 ]; then
-    echo "Unable to find sendmail binary in PATH" >&2
-    exit 2
+FOOTER="This message was generated by StackStorm action `basename $0` running on `hostname`"
+
+# Allow user to provide a custom sendmail binary for more flexibility and easier
+# testing
+SENDMAIL_BINARY=$1
+
+if [ "${SENDMAIL_BINARY}" = "None" ]; then
+    # If path to the sendmail binary is not provided, try to find one in $PATH
+    SENDMAIL=`which sendmail`
+
+    if [ $? -ne 0 ]; then
+        echo "Unable to find sendmail binary in PATH" >&2
+        exit 2
+    fi
+
+    MAIL="$SENDMAIL -t"
+else
+    MAIL="${SENDMAIL_BINARY}"
 fi
+shift
 
-MAIL="$SENDMAIL -t"
-FOOTER="This message was generated by StackStorm action `basename $0` running on `hostname`"
 if [[ $1 =~ '@' ]]; then
   FROM=$1
 else
@@ -52,7 +65,7 @@ if [[ -z $trimmed && $SEND_EMPTY_BODY -eq 1 ]] || [[ -n $trimmed ]]; then
 cat <<EOF
 TO: ${TO}
 FROM: ${FROM}
-SUBJECT: ${SUBJECT}
+SUBJECT: =?UTF-8?B?$(echo ${SUBJECT} | base64)?=
 EOF
 
 if [[ -n $ATTACHMENTS ]]; then
@@ -83,7 +96,7 @@ EOF
 
   cat <<EOF
 --${boundary}
-Content-Type: ${CONTENT_TYPE}
+Content-Type: ${CONTENT_TYPE}; charset=UTF-8
 Content-Transfer-Encoding: 7bit
 Content-Disposition: inline
 
@@ -92,7 +105,7 @@ EOF
 else
 
   cat <<EOF
-Content-Type: ${CONTENT_TYPE}
+Content-Type: ${CONTENT_TYPE}; charset=UTF-8
 
 EOF