fixing enterprise ldap issue with execution view

.travis.yml

@@ -135,7 +135,7 @@ script:
   # as long as PR builds
   - if [ "${TRAVIS_PULL_REQUEST}" = "false" ] && [ "${IS_NIGHTLY_BUILD}" = "no" ]; then COMMAND_THRESHOLD=$(expr ${COMMAND_THRESHOLD} \* 2); fi; ./scripts/travis/time-command.sh "make ${TASK}" ${COMMAND_THRESHOLD}
   # Run any additional nightly checks only as part of a nightly (cron) build
-  - if [ "${IS_NIGHTLY_BUILD}" = "yes" ] && [ "${TASK}" = "ci-checks ci-packs-tests" ]; then make ci-checks-nightly; fi
+  - if [ "${IS_NIGHTLY_BUILD}" = "yes" ]; then ./scripts/travis/run-nightly-make-task-if-exists.sh "${TASK}"; fi
   # NOTE: We only generate and submit coverage report for master and version branches
   # NOTE: We put this here and not after_success so build is marked as failed if this step fails
   # See https://docs.travis-ci.com/user/customizing-the-build/#breaking-the-build
@@ -147,3 +147,13 @@ script:
 # Alternative: use strict pip pinning, including git-based pip packages
 before_cache:
   - if [ ${TRAVIS_PULL_REQUEST} = 'false' ] && [ "${IS_NIGHTLY_BUILD}" = "no" ]; then rm -rf virtualenv/; fi
+
+# We want to be notified when a master or nightly build fails
+notifications:
+  # Post build failures to '#stackstorm' channel in 'stackstorm' Slack
+  slack:
+    rooms:
+      - secure: "rPA22aDgvNe0/S/2e+cp1rSDdDUPufLXnCbfnRzMPVDSQ2UPdLmEl9IeOoEHZmq92AZtzY8UnQaPFuoM0HAPrYDgKopn4n4KpOo+xUlJ92qdNj5qk3Z1TmQHwUYFvCkMvaR/CpX2liRr/YB3qM+1vFAMsYgmqrBX8vcEqNJQy/M="
+    on_pull_requests: false
+    on_success: change # default: always
+    on_failure: always # default: always

CHANGELOG.rst

@@ -4,6 +4,13 @@ Changelog
 in development
 --------------
 
+Added
+~~~~~
+
+* Add support for blacklisting / whitelisting hosts to the HTTP runner by adding new
+  ``url_hosts_blacklist`` and ``url_hosts_whitelist`` runner attribute. (new feature)
+  #4757
+
 Changed
 ~~~~~~~
 
@@ -13,7 +20,10 @@ Changed
 
 Fixed
 ~~~~~
-
+* Fix rbac with execution view where the rbac is unable to verify the pack or uid of the execution
+  because it was not returned from the action execution db. This would result in an internal server
+  error when trying to view the results of a single execution.
+  Contributed by Joshua Meyer (@jdmeyer3) #4758
 * Fixed logging middleware to output a ``content_length`` of ``0`` instead of ``Infinity``
   when the type of data being returned is not supported. Previously, when the value was
   set to ``Infinity`` this would result in invalid JSON being output into structured

Makefile

@@ -25,6 +25,7 @@ COMPONENTS_RUNNERS := $(wildcard contrib/runners/*)
 COMPONENTS_WITHOUT_ST2TESTS := $(shell ls -a | grep ^st2 | grep -v .egg-info | grep -v st2tests | grep -v st2exporter)
 
 COMPONENTS_WITH_RUNNERS := $(COMPONENTS) $(COMPONENTS_RUNNERS)
+COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER := $(foreach component,$(filter-out contrib/runners/mistral_v2,$(COMPONENTS_WITH_RUNNERS)),$(component))
 
 COMPONENTS_TEST_DIRS := $(wildcard st2*/tests) $(wildcard contrib/runners/*/tests)
 
@@ -42,6 +43,7 @@ space_char :=
 space_char +=
 COMPONENT_PYTHONPATH = $(subst $(space_char),:,$(realpath $(COMPONENTS_WITH_RUNNERS)))
 COMPONENTS_TEST := $(foreach component,$(filter-out $(COMPONENT_SPECIFIC_TESTS),$(COMPONENTS_WITH_RUNNERS)),$(component))
+COMPONENTS_TEST_WITHOUT_MISTRAL_RUNNER := $(foreach component,$(filter-out $(COMPONENT_SPECIFIC_TESTS),$(COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER)),$(component))
 COMPONENTS_TEST_COMMA := $(subst $(slash),$(dot),$(subst $(space_char),$(comma),$(COMPONENTS_TEST)))
 COMPONENTS_TEST_MODULES := $(subst $(slash),$(dot),$(COMPONENTS_TEST_DIRS))
 COMPONENTS_TEST_MODULES_COMMA := $(subst $(space_char),$(comma),$(COMPONENTS_TEST_MODULES))
@@ -109,6 +111,8 @@ play:
 	@echo
 	@echo COMPONENTS_WITH_RUNNERS=$(COMPONENTS_WITH_RUNNERS)
 	@echo
+	@echo COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER=$(COMPONENTS_WITH_RUNNERS_WITHOUT_MISTRAL_RUNNER)
+	@echo
 	@echo COMPONENTS_TEST=$(COMPONENTS_TEST)
 	@echo
 	@echo COMPONENTS_TEST_COMMA=$(COMPONENTS_TEST_COMMA)
@@ -119,6 +123,8 @@ play:
 	@echo
 	@echo COMPONENTS_TEST_MODULES_COMMA=$(COMPONENTS_TEST_MODULES_COMMA)
 	@echo
+	@echo COMPONENTS_TEST_WITHOUT_MISTRAL_RUNNER=$(COMPONENTS_TEST_WITHOUT_MISTRAL_RUNNER)
+	@echo
 	@echo COMPONENT_PYTHONPATH=$(COMPONENT_PYTHONPATH)
 	@echo
 	@echo TRAVIS_PULL_REQUEST=$(TRAVIS_PULL_REQUEST)
@@ -573,7 +579,7 @@ endif
 	@echo
 	@echo "----- Dropping st2-test db -----"
 	@mongo st2-test --eval "db.dropDatabase();"
-	for component in $(COMPONENTS_TEST); do\
+	for component in $(COMPONENTS_TEST_WITHOUT_MISTRAL_RUNNER); do\
 		echo "==========================================================="; \
 		echo "Running tests in" $$component; \
 		echo "-----------------------------------------------------------"; \
@@ -939,6 +945,13 @@ ci-py3-unit:
 	NOSE_WITH_TIMER=$(NOSE_WITH_TIMER) tox -e py36-unit -vv
 	NOSE_WITH_TIMER=$(NOSE_WITH_TIMER) tox -e py36-packs -vv
 
+.PHONY: ci-py3-unit-nightly
+ci-py3-unit-nightly:
+	@echo
+	@echo "==================== ci-py3-unit ===================="
+	@echo
+	NOSE_WITH_TIMER=$(NOSE_WITH_TIMER) tox -e py36-unit-nightly -vv
+
 .PHONY: ci-py3-integration
 ci-py3-integration: requirements .ci-prepare-integration .ci-py3-integration
 
@@ -977,6 +990,15 @@ ci-py3-integration: requirements .ci-prepare-integration .ci-py3-integration
 .PHONY: ci-unit
 ci-unit: .unit-tests-coverage-html
 
+.PHONY: ci-unit-nightly
+ci-unit-nightly:
+	# NOTE: We run mistral runner checks only as part of a nightly build to speed up
+	# non nightly builds (Mistral will be deprecated in the future)
+	@echo
+	@echo "============== ci-unit-nightly =============="
+	@echo
+	nosetests $(NOSE_OPTS) -s -v  contrib/runners/mistral_v2/tests/unit
+
 .PHONY: .ci-prepare-integration
 .ci-prepare-integration:
 	sudo -E ./scripts/travis/prepare-integration.sh

contrib/runners/http_runner/http_runner/http_runner.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from __future__ import absolute_import
+
 import ast
 import copy
 import json
@@ -21,6 +22,7 @@
 import requests
 from requests.auth import HTTPBasicAuth
 from oslo_config import cfg
+from six.moves.urllib import parse as urlparse  # pylint: disable=import-error
 
 from st2common.runners.base import ActionRunner
 from st2common.runners.base import get_metadata as get_runner_metadata
@@ -55,6 +57,8 @@
 RUNNER_VERIFY_SSL_CERT = 'verify_ssl_cert'
 RUNNER_USERNAME = 'username'
 RUNNER_PASSWORD = 'password'
+RUNNER_URL_HOSTS_BLACKLIST = 'url_hosts_blacklist'
+RUNNER_URL_HOSTS_WHITELIST = 'url_hosts_whitelist'
 
 # Lookup constants for action params
 ACTION_AUTH = 'auth'
@@ -93,10 +97,17 @@ def pre_run(self):
         self._http_proxy = self.runner_parameters.get(RUNNER_HTTP_PROXY, None)
         self._https_proxy = self.runner_parameters.get(RUNNER_HTTPS_PROXY, None)
         self._verify_ssl_cert = self.runner_parameters.get(RUNNER_VERIFY_SSL_CERT, None)
+        self._url_hosts_blacklist = self.runner_parameters.get(RUNNER_URL_HOSTS_BLACKLIST, [])
+        self._url_hosts_whitelist = self.runner_parameters.get(RUNNER_URL_HOSTS_WHITELIST, [])
 
     def run(self, action_parameters):
         client = self._get_http_client(action_parameters)
 
+        if self._url_hosts_blacklist and self._url_hosts_whitelist:
+            msg = ('"url_hosts_blacklist" and "url_hosts_whitelist" parameters are mutually '
+                   'exclusive. Only one should be provided.')
+            raise ValueError(msg)
+
         try:
             result = client.run()
         except requests.exceptions.Timeout as e:
@@ -147,7 +158,9 @@ def _get_http_client(self, action_parameters):
                           headers=headers, cookies=self._cookies, auth=auth,
                           timeout=timeout, allow_redirects=self._allow_redirects,
                           proxies=proxies, files=files, verify=self._verify_ssl_cert,
-                          username=self._username, password=self._password)
+                          username=self._username, password=self._password,
+                          url_hosts_blacklist=self._url_hosts_blacklist,
+                          url_hosts_whitelist=self._url_hosts_whitelist)
 
     @staticmethod
     def _get_result_status(status_code):
@@ -158,7 +171,8 @@ def _get_result_status(status_code):
 class HTTPClient(object):
     def __init__(self, url=None, method=None, body='', params=None, headers=None, cookies=None,
                  auth=None, timeout=60, allow_redirects=False, proxies=None,
-                 files=None, verify=False, username=None, password=None):
+                 files=None, verify=False, username=None, password=None,
+                 url_hosts_blacklist=None, url_hosts_whitelist=None):
         if url is None:
             raise Exception('URL must be specified.')
 
@@ -188,12 +202,30 @@ def __init__(self, url=None, method=None, body='', params=None, headers=None, co
         self.verify = verify
         self.username = username
         self.password = password
+        self.url_hosts_blacklist = url_hosts_blacklist or []
+        self.url_hosts_whitelist = url_hosts_whitelist or []
+
+        if self.url_hosts_blacklist and self.url_hosts_whitelist:
+            msg = ('"url_hosts_blacklist" and "url_hosts_whitelist" parameters are mutually '
+                   'exclusive. Only one should be provided.')
+            raise ValueError(msg)
 
     def run(self):
         results = {}
         resp = None
         json_content = self._is_json_content()
 
+        # Check if the provided URL is blacklisted
+        is_url_blacklisted = self._is_url_blacklisted(url=self.url)
+
+        if is_url_blacklisted:
+            raise ValueError('URL "%s" is blacklisted' % (self.url))
+
+        is_url_whitelisted = self._is_url_whitelisted(url=self.url)
+
+        if not is_url_whitelisted:
+            raise ValueError('URL "%s" is not whitelisted' % (self.url))
+
         try:
             if json_content:
                 # cast params (body) to dict
@@ -301,6 +333,49 @@ def _cast_object(self, value):
         else:
             return value
 
+    def _is_url_blacklisted(self, url):
+        """
+        Verify if the provided URL is blacklisted via url_hosts_blacklist runner parameter.
+        """
+        if not self.url_hosts_blacklist:
+            # Blacklist is empty
+            return False
+
+        host = self._get_host_from_url(url=url)
+
+        if host in self.url_hosts_blacklist:
+            return True
+
+        return False
+
+    def _is_url_whitelisted(self, url):
+        """
+        Verify if the provided URL is whitelisted via url_hosts_whitelist runner parameter.
+        """
+        if not self.url_hosts_whitelist:
+            return True
+
+        host = self._get_host_from_url(url=url)
+
+        if host in self.url_hosts_whitelist:
+            return True
+
+        return False
+
+    def _get_host_from_url(self, url):
+        """
+        Return sanitized host (netloc) value from the provided url.
+        """
+        parsed = urlparse.urlparse(url)
+
+        # Remove port and []
+        host = parsed.netloc.replace('[', '').replace(']', '')
+
+        if parsed.port is not None:
+            host = host.replace(':%s' % (parsed.port), '')
+
+        return host
+
 
 def get_runner():
     return HttpRunner(str(uuid.uuid4()))

contrib/runners/http_runner/http_runner/runner.yaml

@@ -36,6 +36,22 @@
         CA bundle which comes from Mozilla. Verification using a custom CA bundle
         is not yet supported. Set to False to skip verification.
       type: boolean
+    url_hosts_blacklist:
+      description: Optional list of hosts (network locations) to blacklist (e.g. example.com,
+        127.0.0.1, ::1, etc.). If action will try to access that endpoint, an exception will be
+        thrown and action will be marked as failed.
+      required: false
+      type: array
+      items:
+        type: string
+    url_hosts_whitelist:
+      description: Optional list of hosts (network locations) to whitelist (e.g. example.com,
+        127.0.0.1, ::1, etc.). If specified, actions will only be able to hit hosts on this
+        whitelist.
+      required: false
+      type: array
+      items:
+        type: string
   output_key: body
   output_schema:
     status_code: