Fix st2 execution get not printing action ref

CHANGELOG.rst

@@ -12,7 +12,10 @@ Added
   #4757
 * Add ``user`` parameter to ``re_run`` method of st2client. #4785
 * Install pack dependencies automatically. #4769
-* Add support for `immutable_parameters` on Action Aliases. This feature allows default parameters to be supplied to the action on every execution of the alias. #4786
+* Add support for `immutable_parameters` on Action Aliases. This feature allows default
+  parameters to be supplied to the action on every execution of the alias. #4786
+* Add ``get_entrypoint()`` method to ``ActionResourceManager`` attribute of st2client.
+  #4791
 
 Changed
 ~~~~~~~
@@ -27,6 +30,15 @@ Changed
   writing very large executions (executions with large results) to the database. #4767
 * Improved development instructions in requirements.txt and dist_utils.py comment headers
   (improvement) #4774
+* Add new ``actionrunner.stream_output_buffer_size`` config option and default it to ``-1``
+  (previously default value was ``0``). This should result in a better performance and smaller
+  CPU utilization for Python runner actions which produce a lot of output.
+  (improvement)
+
+  Reported and contributed by Joshua Meyer (@jdmeyer3) #4803
+* Add new ``action_runner.pip_opts`` st2.conf config option which allows user to specify a list
+  of command line option which are passed to ``pip install`` command when installing pack
+  dependencies into a pack specific virtual environment. #4792
 
 Fixed
 ~~~~~
@@ -49,6 +61,21 @@ Fixed
   doesn't depend on internal pip API and so it works with latest pip version. (bug fix) #4750
 * Fix dependency conflicts in pack CI runs: downgrade requests dependency back to 0.21.0, update
   internal dependencies and test expectations (amqp, pyyaml, prance, six) (bugfix) #4774
+* Fix secrets masking in action parameters section defined inside the rule when using
+  ``GET /v1/rules`` and ``GET /v1/rules/<ref>`` API endpoint. (bug fix) #4788 #4807
+
+  Contributed by @Nicodemos305 and @jeansfelix
+* Fix a bug with authentication API endpoint (``POST /auth/v1/tokens``) returning internal
+  server error when running under gunicorn and when``auth.api_url`` config option was not set.
+  (bug fix) #4809
+
+  Reported by @guzzijones
+* Fixed ``st2 execution get`` and ``st2 run`` not printing the ``action.ref`` for non-workflow
+  actions. (bug fix) #4739
+
+  Contributed by Nick Maludy (@nmaludy Encore Technologies)
+* Update ``st2 execution get`` command to always include ``context.user``, ``start_timestamp`` and
+  ``end_timestamp`` attributes. (improvement) #4739
 
 3.1.0 - June 27, 2019
 ---------------------

conf/st2.conf.sample

@@ -8,6 +8,8 @@ enable = True
 emit_when = succeeded,failed,timeout,canceled,abandoned # comma separated list allowed here.
 
 [actionrunner]
+# List of pip options to be passed to "pip install" command when installing pack dependencies into pack virtual environment.
+pip_opts =  # comma separated list allowed here.
 # Internal pool size for dispatcher used by regular actions.
 actions_pool_size = 60
 # Default log level to use for Python runner actions. Can be overriden on invocation basis using "log_level" runner parameter.
@@ -20,6 +22,8 @@ python3_prefix = None
 virtualenv_binary = /usr/bin/virtualenv
 # Python 3 binary which will be used by Python actions for packs which use Python 3 virtual environment.
 python3_binary = /usr/bin/python3
+# Buffer size to use for real time action output streaming. 0 means unbuffered 1 means line buffered, -1 means system default, which usually means fully buffered and any other positive value means use a buffer of (approximately) that size
+stream_output_buffer_size = -1
 # List of virtualenv options to be passsed to "virtualenv" command that creates pack virtualenv.
 virtualenv_opts = --system-site-packages # comma separated list allowed here.
 # True to store and stream action output (stdout and stderr) in real-time.

contrib/runners/python_runner/python_runner/python_runner.py

@@ -248,8 +248,11 @@ def run(self, action_parameters):
         if stdin_params:
             command_string = 'echo %s | %s' % (quote_unix(stdin_params), command_string)
 
-        LOG.debug('Running command: PATH=%s PYTHONPATH=%s %s' % (env['PATH'], env['PYTHONPATH'],
-                                                                 command_string))
+        bufsize = cfg.CONF.actionrunner.stream_output_buffer_size
+
+        LOG.debug('Running command (bufsize=%s): PATH=%s PYTHONPATH=%s %s' % (bufsize, env['PATH'],
+                                                                              env['PYTHONPATH'],
+                                                                              command_string))
         exit_code, stdout, stderr, timed_out = run_command(cmd=args,
                                                            stdin=stdin,
                                                            stdout=subprocess.PIPE,
@@ -261,7 +264,8 @@ def run(self, action_parameters):
                                                            read_stderr_func=read_and_store_stderr,
                                                            read_stdout_buffer=stdout,
                                                            read_stderr_buffer=stderr,
-                                                           stdin_value=stdin_params)
+                                                           stdin_value=stdin_params,
+                                                           bufsize=bufsize)
         LOG.debug('Returning values: %s, %s, %s, %s', exit_code, stdout, stderr, timed_out)
         LOG.debug('Returning.')
         return self._get_output_values(exit_code, stdout, stderr, timed_out)

contrib/runners/python_runner/tests/unit/test_pythonrunner.py

@@ -63,6 +63,8 @@
 
 PRINT_CONFIG_ITEM_ACTION = os.path.join(tests_base.get_resources_path(), 'packs',
                                         'pythonactions/actions/print_config_item_doesnt_exist.py')
+PRINT_TO_STDOUT_STDERR_ACTION = os.path.join(tests_base.get_resources_path(), 'packs',
+                                      'pythonactions/actions/print_to_stdout_and_stderr.py')
 
 
 # Note: runner inherits parent args which doesn't work with tests since test pass additional
@@ -406,6 +408,31 @@ def test_action_stdout_and_stderr_is_stored_in_the_db(self, mock_spawn, mock_pop
         self.assertEqual(output_dbs[1].data, mock_stderr[1])
         self.assertEqual(output_dbs[2].data, mock_stderr[2])
 
+    def test_real_time_output_streaming_bufsize(self):
+        # Test various values for bufsize and verify it works / doesn't hang the process
+        cfg.CONF.set_override(name='stream_output', group='actionrunner', override=True)
+
+        bufsize_values = [-100, -2, -1, 0, 1, 2, 1024, 2048, 4096, 10000]
+
+        for index, bufsize in enumerate(bufsize_values, 1):
+            cfg.CONF.set_override(name='stream_output_buffer_size', override=bufsize,
+                                  group='actionrunner')
+
+            output_dbs = ActionExecutionOutput.get_all()
+            self.assertEqual(len(output_dbs), (index - 1) * 4)
+
+            runner = self._get_mock_runner_obj()
+            runner.entry_point = PRINT_TO_STDOUT_STDERR_ACTION
+            runner.pre_run()
+            (_, output, _) = runner.run({'stdout_count': 2, 'stderr_count': 2})
+
+            self.assertEqual(output['stdout'], 'stdout line 0\nstdout line 1\n')
+            self.assertEqual(output['stderr'], 'stderr line 0\nstderr line 1\n')
+            self.assertEqual(output['exit_code'], 0)
+
+            output_dbs = ActionExecutionOutput.get_all()
+            self.assertEqual(len(output_dbs), (index) * 4)
+
     @mock.patch('st2common.util.concurrency.subprocess_popen')
     def test_stdout_interception_and_parsing(self, mock_popen):
         values = {'delimiter': ACTION_OUTPUT_RESULT_DELIMITER}

scripts/travis/prepare-integration.sh

@@ -15,9 +15,21 @@ source ./virtualenv/bin/activate
 python ./st2client/setup.py develop
 st2 --version
 
+# Clean up old screen log files
+rm -f logs/screen-*.log
+
 # start dev environment in screens
 ./tools/launchdev.sh start -x
 
+# Give processes some time to start and check logs to see if all the services
+# started or if there was any error / failure
+echo "Giving screen processes some time to start..."
+sleep 10
+
+echo " === START: Catting screen process log files. ==="
+cat logs/screen-*.log
+echo " === END: Catting screen process log files. ==="
+
 # This script runs as root on Travis which means other processes which don't run
 # as root can't write to logs/ directory and tests fail
 chmod 777 logs/

st2api/st2api/controllers/v1/rule_views.py

@@ -99,8 +99,9 @@ def get_all(self, exclude_attributes=None, include_attributes=None, sort=None, o
         return rules
 
     def get_one(self, ref_or_id, requester_user):
+        from_model_kwargs = {'mask_secrets': True}
         rule = self._get_one(ref_or_id, permission_type=PermissionType.RULE_VIEW,
-                             requester_user=requester_user)
+                             requester_user=requester_user, from_model_kwargs=from_model_kwargs)
         result = self._append_view_properties([rule.json])[0]
         rule.json = result
         return rule

st2api/st2api/controllers/v1/rules.py

@@ -20,6 +20,7 @@
 from st2common import log as logging
 from st2common.exceptions.apivalidation import ValueValidationException
 from st2common.exceptions.triggers import TriggerDoesNotExistException
+from st2api.controllers.base import BaseRestControllerMixin
 from st2api.controllers.resource import BaseResourceIsolationControllerMixin
 from st2api.controllers.resource import ContentPackResourceController
 from st2api.controllers.controller_transforms import transform_to_bool
@@ -39,7 +40,8 @@
 LOG = logging.getLogger(__name__)
 
 
-class RuleController(BaseResourceIsolationControllerMixin, ContentPackResourceController):
+class RuleController(BaseRestControllerMixin, BaseResourceIsolationControllerMixin,
+                     ContentPackResourceController):
     """
         Implements the RESTful web endpoint that handles
         the lifecycle of Rules in the system.
@@ -68,8 +70,11 @@ class RuleController(BaseResourceIsolationControllerMixin, ContentPackResourceCo
     mandatory_include_fields_retrieve = ['pack', 'name', 'trigger']
 
     def get_all(self, exclude_attributes=None, include_attributes=None, sort=None, offset=0,
-                limit=None, requester_user=None, **raw_filters):
-        from_model_kwargs = {'ignore_missing_trigger': True}
+                limit=None, show_secrets=False, requester_user=None, **raw_filters):
+        from_model_kwargs = {
+            'ignore_missing_trigger': True,
+            'mask_secrets': self._get_mask_secrets(requester_user, show_secrets=show_secrets)
+        }
         return super(RuleController, self)._get_all(exclude_fields=exclude_attributes,
                                                     include_fields=include_attributes,
                                                     from_model_kwargs=from_model_kwargs,
@@ -79,8 +84,11 @@ def get_all(self, exclude_attributes=None, include_attributes=None, sort=None, o
                                                     raw_filters=raw_filters,
                                                     requester_user=requester_user)
 
-    def get_one(self, ref_or_id, requester_user):
-        from_model_kwargs = {'ignore_missing_trigger': True}
+    def get_one(self, ref_or_id, requester_user, show_secrets=False):
+        from_model_kwargs = {
+            'ignore_missing_trigger': True,
+            'mask_secrets': self._get_mask_secrets(requester_user, show_secrets=show_secrets)
+        }
         return super(RuleController, self)._get_one(ref_or_id, from_model_kwargs=from_model_kwargs,
                                                     requester_user=requester_user,
                                                     permission_type=PermissionType.RULE_VIEW)

st2api/tests/unit/controllers/v1/test_action_views.py

@@ -240,7 +240,8 @@ def test_get_one_ref_python_content_type(self):
         try:
             get_resp = self.app.get('/v1/actions/views/entry_point/%s' % action_ref)
             self.assertEqual(get_resp.status_int, 200)
-            self.assertEqual(get_resp.headers['Content-Type'], 'application/x-python')
+            self.assertTrue(get_resp.headers['Content-Type'] in ['application/x-python',
+                                                                 'text/x-python'])
         finally:
             self.app.delete('/v1/actions/%s' % action_id)
 

st2api/tests/unit/controllers/v1/test_rules.py

@@ -20,6 +20,7 @@
 
 from st2common.constants.rules import RULE_TYPE_STANDARD, RULE_TYPE_BACKSTOP
 from st2common.constants.pack import DEFAULT_PACK_NAME
+from st2common.constants.secrets import MASKED_ATTRIBUTE_VALUE
 from st2common.persistence.trigger import Trigger
 from st2common.models.system.common import ResourceReference
 from st2common.transport.publishers import PoolPublisher
@@ -185,6 +186,58 @@ def test_get_all_enabled(self):
         self.__do_delete(self.__get_rule_id(post_resp_rule_1))
         self.__do_delete(self.__get_rule_id(post_resp_rule_3))
 
+    def test_get_all_action_parameters_secrets_masking(self):
+        post_resp_rule_1 = self.__do_post(RulesControllerTestCase.RULE_1)
+
+        # Verify parameter is masked by default
+        resp = self.app.get('/v1/rules')
+        self.assertEqual('action' in resp.json[0], True)
+        self.assertEqual(resp.json[0]['action']['parameters']['action_secret'],
+                         MASKED_ATTRIBUTE_VALUE)
+
+        # Verify ?show_secrets=true works
+        resp = self.app.get('/v1/rules?include_attributes=action&show_secrets=true')
+        self.assertEqual('action' in resp.json[0], True)
+        self.assertEqual(resp.json[0]['action']['parameters']['action_secret'], 'secret')
+
+        self.__do_delete(self.__get_rule_id(post_resp_rule_1))
+
+    def test_get_all_parameters_mask_with_exclude_parameters(self):
+        post_resp_rule_1 = self.__do_post(RulesControllerTestCase.RULE_1)
+        resp = self.app.get('/v1/rules?exclude_attributes=action')
+        self.assertEqual('action' in resp.json[0], False)
+        self.__do_delete(self.__get_rule_id(post_resp_rule_1))
+
+    def test_get_all_parameters_mask_with_include_parameters(self):
+        post_resp_rule_1 = self.__do_post(RulesControllerTestCase.RULE_1)
+
+        # Verify parameter is masked by default
+        resp = self.app.get('/v1/rules?include_attributes=action')
+        self.assertEqual('action' in resp.json[0], True)
+        self.assertEqual(resp.json[0]['action']['parameters']['action_secret'],
+                         MASKED_ATTRIBUTE_VALUE)
+
+        # Verify ?show_secrets=true works
+        resp = self.app.get('/v1/rules?include_attributes=action&show_secrets=true')
+        self.assertEqual('action' in resp.json[0], True)
+        self.assertEqual(resp.json[0]['action']['parameters']['action_secret'], 'secret')
+
+        self.__do_delete(self.__get_rule_id(post_resp_rule_1))
+
+    def test_get_one_action_parameters_secrets_masking(self):
+        post_resp_rule_1 = self.__do_post(RulesControllerTestCase.RULE_1)
+
+        # Verify parameter is masked by default
+        resp = self.app.get('/v1/rules/%s' % (post_resp_rule_1.json['id']))
+        self.assertEqual(resp.json['action']['parameters']['action_secret'],
+                         MASKED_ATTRIBUTE_VALUE)
+
+        # Verify ?show_secrets=true works
+        resp = self.app.get('/v1/rules/%s?show_secrets=true' % (post_resp_rule_1.json['id']))
+        self.assertEqual(resp.json['action']['parameters']['action_secret'], 'secret')
+
+        self.__do_delete(self.__get_rule_id(post_resp_rule_1))
+
     def test_get_one_by_id(self):
         post_resp = self.__do_post(RulesControllerTestCase.RULE_1)
         rule_id = self.__get_rule_id(post_resp)

st2auth/st2auth/controllers/v1/auth.py

@@ -22,6 +22,7 @@
 from st2common.router import exc
 from st2common.router import Response
 from st2common.util import auth as auth_utils
+from st2common.util import api as api_utils
 from st2common import log as logging
 import st2auth.handlers as handlers
 
@@ -80,7 +81,8 @@ def post(self, request, **kwargs):
 
 def process_successful_response(token):
     resp = Response(json=token, status=http_client.CREATED)
-    resp.headers['X-API-URL'] = cfg.CONF.auth.api_url
+    # NOTE: gunicon fails and throws an error if header value is not a string (e.g. if it's None)
+    resp.headers['X-API-URL'] = api_utils.get_base_public_api_url()
     return resp
 
 

st2auth/tests/unit/controllers/v1/test_token.py

@@ -87,6 +87,7 @@ def _test_token_post(self, path=TOKEN_V1_PATH):
         actual_expiry = isotime.parse(response.json['expiry'])
         self.assertLess(timestamp, actual_expiry)
         self.assertLess(actual_expiry, expected_expiry)
+        return response
 
     def test_token_post_unauthorized(self):
         response = self.app.post_json(TOKEN_V1_PATH, {}, expect_errors=True, extra_environ={
@@ -109,6 +110,22 @@ def test_token_post_new_user(self):
     def test_token_post_existing_user(self):
         self._test_token_post()
 
+    @mock.patch.object(
+        User, 'get_by_name',
+        mock.MagicMock(return_value=UserDB(name=USERNAME)))
+    def test_token_post_success_x_api_url_header_value(self):
+        # auth.api_url option is explicitly set
+        cfg.CONF.set_override('api_url', override='https://example.com', group='auth')
+
+        resp = self._test_token_post()
+        self.assertEqual(resp.headers['X-API-URL'], 'https://example.com')
+
+        # auth.api_url option is not set, url is inferred from listen host and port
+        cfg.CONF.set_override('api_url', override=None, group='auth')
+
+        resp = self._test_token_post()
+        self.assertEqual(resp.headers['X-API-URL'], 'http://127.0.0.1:9101')
+
     @mock.patch.object(
         User, 'get_by_name',
         mock.MagicMock(return_value=UserDB(name=USERNAME)))

st2client/st2client/client.py

@@ -25,6 +25,7 @@
 from st2client.models.core import ResourceManager
 from st2client.models.core import ActionAliasResourceManager
 from st2client.models.core import ActionAliasExecutionManager
+from st2client.models.core import ActionResourceManager
 from st2client.models.core import ExecutionResourceManager
 from st2client.models.core import InquiryResourceManager
 from st2client.models.core import TriggerInstanceResourceManager
@@ -118,7 +119,7 @@ def __init__(self, base_url=None, auth_url=None, api_url=None, stream_url=None,
             models.Token, self.endpoints['auth'], cacert=self.cacert, debug=self.debug)
         self.managers['RunnerType'] = ResourceManager(
             models.RunnerType, self.endpoints['api'], cacert=self.cacert, debug=self.debug)
-        self.managers['Action'] = ResourceManager(
+        self.managers['Action'] = ActionResourceManager(
             models.Action, self.endpoints['api'], cacert=self.cacert, debug=self.debug)
         self.managers['ActionAlias'] = ActionAliasResourceManager(
             models.ActionAlias, self.endpoints['api'], cacert=self.cacert, debug=self.debug)