diff --git a/.travis.yml b/.travis.yml index 78d703392a..42b794e571 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,7 +56,7 @@ matrix: name: "Lint Checks, Packs Tests (Python 2.7)" - env: TASK="compilepy3 ci-py3-unit" CACHE_NAME=py3 COMMAND_THRESHOLD=680 python: 3.6 - name: "Unit Tests (Python 3.6)" + name: "Unit Tests, Pack Tests (Python 3.6)" - env: TASK="ci-py3-integration" CACHE_NAME=py3 COMMAND_THRESHOLD=310 python: 3.6 name: "Integration Tests (Python 3.6)" @@ -100,7 +100,7 @@ before_install: install: - ./scripts/travis/install-requirements.sh - - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi + - if [ "${TASK}" = 'ci-unit' ] || [ "${TASK}" = 'ci-integration' ] || [ "${TASK}" = 'ci-checks ci-packs-tests' ] || [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then sudo .circle/add-itest-user.sh; fi # Let's enable rabbitmqadmin # See https://github.com/messagebus/lapine/wiki/Testing-on-Travis. diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9d2aee0cd8..a50ac012d5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -35,6 +35,17 @@ Fixed values. Reported by @dswebbthg, @nickbaum. (bug fix) #4513 #4527 #4528 +* Fix a bug with action positional parameter serialization used in local and remote script runner + not working correctly with non-ascii (unicode) values. + + This would prevent actions such as ``core.sendmail`` which utilize positional parameters from + working correctly when a unicode value was provided. + + Reported by @johandahlberg (bug fix) #4533 +* Fix ``core.sendmail`` action so it specifies ``charset=UTF-8`` in the ``Content-Type`` email + header. This way it works correctly when an email subject and / or body contains unicode data. + + Reported by @johandahlberg (bug fix) #4533 4534 2.10.0 - December 13, 2018 -------------------------- diff --git a/contrib/core/actions/send_mail/send_mail b/contrib/core/actions/send_mail/send_mail index 1d9bdbdc14..6011c43d49 100755 --- a/contrib/core/actions/send_mail/send_mail +++ b/contrib/core/actions/send_mail/send_mail @@ -3,14 +3,27 @@ HOSTNAME=$(hostname -f) LINE_BREAK="" -SENDMAIL=`which sendmail` -if [ $? -ne 0 ]; then - echo "Unable to find sendmail binary in PATH" >&2 - exit 2 +FOOTER="This message was generated by StackStorm action `basename $0` running on `hostname`" + +# Allow user to provide a custom sendmail binary for more flexibility and easier +# testing +SENDMAIL_BINARY=$1 + +if [ "${SENDMAIL_BINARY}" = "None" ]; then + # If path to the sendmail binary is not provided, try to find one in $PATH + SENDMAIL=`which sendmail` + + if [ $? -ne 0 ]; then + echo "Unable to find sendmail binary in PATH" >&2 + exit 2 + fi + + MAIL="$SENDMAIL -t" +else + MAIL="${SENDMAIL_BINARY}" fi +shift -MAIL="$SENDMAIL -t" -FOOTER="This message was generated by StackStorm action `basename $0` running on `hostname`" if [[ $1 =~ '@' ]]; then FROM=$1 else @@ -52,7 +65,7 @@ if [[ -z $trimmed && $SEND_EMPTY_BODY -eq 1 ]] || [[ -n $trimmed ]]; then cat <=3.9.1,<4.0 diff --git a/contrib/core/tests/test_action_sendmail.py b/contrib/core/tests/test_action_sendmail.py new file mode 100644 index 0000000000..4d003aa9be --- /dev/null +++ b/contrib/core/tests/test_action_sendmail.py @@ -0,0 +1,277 @@ +# -*- coding: utf-8 -*- +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid +import base64 +import tempfile +import socket + +import six +import mock +import mailparser + +from st2common.constants import action as action_constants + +from st2tests.fixturesloader import FixturesLoader +from st2tests.base import RunnerTestCase +from st2tests.base import CleanDbTestCase +from st2tests.base import CleanFilesTestCase + +from local_runner.local_shell_script_runner import LocalShellScriptRunner + +__all__ = [ + 'SendmailActionTestCase' +] + +MOCK_EXECUTION = mock.Mock() +MOCK_EXECUTION.id = '598dbf0c0640fd54bffc688b' +HOSTNAME = socket.gethostname() + + +class SendmailActionTestCase(RunnerTestCase, CleanDbTestCase, CleanFilesTestCase): + """ + NOTE: Those tests rely on stanley user being available on the system and having paswordless + sudo access. + """ + fixtures_loader = FixturesLoader() + + def test_sendmail_default_text_html_content_type(self): + action_parameters = { + 'sendmail_binary': 'cat', + + 'from': 'from.user@example.tld1', + 'to': 'to.user@example.tld2', + 'subject': 'this is subject 1', + 'send_empty_body': False, + 'content_type': 'text/html', + 'body': 'Hello there html.', + 'attachments': '' + } + + expected_body = ('Hello there html.\n' + '

\n' + 'This message was generated by StackStorm action ' + 'send_mail running on %s' % (HOSTNAME)) + + status, _, email_data, message = self._run_action(action_parameters=action_parameters) + self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED) + + # Verify subject contains utf-8 charset and is base64 encoded + self.assertTrue('SUBJECT: =?UTF-8?B?' in email_data) + + self.assertEqual(message.to[0][1], action_parameters['to']) + self.assertEqual(message.from_[0][1], action_parameters['from']) + self.assertEqual(message.subject, action_parameters['subject']) + self.assertEqual(message.body, expected_body) + self.assertEqual(message.content_type, 'text/html; charset=UTF-8') + + def test_sendmail_text_plain_content_type(self): + action_parameters = { + 'sendmail_binary': 'cat', + + 'from': 'from.user@example.tld1', + 'to': 'to.user@example.tld2', + 'subject': 'this is subject 2', + 'send_empty_body': False, + 'content_type': 'text/plain', + 'body': 'Hello there plain.', + 'attachments': '' + } + + expected_body = ('Hello there plain.\n\n' + 'This message was generated by StackStorm action ' + 'send_mail running on %s' % (HOSTNAME)) + + status, _, email_data, message = self._run_action(action_parameters=action_parameters) + self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED) + + # Verify subject contains utf-8 charset and is base64 encoded + self.assertTrue('SUBJECT: =?UTF-8?B?' in email_data) + + self.assertEqual(message.to[0][1], action_parameters['to']) + self.assertEqual(message.from_[0][1], action_parameters['from']) + self.assertEqual(message.subject, action_parameters['subject']) + self.assertEqual(message.body, expected_body) + self.assertEqual(message.content_type, 'text/plain; charset=UTF-8') + + def test_sendmail_utf8_subject_and_body(self): + # 1. tex/html + action_parameters = { + 'sendmail_binary': 'cat', + + 'from': 'from.user@example.tld1', + 'to': 'to.user@example.tld2', + 'subject': u'Γ… unicode subject πŸ˜ƒπŸ˜ƒ', + 'send_empty_body': False, + 'content_type': 'text/html', + 'body': u'Hello there πŸ˜ƒπŸ˜ƒ.', + 'attachments': '' + } + + if six.PY2: + expected_body = (u'Hello there πŸ˜ƒπŸ˜ƒ.\n' + u'

\n' + u'This message was generated by StackStorm action ' + u'send_mail running on %s' % (HOSTNAME)) + else: + expected_body = (u'Hello there \\U0001f603\\U0001f603.\n' + u'

\n' + u'This message was generated by StackStorm action ' + u'send_mail running on %s' % (HOSTNAME)) + + status, _, email_data, message = self._run_action(action_parameters=action_parameters) + self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED) + + # Verify subject contains utf-8 charset and is base64 encoded + self.assertTrue('SUBJECT: =?UTF-8?B?' in email_data) + + self.assertEqual(message.to[0][1], action_parameters['to']) + self.assertEqual(message.from_[0][1], action_parameters['from']) + self.assertEqual(message.subject, action_parameters['subject']) + self.assertEqual(message.body, expected_body) + self.assertEqual(message.content_type, 'text/html; charset=UTF-8') + + # 2. text/plain + action_parameters = { + 'sendmail_binary': 'cat', + + 'from': 'from.user@example.tld1', + 'to': 'to.user@example.tld2', + 'subject': u'Γ… unicode subject πŸ˜ƒπŸ˜ƒ', + 'send_empty_body': False, + 'content_type': 'text/plain', + 'body': u'Hello there πŸ˜ƒπŸ˜ƒ.', + 'attachments': '' + } + + if six.PY2: + expected_body = (u'Hello there πŸ˜ƒπŸ˜ƒ.\n\n' + u'This message was generated by StackStorm action ' + u'send_mail running on %s' % (HOSTNAME)) + else: + expected_body = (u'Hello there \\U0001f603\\U0001f603.\n\n' + u'This message was generated by StackStorm action ' + u'send_mail running on %s' % (HOSTNAME)) + + status, _, email_data, message = self._run_action(action_parameters=action_parameters) + self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED) + + self.assertEqual(message.to[0][1], action_parameters['to']) + self.assertEqual(message.from_[0][1], action_parameters['from']) + self.assertEqual(message.subject, action_parameters['subject']) + self.assertEqual(message.body, expected_body) + self.assertEqual(message.content_type, 'text/plain; charset=UTF-8') + + def test_sendmail_with_attachments(self): + _, path_1 = tempfile.mkstemp() + _, path_2 = tempfile.mkstemp() + os.chmod(path_1, 0o755) + os.chmod(path_2, 0o755) + + self.to_delete_files.append(path_1) + self.to_delete_files.append(path_2) + + with open(path_1, 'w') as fp: + fp.write('content 1') + + with open(path_2, 'w') as fp: + fp.write('content 2') + + action_parameters = { + 'sendmail_binary': 'cat', + + 'from': 'from.user@example.tld1', + 'to': 'to.user@example.tld2', + 'subject': 'this is email with attachments', + 'send_empty_body': False, + 'content_type': 'text/plain', + 'body': 'Hello there plain.', + 'attachments': '%s,%s' % (path_1, path_2) + } + + expected_body = ('Hello there plain.\n\n' + 'This message was generated by StackStorm action ' + 'send_mail running on %s' % (HOSTNAME)) + + status, _, email_data, message = self._run_action(action_parameters=action_parameters) + self.assertEquals(status, action_constants.LIVEACTION_STATUS_SUCCEEDED) + + # Verify subject contains utf-8 charset and is base64 encoded + self.assertTrue('SUBJECT: =?UTF-8?B?' in email_data) + + self.assertEqual(message.to[0][1], action_parameters['to']) + self.assertEqual(message.from_[0][1], action_parameters['from']) + self.assertEqual(message.subject, action_parameters['subject']) + self.assertEqual(message.body, expected_body) + self.assertEqual(message.content_type, + 'multipart/mixed; boundary="ZZ_/afg6432dfgkl.94531q"') + + # There should be 3 message parts - 2 for attachments, one for body + self.assertEqual(email_data.count('--ZZ_/afg6432dfgkl.94531q'), 3) + + # There should be 2 attachments + self.assertEqual(email_data.count('Content-Transfer-Encoding: base64'), 2) + self.assertTrue(base64.b64encode(b'content 1').decode('utf-8') in email_data) + self.assertTrue(base64.b64encode(b'content 2').decode('utf-8') in email_data) + + def _run_action(self, action_parameters): + """ + Run action with the provided action parameters, return status output and + parse the output email data. + """ + models = self.fixtures_loader.load_models( + fixtures_pack='packs/core', fixtures_dict={'actions': ['sendmail.yaml']}) + action_db = models['actions']['sendmail.yaml'] + entry_point = self.fixtures_loader.get_fixture_file_path_abs( + 'packs/core', 'actions', 'send_mail/send_mail') + + runner = self._get_runner(action_db, entry_point=entry_point) + runner.pre_run() + status, result, _ = runner.run(action_parameters) + runner.post_run(status, result) + + # Remove footer added by the action which is not part of raw email data and parse + # the message + if 'stdout' in result: + email_data = result['stdout'] + email_data = email_data.split('\n')[:-2] + email_data = '\n'.join(email_data) + + if six.PY2 and isinstance(email_data, six.text_type): + email_data = email_data.encode('utf-8') + + message = mailparser.parse_from_string(email_data) + else: + email_data = None + message = None + + return (status, result, email_data, message) + + def _get_runner(self, action_db, entry_point): + runner = LocalShellScriptRunner(uuid.uuid4().hex) + runner.execution = MOCK_EXECUTION + runner.action = action_db + runner.action_name = action_db.name + runner.liveaction_id = uuid.uuid4().hex + runner.entry_point = entry_point + runner.runner_parameters = {} + runner.context = dict() + runner.callback = dict() + runner.libs_dir_path = None + runner.auth_token = mock.Mock() + runner.auth_token.token = 'mock-token' + return runner diff --git a/st2common/st2common/util/action_db.py b/st2common/st2common/util/action_db.py index 7bd2c33a6c..96969492e6 100644 --- a/st2common/st2common/util/action_db.py +++ b/st2common/st2common/util/action_db.py @@ -268,7 +268,16 @@ def serialize_positional_argument(argument_type, argument_value): serialized). """ if argument_type in ['string', 'number', 'float']: - argument_value = str(argument_value) if argument_value else '' + if argument_value is None: + argument_value = six.text_type('') + return argument_value + + if isinstance(argument_value, (int, float)): + argument_value = str(argument_value) + + if not isinstance(argument_value, six.text_type): + # cast string non-unicode values to unicode + argument_value = argument_value.decode('utf-8') elif argument_type == 'boolean': # Booleans are serialized as string "1" and "0" if argument_value is not None: @@ -285,8 +294,8 @@ def serialize_positional_argument(argument_type, argument_value): # None / null is serialized as en empty string argument_value = '' else: - # Other values are simply cast to strings - argument_value = str(argument_value) if argument_value else '' + # Other values are simply cast to unicode string + argument_value = six.text_type(argument_value) if argument_value else '' return argument_value diff --git a/st2common/tests/unit/test_action_db_utils.py b/st2common/tests/unit/test_action_db_utils.py index 3132a2ebd7..061709d61a 100644 --- a/st2common/tests/unit/test_action_db_utils.py +++ b/st2common/tests/unit/test_action_db_utils.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Licensed to the StackStorm, Inc ('StackStorm') under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -399,6 +400,27 @@ def test_get_args(self): self.assertListEqual(pos_args, expected_pos_args, 'Positional args not parsed / serialized correctly.') + # Test unicode values + params = { + 'actionstr': 'bar č Ε‘ hello Δ‘ č p ΕΎ Ε½ a πŸ’©πŸ˜', + 'actionint': 20, + 'runnerint': 555 + } + expected_pos_args = [ + '20', + '', + u'bar č Ε‘ hello Δ‘ č p ΕΎ Ε½ a πŸ’©πŸ˜', + '', + '', + '', + '' + ] + pos_args, named_args = action_db_utils.get_args(params, ActionDBUtilsTestCase.action_db) + self.assertListEqual(pos_args, expected_pos_args, 'Positional args not parsed correctly.') + self.assertTrue('actionint' not in named_args) + self.assertTrue('actionstr' not in named_args) + self.assertEqual(named_args.get('runnerint'), 555) + @classmethod def _setup_test_models(cls): ActionDBUtilsTestCase.setup_runner()