From 1af107e3d71e871cb69a1da505ecd81112993fb3 Mon Sep 17 00:00:00 2001 From: Nick Maludy Date: Tue, 10 Jul 2018 12:30:55 -0400 Subject: [PATCH 01/27] Fix bug where objects and arrays marked as secret weren't being masked --- CHANGELOG.rst | 9 +++ st2common/st2common/util/secrets.py | 18 +++--- st2common/tests/unit/test_util_secrets.py | 70 ++++++++++++++++++++++- 3 files changed, 87 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index af716c63df..f4674dafd4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,15 @@ in development -------------- + +Fixed +~~~~~ + +* Fixed a bug where ``secret: true`` was not applying to full object and array trees. (bugfix) #4234 + Reported by @jjm + + Contributed by Nick Maludy (Encore Technologies). + 2.8.0 - July 10, 2018 --------------------- diff --git a/st2common/st2common/util/secrets.py b/st2common/st2common/util/secrets.py index af12aa92b6..d605cd4e83 100644 --- a/st2common/st2common/util/secrets.py +++ b/st2common/st2common/util/secrets.py @@ -95,7 +95,15 @@ def get_secret_parameters(parameters): continue parameter_type = options.get('type') - if parameter_type in ['object', 'array']: + if options.get('secret', False): + # if this parameter is secret, then add it our secret parameters + if isinstance(secret_parameters, list): + secret_parameters.append(parameter_type) + elif isinstance(secret_parameters, dict): + secret_parameters[parameter] = parameter_type + else: + return parameter_type + elif parameter_type in ['object', 'array']: sub_params = get_secret_parameters(options) if sub_params: if isinstance(secret_parameters, list): @@ -104,14 +112,6 @@ def get_secret_parameters(parameters): secret_parameters[parameter] = sub_params else: return sub_params - elif options.get('secret', False): - # if this parameter is secret, then add it our secret parameters - if isinstance(secret_parameters, list): - secret_parameters.append(parameter_type) - elif isinstance(secret_parameters, dict): - secret_parameters[parameter] = parameter_type - else: - return parameter_type return secret_parameters diff --git a/st2common/tests/unit/test_util_secrets.py b/st2common/tests/unit/test_util_secrets.py index 848b372574..0faef33523 100644 --- a/st2common/tests/unit/test_util_secrets.py +++ b/st2common/tests/unit/test_util_secrets.py @@ -319,9 +319,34 @@ ] } +################################################################################ + +TEST_SECRET_ARRAY_SCHEMA = { + 'arg_secret_array': { + 'description': 'Mirror', + 'type': 'array', + 'secret': True, + } +} + +TEST_SECRET_ARRAY_SECRET_PARAMS = { + 'arg_secret_array': 'array' +} ################################################################################ +TEST_SECRET_OBJECT_SCHEMA = { + 'arg_secret_object': { + 'type': 'object', + 'secret': True, + } +} + +TEST_SECRET_OBJECT_SECRET_PARAMS = { + 'arg_secret_object': 'object' +} + +################################################################################ class SecretUtilsTestCase(unittest2.TestCase): @@ -361,8 +386,15 @@ def test_get_secret_parameters_nested_array_with_object(self): result = secrets.get_secret_parameters(TEST_NESTED_ARRAY_WITH_OBJECT_SCHEMA) self.assertEqual(TEST_NESTED_ARRAY_WITH_OBJECT_SECRET_PARAMS, result) + def test_get_secret_parameters_secret_array(self): + result = secrets.get_secret_parameters(TEST_SECRET_ARRAY_SCHEMA) + self.assertEqual(TEST_SECRET_ARRAY_SECRET_PARAMS, result) + + def test_get_secret_parameters_secret_object(self): + result = secrets.get_secret_parameters(TEST_SECRET_OBJECT_SCHEMA) + self.assertEqual(TEST_SECRET_OBJECT_SECRET_PARAMS, result) + ############################################################################ - # TODO unit tests for mask_secret_parameters def test_mask_secret_parameters_flat(self): parameters = { @@ -669,3 +701,39 @@ def test_mask_secret_parameters_nested_array_with_object(self): ] } self.assertEqual(expected, result) + + def test_mask_secret_array(self): + parameters = { + 'arg_secret_array': [ + "abc", + 123, + True + ] + } + result = secrets.mask_secret_parameters(parameters, + TEST_SECRET_ARRAY_SECRET_PARAMS) + expected = { + 'arg_secret_array': MASKED_ATTRIBUTE_VALUE + } + self.assertEqual(expected, result) + + def test_mask_secret_object(self): + parameters = { + 'arg_secret_object': + { + "abc": 123, + "key": "value", + "bool": True, + "array": ["x", "y", "z"], + "obj": + { + "x": "deep" + } + } + } + result = secrets.mask_secret_parameters(parameters, + TEST_SECRET_OBJECT_SECRET_PARAMS) + expected = { + 'arg_secret_object': MASKED_ATTRIBUTE_VALUE + } + self.assertEqual(expected, result) From 70cb585eb0e4ef55d9dcc79489aefb976fddf5f1 Mon Sep 17 00:00:00 2001 From: Nick Maludy Date: Tue, 10 Jul 2018 12:44:10 -0400 Subject: [PATCH 02/27] Added comments to code about object/array secret masking --- st2common/st2common/util/secrets.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/st2common/st2common/util/secrets.py b/st2common/st2common/util/secrets.py index d605cd4e83..fbe566dbe1 100644 --- a/st2common/st2common/util/secrets.py +++ b/st2common/st2common/util/secrets.py @@ -96,7 +96,13 @@ def get_secret_parameters(parameters): parameter_type = options.get('type') if options.get('secret', False): - # if this parameter is secret, then add it our secret parameters + # If this parameter is secret, then add it our secret parameters + # + # **This causes the _full_ object / array tree to be secret.** + # + # **Important** that we do this check first, so in case this parameter + # is an `object` or `array`, and the user wants the full thing + # to be secret, that it is marked as secret. if isinstance(secret_parameters, list): secret_parameters.append(parameter_type) elif isinstance(secret_parameters, dict): @@ -104,6 +110,8 @@ def get_secret_parameters(parameters): else: return parameter_type elif parameter_type in ['object', 'array']: + # otherwise recursively dive into the `object`/`array` and + # find individual parameters marked as secret sub_params = get_secret_parameters(options) if sub_params: if isinstance(secret_parameters, list): From 7ccafe01c9e19d7fdf9987c849f815ce885b0a6f Mon Sep 17 00:00:00 2001 From: Nick Maludy Date: Tue, 10 Jul 2018 13:27:38 -0400 Subject: [PATCH 03/27] Added a ton more tests for nested secrets objects and arrays --- st2common/st2common/util/secrets.py | 14 +- st2common/tests/unit/test_util_secrets.py | 363 +++++++++++++++++++++- 2 files changed, 374 insertions(+), 3 deletions(-) diff --git a/st2common/st2common/util/secrets.py b/st2common/st2common/util/secrets.py index fbe566dbe1..57d8a0b08f 100644 --- a/st2common/st2common/util/secrets.py +++ b/st2common/st2common/util/secrets.py @@ -66,6 +66,17 @@ def get_secret_parameters(parameters): secret_parameters = {} parameters_type = parameters.get('type') + # If the parameter itself is secret, then skip all processing below it + # and return the type of this parameter. + # + # **This causes the _full_ object / array tree to be secret (no children will be shown).** + # + # **Important** that we do this check first, so in case this parameter + # is an `object` or `array`, and the user wants the full thing + # to be secret, that it is marked as secret. + if parameters.get('secret', False): + return parameters_type + iterator = None if parameters_type == 'object': # if this is an object, then iterate over the properties within @@ -98,7 +109,8 @@ def get_secret_parameters(parameters): if options.get('secret', False): # If this parameter is secret, then add it our secret parameters # - # **This causes the _full_ object / array tree to be secret.** + # **This causes the _full_ object / array tree to be secret + # (no children will be shown)** # # **Important** that we do this check first, so in case this parameter # is an `object` or `array`, and the user wants the full thing diff --git a/st2common/tests/unit/test_util_secrets.py b/st2common/tests/unit/test_util_secrets.py index 0faef33523..8f7cf4cdaf 100644 --- a/st2common/tests/unit/test_util_secrets.py +++ b/st2common/tests/unit/test_util_secrets.py @@ -145,6 +145,32 @@ ################################################################################ +TEST_ROOT_OBJECT_SCHEMA = { + 'description': 'root', + 'type': 'object', + 'properties': { + 'arg_level_one': { + 'description': 'down', + 'type': 'object', + 'properties': { + 'secret_field_in_object': { + 'type': 'string', + 'secret': True + } + } + } + } +} + +TEST_ROOT_OBJECT_SECRET_PARAMS = { + 'arg_level_one': + { + 'secret_field_in_object': 'string' + } +} + +################################################################################ + TEST_NESTED_ARRAYS_SCHEMA = { 'arg_optional_array': { 'description': 'Mirror', @@ -346,6 +372,182 @@ 'arg_secret_object': 'object' } + +################################################################################ + +TEST_SECRET_ROOT_ARRAY_SCHEMA = { + 'description': 'secret array', + 'type': 'array', + 'secret': True, + 'items': { + 'description': 'down', + 'type': 'object', + 'properties': { + 'secret_field_in_object': { + 'type': 'string', + 'secret': True + } + } + } +} + +TEST_SECRET_ROOT_ARRAY_SECRET_PARAMS = 'array' + +################################################################################ + +TEST_SECRET_ROOT_OBJECT_SCHEMA = { + 'description': 'secret object', + 'type': 'object', + 'secret': True, + 'proeprteis': { + 'arg_level_one': { + 'description': 'down', + 'type': 'object', + 'properties': { + 'secret_field_in_object': { + 'type': 'string', + 'secret': True + } + } + } + } +} + +TEST_SECRET_ROOT_OBJECT_SECRET_PARAMS = 'object' + +################################################################################ + +TEST_SECRET_NESTED_OBJECTS_SCHEMA = { + 'arg_object': { + 'description': 'Mirror', + 'type': 'object', + 'properties': { + 'arg_nested_object': { + 'description': 'Mirror mirror', + 'type': 'object', + 'secret': True, + 'properties': { + 'arg_double_nested_secret': { + 'description': 'Deep, deep down', + 'type': 'string', + 'secret': True + } + } + }, + 'arg_nested_secret': { + 'description': 'Deep down', + 'type': 'string', + 'secret': True + } + } + }, + 'arg_secret_object': { + 'description': 'Mirror', + 'type': 'object', + 'secret': True, + 'properties': { + 'arg_nested_object': { + 'description': 'Mirror mirror', + 'type': 'object', + 'secret': True, + 'properties': { + 'arg_double_nested_secret': { + 'description': 'Deep, deep down', + 'type': 'string', + 'secret': True + } + } + }, + 'arg_nested_secret': { + 'description': 'Deep down', + 'type': 'string', + 'secret': True + } + } + } +} + +TEST_SECRET_NESTED_OBJECTS_SECRET_PARAMS = { + 'arg_object': { + 'arg_nested_secret': 'string', + 'arg_nested_object': 'object' + }, + 'arg_secret_object': 'object' +} + + +################################################################################ + +TEST_SECRET_NESTED_ARRAYS_SCHEMA = { + 'arg_optional_array': { + 'description': 'Mirror', + 'type': 'array', + 'secret': True, + 'items': { + 'description': 'Deep down', + 'type': 'string' + } + }, + 'arg_optional_double_array': { + 'description': 'Mirror', + 'type': 'array', + 'secret': True, + 'items': { + 'type': 'array', + 'items': { + 'description': 'Deep down', + 'type': 'string', + } + } + }, + 'arg_optional_tripple_array': { + 'description': 'Mirror', + 'type': 'array', + 'items': { + 'type': 'array', + 'secret': True, + 'items': { + 'type': 'array', + 'items': { + 'description': 'Deep down', + 'type': 'string', + } + } + } + }, + 'arg_optional_quad_array': { + 'description': 'Mirror', + 'type': 'array', + 'items': { + 'type': 'array', + 'items': { + 'type': 'array', + 'secret': True, + 'items': { + 'type': 'array', + 'items': { + 'description': 'Deep down', + 'type': 'string', + } + } + } + } + } +} + +TEST_SECRET_NESTED_ARRAYS_SECRET_PARAMS = { + 'arg_optional_array': 'array', + 'arg_optional_double_array': 'array', + 'arg_optional_tripple_array': [ + 'array' + ], + 'arg_optional_quad_array': [ + [ + 'array' + ] + ] +} + ################################################################################ class SecretUtilsTestCase(unittest2.TestCase): @@ -370,6 +572,10 @@ def test_get_secret_parameters_root_array(self): result = secrets.get_secret_parameters(TEST_ROOT_ARRAY_SCHEMA) self.assertEqual(TEST_ROOT_ARRAY_SECRET_PARAMS, result) + def test_get_secret_parameters_root_object(self): + result = secrets.get_secret_parameters(TEST_ROOT_OBJECT_SCHEMA) + self.assertEqual(TEST_ROOT_OBJECT_SECRET_PARAMS, result) + def test_get_secret_parameters_nested_arrays(self): result = secrets.get_secret_parameters(TEST_NESTED_ARRAYS_SCHEMA) self.assertEqual(TEST_NESTED_ARRAYS_SECRET_PARAMS, result) @@ -394,6 +600,22 @@ def test_get_secret_parameters_secret_object(self): result = secrets.get_secret_parameters(TEST_SECRET_OBJECT_SCHEMA) self.assertEqual(TEST_SECRET_OBJECT_SECRET_PARAMS, result) + def test_get_secret_parameters_secret_root_array(self): + result = secrets.get_secret_parameters(TEST_SECRET_ROOT_ARRAY_SCHEMA) + self.assertEqual(TEST_SECRET_ROOT_ARRAY_SECRET_PARAMS, result) + + def test_get_secret_parameters_secret_root_object(self): + result = secrets.get_secret_parameters(TEST_SECRET_ROOT_OBJECT_SCHEMA) + self.assertEqual(TEST_SECRET_ROOT_OBJECT_SECRET_PARAMS, result) + + def test_get_secret_parameters_secret_nested_arrays(self): + result = secrets.get_secret_parameters(TEST_SECRET_NESTED_ARRAYS_SCHEMA) + self.assertEqual(TEST_SECRET_NESTED_ARRAYS_SECRET_PARAMS, result) + + def test_get_secret_parameters_secret_nested_objects(self): + result = secrets.get_secret_parameters(TEST_SECRET_NESTED_OBJECTS_SCHEMA) + self.assertEqual(TEST_SECRET_NESTED_OBJECTS_SECRET_PARAMS, result) + ############################################################################ def test_mask_secret_parameters_flat(self): @@ -491,6 +713,23 @@ def test_mask_secret_parameters_root_array(self): ] self.assertEqual(expected, result) + def test_mask_secret_parameters_root_object(self): + parameters = { + 'arg_level_one': + { + 'secret_field_in_object': 'Secret $tr!ng' + } + } + + result = secrets.mask_secret_parameters(parameters, TEST_ROOT_OBJECT_SECRET_PARAMS) + expected = { + 'arg_level_one': + { + 'secret_field_in_object': MASKED_ATTRIBUTE_VALUE + } + } + self.assertEqual(expected, result) + def test_mask_secret_parameters_nested_arrays(self): parameters = { 'arg_optional_array': [ @@ -702,7 +941,7 @@ def test_mask_secret_parameters_nested_array_with_object(self): } self.assertEqual(expected, result) - def test_mask_secret_array(self): + def test_mask_secret_parameters_secret_array(self): parameters = { 'arg_secret_array': [ "abc", @@ -717,7 +956,7 @@ def test_mask_secret_array(self): } self.assertEqual(expected, result) - def test_mask_secret_object(self): + def test_mask_secret_parameters_secret_object(self): parameters = { 'arg_secret_object': { @@ -737,3 +976,123 @@ def test_mask_secret_object(self): 'arg_secret_object': MASKED_ATTRIBUTE_VALUE } self.assertEqual(expected, result) + + def test_mask_secret_parameters_secret_root_array(self): + parameters = [ + "abc", + 123, + True + ] + result = secrets.mask_secret_parameters(parameters, + TEST_SECRET_ROOT_ARRAY_SECRET_PARAMS) + expected = MASKED_ATTRIBUTE_VALUE + self.assertEqual(expected, result) + + def test_mask_secret_parameters_secret_root_object(self): + parameters = { + 'arg_level_one': + { + 'secret_field_in_object': 'Secret $tr!ng' + } + } + result = secrets.mask_secret_parameters(parameters, + TEST_SECRET_ROOT_OBJECT_SECRET_PARAMS) + expected = MASKED_ATTRIBUTE_VALUE + self.assertEqual(expected, result) + + def test_mask_secret_parameters_secret_nested_arrays(self): + parameters = { + 'arg_optional_array': [ + 'secret 1', + 'secret 2', + 'secret 3', + ], + 'arg_optional_double_array': [ + [ + 'secret 4', + 'secret 5', + 'secret 6', + ], + [ + 'secret 7', + 'secret 8', + 'secret 9', + ] + ], + 'arg_optional_tripple_array': [ + [ + [ + 'secret 10', + 'secret 11' + ], + [ + 'secret 12', + 'secret 13', + 'secret 14' + ] + ], + [ + [ + 'secret 15', + 'secret 16' + ] + ] + ], + 'arg_optional_quad_array': [ + [ + [ + [ + 'secret 17', + 'secret 18' + ], + [ + 'secret 19' + ] + ] + ] + ] + } + + result = secrets.mask_secret_parameters(parameters, + TEST_SECRET_NESTED_ARRAYS_SECRET_PARAMS) + + expected = { + 'arg_optional_array': MASKED_ATTRIBUTE_VALUE, + 'arg_optional_double_array': MASKED_ATTRIBUTE_VALUE, + 'arg_optional_tripple_array': [ + MASKED_ATTRIBUTE_VALUE, + MASKED_ATTRIBUTE_VALUE, + ], + 'arg_optional_quad_array': [ + [ + MASKED_ATTRIBUTE_VALUE, + ] + ] + } + self.assertEqual(expected, result) + + def test_mask_secret_parameters_secret_nested_objects(self): + parameters = { + 'arg_object': { + 'arg_nested_secret': 'nested Secret', + 'arg_nested_object': { + 'arg_double_nested_secret': 'double nested $ecret', + } + }, + 'arg_secret_object': { + 'arg_nested_secret': 'secret data', + 'arg_nested_object': { + 'arg_double_nested_secret': 'double nested $ecret', + } + } + } + result = secrets.mask_secret_parameters(parameters, + TEST_SECRET_NESTED_OBJECTS_SECRET_PARAMS) + expected = { + 'arg_object': { + 'arg_nested_secret': MASKED_ATTRIBUTE_VALUE, + 'arg_nested_object': MASKED_ATTRIBUTE_VALUE, + }, + 'arg_secret_object': MASKED_ATTRIBUTE_VALUE, + } + self.assertEqual(expected, result) From 6f0a3f0cf08d1247d70dc8d1c4c1e4e7dc991399 Mon Sep 17 00:00:00 2001 From: Nick Maludy Date: Tue, 10 Jul 2018 13:31:25 -0400 Subject: [PATCH 04/27] Fix flake8 for unit tests --- st2common/tests/unit/test_util_secrets.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/st2common/tests/unit/test_util_secrets.py b/st2common/tests/unit/test_util_secrets.py index 8f7cf4cdaf..b110492180 100644 --- a/st2common/tests/unit/test_util_secrets.py +++ b/st2common/tests/unit/test_util_secrets.py @@ -550,6 +550,7 @@ ################################################################################ + class SecretUtilsTestCase(unittest2.TestCase): def test_get_secret_parameters_flat(self): @@ -978,7 +979,7 @@ def test_mask_secret_parameters_secret_object(self): self.assertEqual(expected, result) def test_mask_secret_parameters_secret_root_array(self): - parameters = [ + parameters = [ "abc", 123, True From 2ccff02678e67b6c03c0060323e0be2fc17826e0 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 11 Jul 2018 11:04:11 +0200 Subject: [PATCH 05/27] Update changelog. --- CHANGELOG.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f4674dafd4..1e77a53577 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,11 +1,9 @@ Changelog ========= -in development --------------- +2.8.1 - TBD +----------- - - Fixed ~~~~~ From 65686006e246f5cea94581ebbb6ebfe69839aa81 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 13 Jul 2018 13:48:02 +0200 Subject: [PATCH 06/27] Allow user to force terminal size used by the st2 CLI formattes. This comes handy in scenarios where terminal size can't be obtained (e.g. when running tests outside tty). --- st2client/st2client/utils/terminal.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/st2client/st2client/utils/terminal.py b/st2client/st2client/utils/terminal.py index d53a342fc5..f9c622e85e 100644 --- a/st2client/st2client/utils/terminal.py +++ b/st2client/st2client/utils/terminal.py @@ -14,6 +14,7 @@ # limitations under the License. from __future__ import absolute_import + import os import struct import subprocess @@ -30,6 +31,16 @@ def get_terminal_size(default=(80, 20)): """ :return: (lines, cols) """ + # Allow user to force terminal size using a environment variables + # E.g. ST2_CLI_FORCE_TERMINAL_SIZE=80,200 # lines, columns + force_terminal_size = os.environ.get('ST2_CLI_FORCE_TERMINAL_SIZE', None) + if force_terminal_size: + split = force_terminal_size.split(',') + lines = split[0] + columns = split[1] if len(split) >= 2 else default[1] + + return lines, columns + def ioctl_GWINSZ(fd): import fcntl import termios From 03e959836028e8188fbcebbf941db5fcf9c7db33 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 13 Jul 2018 13:48:59 +0200 Subject: [PATCH 07/27] Also use a more reasonable default terminal size. --- st2client/st2client/utils/terminal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2client/st2client/utils/terminal.py b/st2client/st2client/utils/terminal.py index f9c622e85e..1c1224867e 100644 --- a/st2client/st2client/utils/terminal.py +++ b/st2client/st2client/utils/terminal.py @@ -27,7 +27,7 @@ ] -def get_terminal_size(default=(80, 20)): +def get_terminal_size(default=(80, 200)): """ :return: (lines, cols) """ From 02b4e690b8cc14243281aa42e5baed59541a6219 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 13 Jul 2018 13:51:19 +0200 Subject: [PATCH 08/27] 200 -> 150.. --- st2client/st2client/utils/terminal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2client/st2client/utils/terminal.py b/st2client/st2client/utils/terminal.py index 1c1224867e..84dff78e3f 100644 --- a/st2client/st2client/utils/terminal.py +++ b/st2client/st2client/utils/terminal.py @@ -27,7 +27,7 @@ ] -def get_terminal_size(default=(80, 200)): +def get_terminal_size(default=(80, 150)): """ :return: (lines, cols) """ From 4cfe2eb05b26cbe833dd891a363a4e3db1d00eee Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 13 Jul 2018 13:57:33 +0200 Subject: [PATCH 09/27] Make sure we cast it to int. --- st2client/st2client/utils/terminal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/st2client/st2client/utils/terminal.py b/st2client/st2client/utils/terminal.py index 84dff78e3f..cfc5ae3e55 100644 --- a/st2client/st2client/utils/terminal.py +++ b/st2client/st2client/utils/terminal.py @@ -36,8 +36,8 @@ def get_terminal_size(default=(80, 150)): force_terminal_size = os.environ.get('ST2_CLI_FORCE_TERMINAL_SIZE', None) if force_terminal_size: split = force_terminal_size.split(',') - lines = split[0] - columns = split[1] if len(split) >= 2 else default[1] + lines = int(split[0]) + columns = int(split[1] if len(split) >= 2 else default[1]) return lines, columns From d75d5006f6963604cb623d20695bbdc0af71c16c Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 13 Jul 2018 14:01:40 +0200 Subject: [PATCH 10/27] Truncate extra whitespace. --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1e77a53577..0386a62f3a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,7 +11,7 @@ Fixed Reported by @jjm Contributed by Nick Maludy (Encore Technologies). - + 2.8.0 - July 10, 2018 --------------------- From 2aba4e9baa10d72cf37b873e1baa17cd10522d87 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 13 Jul 2018 17:01:31 +0200 Subject: [PATCH 11/27] Add a note. --- st2client/st2client/utils/terminal.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/st2client/st2client/utils/terminal.py b/st2client/st2client/utils/terminal.py index cfc5ae3e55..cc03152221 100644 --- a/st2client/st2client/utils/terminal.py +++ b/st2client/st2client/utils/terminal.py @@ -29,6 +29,15 @@ def get_terminal_size(default=(80, 150)): """ + Try to retrieve a default terminal size using various system specific approaches. + + If terminal size can't be retrieved, default value is returned. + + NOTE 1: LINES and COLUMNS environment variables are checked first, if those values are not set / + available, other methods are tried. + + NOTE 2: This method requires both environment variables to be specified together. + :return: (lines, cols) """ # Allow user to force terminal size using a environment variables From 88165894e8aac404290aa0c587fd5f34659ab8bc Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 13 Jul 2018 17:12:37 +0200 Subject: [PATCH 12/27] Add tests for get_terminal_size. --- st2client/tests/unit/test_util_terminal.py | 62 ++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 st2client/tests/unit/test_util_terminal.py diff --git a/st2client/tests/unit/test_util_terminal.py b/st2client/tests/unit/test_util_terminal.py new file mode 100644 index 0000000000..95f3e243a5 --- /dev/null +++ b/st2client/tests/unit/test_util_terminal.py @@ -0,0 +1,62 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +import unittest2 +import mock + +from st2client.utils.terminal import DEFAULT_TERMINAL_SIZE_LINES +from st2client.utils.terminal import DEFAULT_TERMINAL_SIZE_COLUMNS +from st2client.utils.terminal import get_terminal_size + +__all__ = [ + 'TerminalUtilsTestCase' +] + + +class TerminalUtilsTestCase(unittest2.TestCase): + @mock.patch.dict(os.environ, {'LINES': '111', 'COLUMNS': '222'}) + def test_get_terminal_size_lines_columns_environment_variable_have_precedence(self): + # Verify that LINES and COLUMNS environment variables have precedence over other approaches + lines, columns = get_terminal_size() + + self.assertEqual(lines, 111) + self.assertEqual(columns, 222) + + @mock.patch('struct.unpack', mock.Mock(side_effect=Exception('a'))) + @mock.patch('subprocess.Popen') + def test_get_terminal_size_subprocess_popen_is_used(self, mock_popen): + mock_communicate = mock.Mock(return_value=['555 666']) + + mock_process = mock.Mock() + mock_process.returncode = 0 + mock_process.communicate = mock_communicate + + mock_popen.return_value = mock_process + + lines, columns = get_terminal_size() + self.assertEqual(lines, 555) + self.assertEqual(columns, 666) + + @mock.patch('struct.unpack', mock.Mock(side_effect=Exception('a'))) + @mock.patch('subprocess.Popen', mock.Mock(side_effect=Exception('b'))) + def test_get_terminal_size_default_values_are_used(self): + lines, columns = get_terminal_size() + + self.assertEqual(lines, DEFAULT_TERMINAL_SIZE_LINES) + self.assertEqual(columns, DEFAULT_TERMINAL_SIZE_COLUMNS) From d2dc4e697f766fa7f8d096cc98bd8aeb8037e227 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 13 Jul 2018 17:18:31 +0200 Subject: [PATCH 13/27] Number the various fallbacks. --- st2client/st2client/utils/terminal.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/st2client/st2client/utils/terminal.py b/st2client/st2client/utils/terminal.py index cc03152221..fc1ceceefd 100644 --- a/st2client/st2client/utils/terminal.py +++ b/st2client/st2client/utils/terminal.py @@ -40,13 +40,13 @@ def get_terminal_size(default=(80, 150)): :return: (lines, cols) """ - # Allow user to force terminal size using a environment variables - # E.g. ST2_CLI_FORCE_TERMINAL_SIZE=80,200 # lines, columns - force_terminal_size = os.environ.get('ST2_CLI_FORCE_TERMINAL_SIZE', None) - if force_terminal_size: - split = force_terminal_size.split(',') - lines = int(split[0]) - columns = int(split[1] if len(split) >= 2 else default[1]) + # 1. Try LINES and COLUMNS environment variables first like in upstream Python 3 method - + # https://github.com/python/cpython/blob/master/Lib/shutil.py#L1203 + # This way it's consistent with upstream implementation. In the past, our implementation + # checked those variables at the end as a fall back. + try: + lines = os.environ['LINES'] + columns = os.environ['COLUMNS'] return lines, columns @@ -54,13 +54,15 @@ def ioctl_GWINSZ(fd): import fcntl import termios return struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) - # try stdin, stdout, stderr + + # 2. try stdin, stdout, stderr for fd in (0, 1, 2): try: return ioctl_GWINSZ(fd) except: pass - # try os.ctermid() + + # 3. try os.ctermid() try: fd = os.open(os.ctermid(), os.O_RDONLY) try: @@ -69,7 +71,8 @@ def ioctl_GWINSZ(fd): os.close(fd) except: pass - # try `stty size` + + # 4. try `stty size` try: process = subprocess.Popen(['stty', 'size'], shell=False, @@ -80,12 +83,14 @@ def ioctl_GWINSZ(fd): return tuple(int(x) for x in result[0].split()) except: pass + # try environment variables try: return tuple(int(os.getenv(var)) for var in ('LINES', 'COLUMNS')) except: pass - # return default. + + # 5. return default value return default From 82edde6a35e2017fad9f53780c00910217a74d46 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 13 Jul 2018 17:45:22 +0200 Subject: [PATCH 14/27] Replace get_terminal_size with get_terminal_size_columns. We only use columns value of terminal size so this allows us to simplify code for retrieving terminal size. --- st2client/st2client/formatters/table.py | 4 +-- st2client/st2client/utils/terminal.py | 41 ++++++++++++---------- st2client/tests/unit/test_util_terminal.py | 21 +++++------ 3 files changed, 36 insertions(+), 30 deletions(-) diff --git a/st2client/st2client/formatters/table.py b/st2client/st2client/formatters/table.py index ff783349fc..c29c583426 100644 --- a/st2client/st2client/formatters/table.py +++ b/st2client/st2client/formatters/table.py @@ -27,7 +27,7 @@ from st2client import formatters from st2client.utils import strutil -from st2client.utils.terminal import get_terminal_size +from st2client.utils.terminal import get_terminal_size_columns LOG = logging.getLogger(__name__) @@ -65,7 +65,7 @@ def format(cls, entries, *args, **kwargs): if not widths and attributes: # Dynamically calculate column size based on the terminal size - lines, cols = get_terminal_size() + cols = get_terminal_size_columns() if attributes[0] == 'id': # consume iterator and save as entries so collection is accessible later. diff --git a/st2client/st2client/utils/terminal.py b/st2client/st2client/utils/terminal.py index fc1ceceefd..c65bc378e2 100644 --- a/st2client/st2client/utils/terminal.py +++ b/st2client/st2client/utils/terminal.py @@ -22,54 +22,59 @@ from st2client.utils.color import format_status +DEFAULT_TERMINAL_SIZE_COLUMNS = 150 + __all__ = [ - 'get_terminal_size' + 'DEFAULT_TERMINAL_SIZE_COLUMNS', + + 'get_terminal_size_columns' ] -def get_terminal_size(default=(80, 150)): +def get_terminal_size_columns(default=DEFAULT_TERMINAL_SIZE_COLUMNS): """ - Try to retrieve a default terminal size using various system specific approaches. + Try to retrieve COLUMNS value of terminal size using various system specific approaches. If terminal size can't be retrieved, default value is returned. - NOTE 1: LINES and COLUMNS environment variables are checked first, if those values are not set / - available, other methods are tried. - - NOTE 2: This method requires both environment variables to be specified together. + NOTE 1: COLUMNS environment variable is checked first, if the value is not set / available, + other methods are tried. - :return: (lines, cols) + :rtype: ``int`` + :return: columns """ - # 1. Try LINES and COLUMNS environment variables first like in upstream Python 3 method - + # 1. Try COLUMNS environment variable first like in upstream Python 3 method - # https://github.com/python/cpython/blob/master/Lib/shutil.py#L1203 # This way it's consistent with upstream implementation. In the past, our implementation # checked those variables at the end as a fall back. try: - lines = os.environ['LINES'] columns = os.environ['COLUMNS'] - return lines, columns + return int(columns) + except (KeyError, ValueError): + pass def ioctl_GWINSZ(fd): import fcntl import termios + # Return a tuple (lines, columns) return struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) # 2. try stdin, stdout, stderr for fd in (0, 1, 2): try: - return ioctl_GWINSZ(fd) - except: + return ioctl_GWINSZ(fd)[1] + except Exception: pass # 3. try os.ctermid() try: fd = os.open(os.ctermid(), os.O_RDONLY) try: - return ioctl_GWINSZ(fd) + return ioctl_GWINSZ(fd)[1] finally: os.close(fd) - except: + except Exception: pass # 4. try `stty size` @@ -80,8 +85,8 @@ def ioctl_GWINSZ(fd): stderr=open(os.devnull, 'w')) result = process.communicate() if process.returncode == 0: - return tuple(int(x) for x in result[0].split()) - except: + return tuple(int(x) for x in result[0].split())[1] + except Exception: pass # try environment variables @@ -90,7 +95,7 @@ def ioctl_GWINSZ(fd): except: pass - # 5. return default value + # 5. return default fallback value return default diff --git a/st2client/tests/unit/test_util_terminal.py b/st2client/tests/unit/test_util_terminal.py index 95f3e243a5..b7635f2494 100644 --- a/st2client/tests/unit/test_util_terminal.py +++ b/st2client/tests/unit/test_util_terminal.py @@ -20,9 +20,8 @@ import unittest2 import mock -from st2client.utils.terminal import DEFAULT_TERMINAL_SIZE_LINES from st2client.utils.terminal import DEFAULT_TERMINAL_SIZE_COLUMNS -from st2client.utils.terminal import get_terminal_size +from st2client.utils.terminal import get_terminal_size_columns __all__ = [ 'TerminalUtilsTestCase' @@ -31,13 +30,17 @@ class TerminalUtilsTestCase(unittest2.TestCase): @mock.patch.dict(os.environ, {'LINES': '111', 'COLUMNS': '222'}) - def test_get_terminal_size_lines_columns_environment_variable_have_precedence(self): - # Verify that LINES and COLUMNS environment variables have precedence over other approaches - lines, columns = get_terminal_size() + def test_get_terminal_size_columns_columns_environment_variable_has_precedence(self): + # Verify that COLUMNS environment variables has precedence over other approaches + columns = get_terminal_size_columns() - self.assertEqual(lines, 111) self.assertEqual(columns, 222) + @mock.patch('struct.unpack', mock.Mock(return_value=(333, 444))) + def test_get_terminal_size_columns_stdout_is_used(self): + columns = get_terminal_size_columns() + self.assertEqual(columns, 444) + @mock.patch('struct.unpack', mock.Mock(side_effect=Exception('a'))) @mock.patch('subprocess.Popen') def test_get_terminal_size_subprocess_popen_is_used(self, mock_popen): @@ -49,14 +52,12 @@ def test_get_terminal_size_subprocess_popen_is_used(self, mock_popen): mock_popen.return_value = mock_process - lines, columns = get_terminal_size() - self.assertEqual(lines, 555) + columns = get_terminal_size_columns() self.assertEqual(columns, 666) @mock.patch('struct.unpack', mock.Mock(side_effect=Exception('a'))) @mock.patch('subprocess.Popen', mock.Mock(side_effect=Exception('b'))) def test_get_terminal_size_default_values_are_used(self): - lines, columns = get_terminal_size() + columns = get_terminal_size_columns() - self.assertEqual(lines, DEFAULT_TERMINAL_SIZE_LINES) self.assertEqual(columns, DEFAULT_TERMINAL_SIZE_COLUMNS) From 5e017e6baabb48946f1b706e6704d0af09f1a3b1 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 16 Jul 2018 10:48:49 +0200 Subject: [PATCH 15/27] Update changelog. --- CHANGELOG.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0386a62f3a..322415ac2b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,28 @@ Changelog 2.8.1 - TBD ----------- +Added +----- + +* Update ``st2`` CLI to inspect ``COLUMNS`` environment variable first when determining the + terminal size. Previously this environment variable was checked second last (after trying to + retrieve terminal size using various OS specific methods and before falling back to the default + value). + + This approach is more performant and allows user to easily overwrite the default value or value + returned by the operating system checks - e.g. by running ``COLUMNS=200 st2 action list``. + (improvement) #4242 + +Changed +------- + +* Update ``st2`` CLI to use a more sensible default terminal size for table formatting purposes if + we are unable to retrieve terminal size using various system-specific approaches. + + Previously we would fall back to a very unfriendly default of 20 columns for a total terminal + width. This would cause every table column to wrap and make output impossible / hard to read. + (improvement) #4242 + Fixed ~~~~~ From 27227b59956cd9c72fc92df61cef7054a7f0483a Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 3 Jul 2018 18:36:14 +1000 Subject: [PATCH 16/27] swap yaml.load with yaml.safe_load --- contrib/packs/actions/pack_mgmt/get_installed.py | 2 +- st2common/st2common/util/spec_loader.py | 2 +- st2common/tests/unit/test_jinja_render_data_filters.py | 2 +- st2tests/integration/mistral/test_filters.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/packs/actions/pack_mgmt/get_installed.py b/contrib/packs/actions/pack_mgmt/get_installed.py index f768172cba..07a57c006c 100644 --- a/contrib/packs/actions/pack_mgmt/get_installed.py +++ b/contrib/packs/actions/pack_mgmt/get_installed.py @@ -87,5 +87,5 @@ def run(self, pack): def _parse_yaml_file(self, file_path): with open(file_path) as data_file: - details = yaml.load(data_file) + details = yaml.safe_load(data_file) return details diff --git a/st2common/st2common/util/spec_loader.py b/st2common/st2common/util/spec_loader.py index c2ce023017..743d897ebb 100644 --- a/st2common/st2common/util/spec_loader.py +++ b/st2common/st2common/util/spec_loader.py @@ -39,7 +39,7 @@ def load_spec(module_name, spec_file): spec_string = generate_spec(module_name, spec_file) - spec = yaml.load(spec_string) + spec = yaml.safe_load(spec_string) return spec diff --git a/st2common/tests/unit/test_jinja_render_data_filters.py b/st2common/tests/unit/test_jinja_render_data_filters.py index 864afe33b8..594de540da 100644 --- a/st2common/tests/unit/test_jinja_render_data_filters.py +++ b/st2common/tests/unit/test_jinja_render_data_filters.py @@ -96,5 +96,5 @@ def test_filter_to_yaml_string(self): template = '{{k1 | to_yaml_string}}' obj_yaml_str = env.from_string(template).render({'k1': obj}) - actual_obj = yaml.load(obj_yaml_str) + actual_obj = yaml.safe_load(obj_yaml_str) self.assertDictEqual(obj, actual_obj) diff --git a/st2tests/integration/mistral/test_filters.py b/st2tests/integration/mistral/test_filters.py index ed26075c9f..916f52a42f 100644 --- a/st2tests/integration/mistral/test_filters.py +++ b/st2tests/integration/mistral/test_filters.py @@ -243,8 +243,8 @@ def test_to_yaml_string(self): ex = self._execute_workflow('examples.mistral-test-func-to-yaml-string', params) ex = self._wait_for_completion(ex) self.assertEqual(ex.status, action_constants.LIVEACTION_STATUS_SUCCEEDED) - jinja_dict = yaml.load(ex.result['result_jinja']) - yaql_dict = yaml.load(ex.result['result_yaql']) + jinja_dict = yaml.safe_load(ex.result['result_jinja']) + yaql_dict = yaml.safe_load(ex.result['result_yaql']) self.assertTrue(isinstance(jinja_dict, dict)) self.assertEqual(jinja_dict['a'], 'b') self.assertTrue(isinstance(yaql_dict, dict)) From 2017b2ebc29ec5ef2024d592ad9396b4677454d5 Mon Sep 17 00:00:00 2001 From: Lindsay Hill Date: Wed, 27 Jun 2018 17:15:29 -0700 Subject: [PATCH 17/27] Make setup requirements consistent --- st2client/setup.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/st2client/setup.py b/st2client/setup.py index f673d207d3..393894fc01 100644 --- a/st2client/setup.py +++ b/st2client/setup.py @@ -53,12 +53,17 @@ 'Programming Language :: Python :: 2.7' ], install_requires=[ - 'jsonpath-rw>=1.3.0', + 'argcomplete', + 'jsonpath-rw>=1.4.0', 'prettytable', + 'prompt-toolkit==1.0.15', 'python-dateutil', - 'pyyaml<4.0,>=3.11', + 'python-editor==1.0.3', + 'pytz==2018.4', + 'pyyaml<4.0,>=3.12', 'requests[security]<2.15,>=2.14.1', - 'six==1.11.0' + 'six==1.11.0', + 'sseclient==0.0.19' ], dependency_links=[], test_suite=ST2_COMPONENT, From a1bbddc39c45e70a6a3e0f59fe01fc82c5178fb8 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 28 Jun 2018 10:36:04 +0200 Subject: [PATCH 18/27] Update st2client setup.py file to dynamically load requirements from requirements.txt file. This is now possible since we install pip 9.0.0 in pack and st2 virtual environments. Keep in mind that this won't work with very old versions of pip (< 9.0). --- st2client/setup.py | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/st2client/setup.py b/st2client/setup.py index 393894fc01..587f2c0faf 100644 --- a/st2client/setup.py +++ b/st2client/setup.py @@ -18,19 +18,20 @@ from setuptools import setup, find_packages -# Note: We should re-enable usage of dist_utils once we ensure -# that we install new version of virtualenv which ships with -# pip >= 6.1 in all the environments -# from dist_utils import fetch_requirements -# from dist_utils import apply_vagrant_workaround +from dist_utils import check_pip_version +from dist_utils import fetch_requirements +from dist_utils import apply_vagrant_workaround + from st2client import __version__ +check_pip_version() + ST2_COMPONENT = 'st2client' BASE_DIR = os.path.dirname(os.path.abspath(__file__)) REQUIREMENTS_FILE = os.path.join(BASE_DIR, 'requirements.txt') -# install_reqs, dep_links = fetch_requirements(REQUIREMENTS_FILE) -# apply_vagrant_workaround() +install_reqs, dep_links = fetch_requirements(REQUIREMENTS_FILE) +apply_vagrant_workaround() setup( name=ST2_COMPONENT, @@ -52,20 +53,8 @@ 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7' ], - install_requires=[ - 'argcomplete', - 'jsonpath-rw>=1.4.0', - 'prettytable', - 'prompt-toolkit==1.0.15', - 'python-dateutil', - 'python-editor==1.0.3', - 'pytz==2018.4', - 'pyyaml<4.0,>=3.12', - 'requests[security]<2.15,>=2.14.1', - 'six==1.11.0', - 'sseclient==0.0.19' - ], - dependency_links=[], + install_requires=install_reqs, + dependency_links=dep_links, test_suite=ST2_COMPONENT, zip_safe=False, include_package_data=True, From 173cc64fe14bbc587e57f8e775d6134e043abe9f Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 28 Jun 2018 10:40:50 +0200 Subject: [PATCH 19/27] Allow user to pass minimum supported pip version to the function as an argument. --- scripts/dist_utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/dist_utils.py b/scripts/dist_utils.py index ede9e4a358..91a4ff489b 100644 --- a/scripts/dist_utils.py +++ b/scripts/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) From 09aefc3972fee7e7305052fc87afe34a95bab303 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 28 Jun 2018 10:51:46 +0200 Subject: [PATCH 20/27] Update fetch_requirements function so it also works with older versions of pip where the attribute was called req.url (in newer versions it's called req.link). --- scripts/dist_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/dist_utils.py b/scripts/dist_utils.py index 91a4ff489b..965e6387da 100644 --- a/scripts/dist_utils.py +++ b/scripts/dist_utils.py @@ -73,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) From 018ffa0a628207e3fabc2445fb4853aaff0a7d5a Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 28 Jun 2018 10:52:38 +0200 Subject: [PATCH 21/27] Re-generate dist_utils.py files. --- .../runners/action_chain_runner/dist_utils.py | 16 ++++++++++------ .../runners/announcement_runner/dist_utils.py | 16 ++++++++++------ contrib/runners/cloudslang_runner/dist_utils.py | 16 ++++++++++------ contrib/runners/http_runner/dist_utils.py | 16 ++++++++++------ contrib/runners/inquirer_runner/dist_utils.py | 16 ++++++++++------ contrib/runners/local_runner/dist_utils.py | 16 ++++++++++------ contrib/runners/mistral_v2/dist_utils.py | 16 ++++++++++------ contrib/runners/noop_runner/dist_utils.py | 16 ++++++++++------ contrib/runners/orchestra_runner/dist_utils.py | 16 ++++++++++------ contrib/runners/python_runner/dist_utils.py | 16 ++++++++++------ contrib/runners/remote_runner/dist_utils.py | 16 ++++++++++------ contrib/runners/windows_runner/dist_utils.py | 16 ++++++++++------ st2actions/dist_utils.py | 16 ++++++++++------ st2api/dist_utils.py | 16 ++++++++++------ st2auth/dist_utils.py | 16 ++++++++++------ st2client/dist_utils.py | 16 ++++++++++------ st2client/requirements.txt | 1 + st2common/dist_utils.py | 16 ++++++++++------ st2debug/dist_utils.py | 16 ++++++++++------ st2exporter/dist_utils.py | 16 ++++++++++------ st2reactor/dist_utils.py | 16 ++++++++++------ st2stream/dist_utils.py | 16 ++++++++++------ st2tests/dist_utils.py | 16 ++++++++++------ 23 files changed, 221 insertions(+), 132 deletions(-) diff --git a/contrib/runners/action_chain_runner/dist_utils.py b/contrib/runners/action_chain_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/action_chain_runner/dist_utils.py +++ b/contrib/runners/action_chain_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/announcement_runner/dist_utils.py b/contrib/runners/announcement_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/announcement_runner/dist_utils.py +++ b/contrib/runners/announcement_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/cloudslang_runner/dist_utils.py b/contrib/runners/cloudslang_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/cloudslang_runner/dist_utils.py +++ b/contrib/runners/cloudslang_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/http_runner/dist_utils.py b/contrib/runners/http_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/http_runner/dist_utils.py +++ b/contrib/runners/http_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/inquirer_runner/dist_utils.py b/contrib/runners/inquirer_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/inquirer_runner/dist_utils.py +++ b/contrib/runners/inquirer_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/local_runner/dist_utils.py b/contrib/runners/local_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/local_runner/dist_utils.py +++ b/contrib/runners/local_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/mistral_v2/dist_utils.py b/contrib/runners/mistral_v2/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/mistral_v2/dist_utils.py +++ b/contrib/runners/mistral_v2/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/noop_runner/dist_utils.py b/contrib/runners/noop_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/noop_runner/dist_utils.py +++ b/contrib/runners/noop_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/orchestra_runner/dist_utils.py b/contrib/runners/orchestra_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/orchestra_runner/dist_utils.py +++ b/contrib/runners/orchestra_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/python_runner/dist_utils.py b/contrib/runners/python_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/python_runner/dist_utils.py +++ b/contrib/runners/python_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/remote_runner/dist_utils.py b/contrib/runners/remote_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/remote_runner/dist_utils.py +++ b/contrib/runners/remote_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/windows_runner/dist_utils.py b/contrib/runners/windows_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/windows_runner/dist_utils.py +++ b/contrib/runners/windows_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2actions/dist_utils.py b/st2actions/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2actions/dist_utils.py +++ b/st2actions/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2api/dist_utils.py b/st2api/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2api/dist_utils.py +++ b/st2api/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2auth/dist_utils.py b/st2auth/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2auth/dist_utils.py +++ b/st2auth/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2client/dist_utils.py b/st2client/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2client/dist_utils.py +++ b/st2client/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2client/requirements.txt b/st2client/requirements.txt index f514d2b738..1f0a694696 100644 --- a/st2client/requirements.txt +++ b/st2client/requirements.txt @@ -1,4 +1,5 @@ # Don't edit this file. It's generated automatically! +git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper argcomplete jsonpath-rw==1.4.0 prettytable diff --git a/st2common/dist_utils.py b/st2common/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2common/dist_utils.py +++ b/st2common/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2debug/dist_utils.py b/st2debug/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2debug/dist_utils.py +++ b/st2debug/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2exporter/dist_utils.py b/st2exporter/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2exporter/dist_utils.py +++ b/st2exporter/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2reactor/dist_utils.py b/st2reactor/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2reactor/dist_utils.py +++ b/st2reactor/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2stream/dist_utils.py b/st2stream/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2stream/dist_utils.py +++ b/st2stream/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2tests/dist_utils.py b/st2tests/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2tests/dist_utils.py +++ b/st2tests/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) From 3389d1a7b8c87bf4955f4a344e4c7377f291e156 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 3 Jul 2018 16:16:41 +0200 Subject: [PATCH 22/27] st2client doesn't require logshipper package to work. --- st2client/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/st2client/requirements.txt b/st2client/requirements.txt index 1f0a694696..f514d2b738 100644 --- a/st2client/requirements.txt +++ b/st2client/requirements.txt @@ -1,5 +1,4 @@ # Don't edit this file. It's generated automatically! -git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper argcomplete jsonpath-rw==1.4.0 prettytable From 1d10c30562732f777555d67d140ce77e3b95fd23 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 3 Jul 2018 16:20:06 +0200 Subject: [PATCH 23/27] st2client also depends on jsonschema. --- st2client/in-requirements.txt | 1 + st2client/requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/st2client/in-requirements.txt b/st2client/in-requirements.txt index 3214109105..1bcf2d0b7a 100644 --- a/st2client/in-requirements.txt +++ b/st2client/in-requirements.txt @@ -4,6 +4,7 @@ prettytable pytz python-dateutil pyyaml +jsonschema jsonpath-rw requests six diff --git a/st2client/requirements.txt b/st2client/requirements.txt index f514d2b738..604d661fff 100644 --- a/st2client/requirements.txt +++ b/st2client/requirements.txt @@ -1,6 +1,7 @@ # Don't edit this file. It's generated automatically! argcomplete jsonpath-rw==1.4.0 +jsonschema==2.6.0 prettytable prompt-toolkit==1.0.15 python-dateutil From 68ebc0ff151d2de60e92cce0cac9c343cd1d72f4 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Tue, 3 Jul 2018 21:15:04 +0200 Subject: [PATCH 24/27] Make trigger example more complete and also add payload_schema attribute to it. --- contrib/examples/triggers/sample-trigger.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/contrib/examples/triggers/sample-trigger.yaml b/contrib/examples/triggers/sample-trigger.yaml index ae140cad67..105e427cd3 100644 --- a/contrib/examples/triggers/sample-trigger.yaml +++ b/contrib/examples/triggers/sample-trigger.yaml @@ -1,3 +1,10 @@ --- name: sample_trigger description: Sample trigger +payload_schema: + type: "object" + properties: + executed_at: + type: "string" + format: "date-time" + default: "2014-07-30 05:04:24.578325" From 31e7b27fcec710e19ffd9bab46804675c15a5c26 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 16 Jul 2018 11:09:41 +0200 Subject: [PATCH 25/27] Add changelog entry. --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 322415ac2b..1efce25f3a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,6 +19,9 @@ Added Changed ------- +* Update ``st2client/setup.py`` file to dynamically load requirements from + ``st2client/requirements.txt`` file. The code works with pip >= 6.0.0, although using pip 9.0.0 + or higher is strongly recommended. (improvement) #4209 * Update ``st2`` CLI to use a more sensible default terminal size for table formatting purposes if we are unable to retrieve terminal size using various system-specific approaches. From 6c4abc307946528da86e9db9a4d25e678472b2c7 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 16 Jul 2018 11:10:53 +0200 Subject: [PATCH 26/27] Pull runners in-requirements.txt change from https://github.com/StackStorm/st2/pull/4169. --- Makefile | 4 ++-- contrib/runners/action_chain_runner/in-requirements.txt | 0 contrib/runners/announcement_runner/in-requirements.txt | 0 contrib/runners/cloudslang_runner/in-requirements.txt | 0 contrib/runners/http_runner/in-requirements.txt | 0 contrib/runners/inquirer_runner/in-requirements.txt | 0 contrib/runners/local_runner/in-requirements.txt | 0 contrib/runners/mistral_v2/in-requirements.txt | 0 contrib/runners/noop_runner/in-requirements.txt | 0 contrib/runners/orchestra_runner/in-requirements.txt | 1 + contrib/runners/python_runner/in-requirements.txt | 0 contrib/runners/remote_runner/in-requirements.txt | 0 contrib/runners/windows_runner/in-requirements.txt | 0 13 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 contrib/runners/action_chain_runner/in-requirements.txt create mode 100644 contrib/runners/announcement_runner/in-requirements.txt create mode 100644 contrib/runners/cloudslang_runner/in-requirements.txt create mode 100644 contrib/runners/http_runner/in-requirements.txt create mode 100644 contrib/runners/inquirer_runner/in-requirements.txt create mode 100644 contrib/runners/local_runner/in-requirements.txt create mode 100644 contrib/runners/mistral_v2/in-requirements.txt create mode 100644 contrib/runners/noop_runner/in-requirements.txt create mode 100644 contrib/runners/orchestra_runner/in-requirements.txt create mode 100644 contrib/runners/python_runner/in-requirements.txt create mode 100644 contrib/runners/remote_runner/in-requirements.txt create mode 100644 contrib/runners/windows_runner/in-requirements.txt diff --git a/Makefile b/Makefile index 6226f4e3f2..0227bab172 100644 --- a/Makefile +++ b/Makefile @@ -316,10 +316,10 @@ requirements: virtualenv .sdist-requirements $(VIRTUALENV_DIR)/bin/pip install --upgrade "virtualenv==15.1.0" # Required for packs.install in dev envs. # Generate all requirements to support current CI pipeline. - $(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv -s st2*/in-requirements.txt -f fixed-requirements.txt -o requirements.txt + $(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv -s st2*/in-requirements.txt contrib/runners/*/in-requirements.txt -f fixed-requirements.txt -o requirements.txt # Generate finall requirements.txt file for each component - @for component in $(COMPONENTS); do\ + @for component in $(COMPONENTS_WITH_RUNNERS); do\ echo "==========================================================="; \ echo "Generating requirements.txt for" $$component; \ echo "==========================================================="; \ diff --git a/contrib/runners/action_chain_runner/in-requirements.txt b/contrib/runners/action_chain_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/announcement_runner/in-requirements.txt b/contrib/runners/announcement_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/cloudslang_runner/in-requirements.txt b/contrib/runners/cloudslang_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/http_runner/in-requirements.txt b/contrib/runners/http_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/inquirer_runner/in-requirements.txt b/contrib/runners/inquirer_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/local_runner/in-requirements.txt b/contrib/runners/local_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/mistral_v2/in-requirements.txt b/contrib/runners/mistral_v2/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/noop_runner/in-requirements.txt b/contrib/runners/noop_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/orchestra_runner/in-requirements.txt b/contrib/runners/orchestra_runner/in-requirements.txt new file mode 100644 index 0000000000..236a1e3040 --- /dev/null +++ b/contrib/runners/orchestra_runner/in-requirements.txt @@ -0,0 +1 @@ +git+https://github.com/StackStorm/orchestra.git#egg=orchestra diff --git a/contrib/runners/python_runner/in-requirements.txt b/contrib/runners/python_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/remote_runner/in-requirements.txt b/contrib/runners/remote_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/windows_runner/in-requirements.txt b/contrib/runners/windows_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 From 84ae753b9b440abfaf9728fff4606aff47341fb4 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Mon, 16 Jul 2018 11:15:34 +0200 Subject: [PATCH 27/27] Re-generate runner requirements.txt files. --- contrib/runners/action_chain_runner/requirements.txt | 2 ++ contrib/runners/announcement_runner/requirements.txt | 2 ++ contrib/runners/cloudslang_runner/requirements.txt | 2 ++ contrib/runners/http_runner/requirements.txt | 2 ++ contrib/runners/inquirer_runner/requirements.txt | 2 ++ contrib/runners/local_runner/requirements.txt | 2 ++ contrib/runners/mistral_v2/requirements.txt | 2 ++ contrib/runners/noop_runner/requirements.txt | 2 ++ contrib/runners/orchestra_runner/requirements.txt | 1 + contrib/runners/python_runner/requirements.txt | 2 ++ contrib/runners/remote_runner/requirements.txt | 2 ++ contrib/runners/windows_runner/requirements.txt | 2 ++ 12 files changed, 23 insertions(+) diff --git a/contrib/runners/action_chain_runner/requirements.txt b/contrib/runners/action_chain_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/action_chain_runner/requirements.txt +++ b/contrib/runners/action_chain_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/announcement_runner/requirements.txt b/contrib/runners/announcement_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/announcement_runner/requirements.txt +++ b/contrib/runners/announcement_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/cloudslang_runner/requirements.txt b/contrib/runners/cloudslang_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/cloudslang_runner/requirements.txt +++ b/contrib/runners/cloudslang_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/http_runner/requirements.txt b/contrib/runners/http_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/http_runner/requirements.txt +++ b/contrib/runners/http_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/inquirer_runner/requirements.txt b/contrib/runners/inquirer_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/inquirer_runner/requirements.txt +++ b/contrib/runners/inquirer_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/local_runner/requirements.txt b/contrib/runners/local_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/local_runner/requirements.txt +++ b/contrib/runners/local_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/mistral_v2/requirements.txt b/contrib/runners/mistral_v2/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/mistral_v2/requirements.txt +++ b/contrib/runners/mistral_v2/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/noop_runner/requirements.txt b/contrib/runners/noop_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/noop_runner/requirements.txt +++ b/contrib/runners/noop_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/orchestra_runner/requirements.txt b/contrib/runners/orchestra_runner/requirements.txt index 236a1e3040..5aa11564d7 100644 --- a/contrib/runners/orchestra_runner/requirements.txt +++ b/contrib/runners/orchestra_runner/requirements.txt @@ -1 +1,2 @@ +# Don't edit this file. It's generated automatically! git+https://github.com/StackStorm/orchestra.git#egg=orchestra diff --git a/contrib/runners/python_runner/requirements.txt b/contrib/runners/python_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/python_runner/requirements.txt +++ b/contrib/runners/python_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/remote_runner/requirements.txt b/contrib/runners/remote_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/remote_runner/requirements.txt +++ b/contrib/runners/remote_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/windows_runner/requirements.txt b/contrib/runners/windows_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/windows_runner/requirements.txt +++ b/contrib/runners/windows_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! +