Upgrade various dependencies

Makefile

@@ -7,7 +7,10 @@ BINARIES := bin
 
 # All components are prefixed by st2
 COMPONENTS := $(wildcard st2*)
-COMPONENTS += $(wildcard contrib/runners/*)
+COMPONENTS_RUNNERS := $(wildcard contrib/runners/*)
+
+COMPONENTS_WITH_RUNNERS := $(wildcard st2*)
+COMPONENTS_WITH_RUNNERS += $(wildcard contrib/runners/*)
 
 # Components that implement a component-controlled test-runner. These components provide an
 # in-component Makefile. (Temporary fix until I can generalize the pecan unittest setup. -mar)
@@ -18,8 +21,8 @@ COMPONENT_SPECIFIC_TESTS := st2tests st2client.egg-info
 space_char :=
 space_char +=
 comma := ,
-COMPONENT_PYTHONPATH = $(subst $(space_char),:,$(realpath $(COMPONENTS)))
-COMPONENTS_TEST := $(foreach component,$(filter-out $(COMPONENT_SPECIFIC_TESTS),$(COMPONENTS)),$(component))
+COMPONENT_PYTHONPATH = $(subst $(space_char),:,$(realpath $(COMPONENTS_WITH_RUNNERS)))
+COMPONENTS_TEST := $(foreach component,$(filter-out $(COMPONENT_SPECIFIC_TESTS),$(COMPONENTS_WITH_RUNNERS)),$(component))
 COMPONENTS_TEST_COMMA := $(subst $(space_char),$(comma),$(COMPONENTS_TEST))
 
 PYTHON_TARGET := 2.7
@@ -86,11 +89,19 @@ configgen:
 		echo "==========================================================="; \
 		. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models --load-plugins=pylint_plugins.db_models $$component/$$component || exit 1; \
 	done
+	# Lint runner modules and packages
+	@for component in $(COMPONENTS_RUNNERS); do\
+		echo "==========================================================="; \
+		echo "Running pylint on" $$component; \
+		echo "==========================================================="; \
+		. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models --load-plugins=pylint_plugins.db_models $$component/*.py || exit 1; \
+	done
 	# Lint Python pack management actions
-	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/packs/actions/ || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/packs/actions/*.py || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/packs/actions/*/*.py || exit 1;
 	# Lint other packs
-	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/linux || exit 1;
-	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/chatops || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/linux/*/*.py || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/chatops/*/*.py || exit 1;
 	# Lint Python scripts
 	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models scripts/*.py || exit 1;
 	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models tools/*.py || exit 1;
@@ -105,6 +116,7 @@ flake8: requirements .flake8
 	@echo "==================== flake ===================="
 	@echo
 	. $(VIRTUALENV_DIR)/bin/activate; flake8 --config ./lint-configs/python/.flake8 $(COMPONENTS)
+	. $(VIRTUALENV_DIR)/bin/activate; flake8 --config ./lint-configs/python/.flake8 $(COMPONENTS_RUNNERS)
 	. $(VIRTUALENV_DIR)/bin/activate; flake8 --config ./lint-configs/python/.flake8 contrib/packs/actions/
 	. $(VIRTUALENV_DIR)/bin/activate; flake8 --config ./lint-configs/python/.flake8 contrib/linux
 	. $(VIRTUALENV_DIR)/bin/activate; flake8 --config ./lint-configs/python/.flake8 contrib/chatops/
@@ -120,7 +132,7 @@ bandit: requirements .bandit
 	@echo
 	@echo "==================== bandit ===================="
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; bandit -r $(COMPONENTS) -lll
+	. $(VIRTUALENV_DIR)/bin/activate; bandit -r $(COMPONENTS_WITH_RUNNERS) -lll
 
 .PHONY: lint
 lint: requirements .lint
@@ -140,7 +152,7 @@ compile:
 .PHONY: .cleanpycs
 .cleanpycs:
 	@echo "Removing all .pyc files"
-	find $(COMPONENTS)  -name \*.pyc -type f -print0 | xargs -0 -I {} rm {}
+	find $(COMPONENTS_WITH_RUNNERS)  -name \*.pyc -type f -print0 | xargs -0 -I {} rm {}
 
 .PHONY: .st2client-dependencies-check
 .st2client-dependencies-check:

contrib/packs/actions/pack_mgmt/download.py

@@ -138,7 +138,7 @@ def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'):
 
         # We're trying to figure out which branch the ref is actually on,
         # since there's no direct way to check for this in git-python.
-        branches = repo.git.branch('-a', '--contains', gitref.hexsha)
+        branches = repo.git.branch('-a', '--contains', gitref.hexsha)  # pylint: disable=no-member
         branches = branches.replace('*', '').split()
 
         if active_branch.name not in branches or use_branch:
@@ -149,8 +149,8 @@ def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'):
         else:
             branch = repo.active_branch.name
 
-        repo.git.checkout(gitref.hexsha)
-        repo.git.branch('-f', branch, gitref.hexsha)
+        repo.git.checkout(gitref.hexsha)  # pylint: disable=no-member
+        repo.git.branch('-f', branch, gitref.hexsha)  # pylint: disable=no-member
         repo.git.checkout(branch)
 
         return temp_dir

fixed-requirements.txt

@@ -1,37 +1,38 @@
 # Packages versions fixed for the whole st2 stack
 # Note: greenlet is used by eventlet
-greenlet>=0.4.10,<0.5
-eventlet>=0.18.4,<0.19
+greenlet>=0.4.12,<0.5
+# Note: Eventlet 0.20.0 removes select.poll() which we rely on
+eventlet==0.19.0
 gunicorn==19.6.0
-kombu==3.0.37
+kombu==4.0.2
 # Note: amqp is used by kombu
-amqp==1.4.9
+amqp==2.1.4
 oslo.config>=1.12.1,<1.13
 oslo.utils<3.1.0
 six==1.10.0
-pyyaml>=3.11,<4.0
+pyyaml>=3.12,<4.0
 requests[security]>=2.11.1,<2.12
-apscheduler==3.3.0
+apscheduler==3.3.1
 gitpython==2.1.0
-jsonschema>=2.5.0,<2.6
+jsonschema==2.6.0
 mongoengine==0.11.0
 pymongo==3.4.0
 passlib==1.6.5
 lockfile>=0.10.2,<0.11
 python-gnupg==0.3.9
-jsonpath-rw>=1.3.0
-pyinotify>=0.9.5,<=0.10
-semver==2.7.2
+jsonpath-rw==1.4.0
+pyinotify==0.9.6
+semver==2.7.5
 # Note: Any version after 1.20.0 causes our tests to hand / dead-lock with zake
 # and file driver
 tooz==1.20.0
 stevedore>=1.7.0,<1.8
-paramiko>=2.0.2,<2.1
-networkx==1.10
+paramiko>=2.1.1,<2.2
+networkx==1.11
 python-keyczar==0.716
-retrying>=1.3,<1.4
+retrying>=1.3.3,<1.4
 # Note: We use latest version of virtualenv which uses pip 9.0
 virtualenv==15.1.0
-sseclient==0.0.12
+sseclient==0.0.14
 python-editor==1.0.1
 prompt-toolkit==1.0.7

requirements.txt

@@ -1,8 +1,8 @@
 # Don't edit this file. It's generated automatically!
-apscheduler==3.3.0
+apscheduler==3.3.1
 argcomplete
 bcrypt
-eventlet<0.19,>=0.18.4
+eventlet==0.19.0
 git+https://github.com/Kami/entrypoints.git@dont_use_backports#egg=entrypoints
 git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper
 git+https://github.com/StackStorm/pecan.git@st2-patched#egg=pecan
@@ -12,31 +12,31 @@ gitpython==2.1.0
 gunicorn==19.6.0
 ipaddr
 jinja2
-jsonpath-rw>=1.3.0
-jsonschema<2.6,>=2.5.0
-kombu==3.0.37
+jsonpath-rw==1.4.0
+jsonschema==2.6.0
+kombu==4.0.2
 lockfile<0.11,>=0.10.2
 mongoengine==0.11.0
-networkx==1.10
+networkx==1.11
 oslo.config<1.13,>=1.12.1
 oslo.utils<3.1.0
-paramiko<2.1,>=2.0.2
+paramiko<2.2,>=2.1.1
 passlib==1.6.5
 prettytable
 prompt-toolkit==1.0.7
-pyinotify<=0.10,>=0.9.5
+pyinotify==0.9.6
 pymongo==3.4.0
 python-dateutil
 python-editor==1.0.1
 python-gnupg==0.3.9
 python-json-logger
 python-keyczar==0.716
 pytz
-pyyaml<4.0,>=3.11
+pyyaml<4.0,>=3.12
 requests[security]<2.12,>=2.11.1
-retrying<1.4,>=1.3
-semver==2.7.2
+retrying<1.4,>=1.3.3
+semver==2.7.5
 six==1.10.0
-sseclient==0.0.12
+sseclient==0.0.14
 stevedore<1.8,>=1.7.0
 tooz==1.20.0

st2api/st2api/controllers/resource.py

@@ -51,6 +51,7 @@ def split_id_value(value):
 
     return split
 
+
 DEFAULT_FILTER_TRANSFORM_FUNCTIONS = {
     # Support for filtering on multiple ids when a commona delimited string is provided
     # (e.g. ?id=1,2,3)

st2client/st2client/commands/action.py

@@ -105,6 +105,7 @@ def format_parameters(value):
 
     return value
 
+
 # String for indenting etc.
 WF_PREFIX = '+ '
 NON_WF_PREFIX = '  '

st2common/bin/migrations/v1.5/st2-migrate-datastore-to-include-scope-secret.py

@@ -67,5 +67,6 @@ def main():
     db_teardown()
     sys.exit(exit_code)
 
+
 if __name__ == '__main__':
     main()

st2common/bin/migrations/v2.1/st2-migrate-datastore-scopes.py

@@ -71,5 +71,6 @@ def main():
     db_teardown()
     sys.exit(exit_code)
 
+
 if __name__ == '__main__':
     main()

st2common/st2common/bootstrap/actionsregistrar.py

@@ -131,17 +131,7 @@ def _register_action(self, pack, action):
             action_api.validate()
         except jsonschema.ValidationError as e:
             # We throw a more user-friendly exception on invalid parameter name
-            msg = str(e)
-
-            is_invalid_parameter_name = 'Additional properties are not allowed' in msg
-            is_invalid_parameter_name &= 'in schema[\'properties\'][\'parameters\']' in msg
-
-            if is_invalid_parameter_name:
-                parameter_name = re.search('\'(.+?)\' was unexpected', msg).groups()[0]
-                new_msg = ('Parameter name "%s" is invalid. Valid characters for parameter name '
-                           'are [a-zA-Z0-0_].' % (parameter_name))
-                new_msg += '\n\n' + msg
-                raise jsonschema.ValidationError(new_msg)
+            e = self._get_error_for_invalid_parameter_name(e=e)
             raise e
 
         action_validator.validate_action(action_api)
@@ -184,6 +174,37 @@ def _register_actions_from_pack(self, pack, actions):
 
         return registered_count
 
+    def _get_error_for_invalid_parameter_name(self, e):
+        """
+        Return a more user-friendly exception for scenarios where action parameter name fails
+        validation (e.g. it contains invalid characters or similar).
+        """
+        msg = str(e)
+
+        is_invalid_parameter_name_error = 'Additional properties are not allowed' in msg
+        is_invalid_parameter_name_error &= 'in schema[\'properties\'][\'parameters\']' in msg
+
+        # New error notation introduced in jsonschema v2.6.0
+        is_invalid_parameter_name_error |= 'does not match any of the regexes:' in msg
+
+        if is_invalid_parameter_name_error:
+            parameter_name_1 = re.search('\'(.+?)\' was unexpected', msg)
+            parameter_name_2 = re.search('\'(.+?)\' does not match any of the regexes:', msg)
+
+            if parameter_name_1:
+                parameter_name = parameter_name_1.groups()[0]
+            elif parameter_name_2:
+                parameter_name = parameter_name_2.groups()[0]
+            else:
+                parameter_name = 'unknown'
+
+            new_msg = ('Parameter name "%s" is invalid. Valid characters for parameter name '
+                       'are [a-zA-Z0-0_].' % (parameter_name))
+            new_msg += '\n\n' + msg
+            return jsonschema.ValidationError(new_msg)
+
+        return e
+
 
 def register_actions(packs_base_paths=None, pack_dir=None, use_pack_cache=True,
                      fail_on_failure=False):

st2common/st2common/content/bootstrap.py

@@ -74,6 +74,8 @@ def register_opts():
         cfg.CONF.register_cli_opts(content_opts, group='register')
     except:
         sys.stderr.write('Failed registering opts.\n')
+
+
 register_opts()
 
 

st2common/st2common/log.py

@@ -161,6 +161,7 @@ def _audit(logger, msg, *args, **kwargs):
     if logger.isEnabledFor(logging.AUDIT):
         logger._log(logging.AUDIT, msg, args, **kwargs)
 
+
 logging.Logger.audit = _audit
 
 

st2common/st2common/models/db/action.py

@@ -95,6 +95,7 @@ def is_workflow(self):
         # pylint: disable=unsubscriptable-object
         return self.runner_type['name'] in WORKFLOW_RUNNER_TYPES
 
+
 # specialized access objects
 action_access = MongoDBAccess(ActionDB)
 

st2common/st2common/models/db/executionstate.py

@@ -48,6 +48,7 @@ class ActionExecutionStateDB(stormbase.StormFoundationDB):
         'indexes': ['query_module']
     }
 
+
 # specialized access objects
 actionexecstate_access = MongoDBAccess(ActionExecutionStateDB)
 

st2common/st2common/models/db/marker.py

@@ -52,4 +52,5 @@ class DumperMarkerDB(MarkerDB):
     """
     pass
 
+
 MODELS = [MarkerDB, DumperMarkerDB]

st2common/st2common/models/db/rbac.py

@@ -75,6 +75,7 @@ class PermissionGrantDB(stormbase.StormFoundationDB):
     resource_type = me.StringField(required=False)
     permission_types = me.ListField(field=me.StringField())
 
+
 # Specialized access objects
 role_access = MongoDBAccess(RoleDB)
 user_role_assignment_access = MongoDBAccess(UserRoleAssignmentDB)

st2common/st2common/models/db/rule.py

@@ -98,6 +98,7 @@ def __init__(self, *args, **values):
         self.ref = self.get_reference().ref
         self.uid = self.get_uid()
 
+
 rule_access = MongoDBAccess(RuleDB)
 rule_type_access = MongoDBAccess(RuleTypeDB)
 

st2common/st2common/models/db/rule_enforcement.py

@@ -65,6 +65,7 @@ def get_uid(self):
         uid = [self.RESOURCE_TYPE, str(self.id)]
         return ':'.join(uid)
 
+
 rule_enforcement_access = MongoDBAccess(RuleEnforcementDB)
 
 MODELS = [RuleEnforcementDB]

st2common/st2common/models/db/runner.py

@@ -67,6 +67,7 @@ def __init__(self, *args, **values):
         super(RunnerTypeDB, self).__init__(*args, **values)
         self.uid = self.get_uid()
 
+
 # specialized access objects
 runnertype_access = MongoDBAccess(RunnerTypeDB)
 

st2common/st2common/models/db/sensor.py

@@ -59,6 +59,7 @@ def __init__(self, *args, **values):
         self.ref = self.get_reference().ref
         self.uid = self.get_uid()
 
+
 sensor_type_access = MongoDBAccess(SensorTypeDB)
 
 MODELS = [SensorTypeDB]

st2common/st2common/models/db/trace.py

@@ -84,6 +84,7 @@ class TraceDB(stormbase.StormFoundationDB):
         ]
     }
 
+
 # specialized access objects
 trace_access = MongoDBAccess(TraceDB)
 

st2common/st2common/operators.py

@@ -185,6 +185,7 @@ def exists(value, criteria_pattern):
 def nexists(value, criteria_pattern):
     return value is None
 
+
 # operator match strings
 MATCH_WILDCARD = 'matchwildcard'
 MATCH_REGEX = 'matchregex'

st2common/tests/unit/test_db_model_uids.py

@@ -51,11 +51,11 @@ def test_get_uid(self):
         self.assertTrue(trigger_db.get_uid().startswith('trigger:tpack:tname:'))
 
         # Verify that same set of parameters always results in the same hash
-        parameters = {'a': 1, 'b': 2, 'c': [1, 2, 3], 'd': {'g': 1, 'h': 2}, 'b': u'unicode'}
+        parameters = {'a': 1, 'b': 'unicode', 'c': [1, 2, 3], 'd': {'g': 1, 'h': 2}}
         paramers_hash = json.dumps(parameters, sort_keys=True)
         paramers_hash = hashlib.md5(paramers_hash).hexdigest()
 
-        parameters = {'a': 1, 'b': 2, 'c': [1, 2, 3], 'b': u'unicode', 'd': {'g': 1, 'h': 2}}
+        parameters = {'a': 1, 'b': 'unicode', 'c': [1, 2, 3], 'd': {'g': 1, 'h': 2}}
         trigger_db = TriggerDB(name='tname', pack='tpack', parameters=parameters)
         self.assertEqual(trigger_db.get_uid(), 'trigger:tpack:tname:%s' % (paramers_hash))
 

st2reactor/st2reactor/garbage_collector/config.py

@@ -63,4 +63,5 @@ def _register_garbage_collector_opts():
     ]
     CONF.register_opts(ttl_opts, group='garbagecollector')
 
+
 register_opts()