Fix pylint on runner Python modules

Makefile

@@ -7,7 +7,10 @@ BINARIES := bin
 
 # All components are prefixed by st2
 COMPONENTS := $(wildcard st2*)
-COMPONENTS += $(wildcard contrib/runners/*)
+COMPONENTS_RUNNERS := $(wildcard contrib/runners/*)
+
+COMPONENTS_WITH_RUNNERS := $(wildcard st2*)
+COMPONENTS_WITH_RUNNERS += $(wildcard contrib/runners/*)
 
 # Components that implement a component-controlled test-runner. These components provide an
 # in-component Makefile. (Temporary fix until I can generalize the pecan unittest setup. -mar)
@@ -18,8 +21,8 @@ COMPONENT_SPECIFIC_TESTS := st2tests st2client.egg-info
 space_char :=
 space_char +=
 comma := ,
-COMPONENT_PYTHONPATH = $(subst $(space_char),:,$(realpath $(COMPONENTS)))
-COMPONENTS_TEST := $(foreach component,$(filter-out $(COMPONENT_SPECIFIC_TESTS),$(COMPONENTS)),$(component))
+COMPONENT_PYTHONPATH = $(subst $(space_char),:,$(realpath $(COMPONENTS_WITH_RUNNERS)))
+COMPONENTS_TEST := $(foreach component,$(filter-out $(COMPONENT_SPECIFIC_TESTS),$(COMPONENTS_WITH_RUNNERS)),$(component))
 COMPONENTS_TEST_COMMA := $(subst $(space_char),$(comma),$(COMPONENTS_TEST))
 
 PYTHON_TARGET := 2.7
@@ -86,11 +89,19 @@ configgen:
 		echo "==========================================================="; \
 		. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models --load-plugins=pylint_plugins.db_models $$component/$$component || exit 1; \
 	done
+	# Lint runner modules and packages
+	@for component in $(COMPONENTS_RUNNERS); do\
+		echo "==========================================================="; \
+		echo "Running pylint on" $$component; \
+		echo "==========================================================="; \
+		. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models --load-plugins=pylint_plugins.db_models $$component/*.py || exit 1; \
+	done
 	# Lint Python pack management actions
-	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/packs/actions/ || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/packs/actions/*.py || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/packs/actions/*/*.py || exit 1;
 	# Lint other packs
-	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/linux || exit 1;
-	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/chatops || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/linux/*/*.py || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models contrib/chatops/*/*.py || exit 1;
 	# Lint Python scripts
 	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models scripts/*.py || exit 1;
 	. $(VIRTUALENV_DIR)/bin/activate; pylint -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models tools/*.py || exit 1;
@@ -105,6 +116,7 @@ flake8: requirements .flake8
 	@echo "==================== flake ===================="
 	@echo
 	. $(VIRTUALENV_DIR)/bin/activate; flake8 --config ./lint-configs/python/.flake8 $(COMPONENTS)
+	. $(VIRTUALENV_DIR)/bin/activate; flake8 --config ./lint-configs/python/.flake8 $(COMPONENTS_RUNNERS)
 	. $(VIRTUALENV_DIR)/bin/activate; flake8 --config ./lint-configs/python/.flake8 contrib/packs/actions/
 	. $(VIRTUALENV_DIR)/bin/activate; flake8 --config ./lint-configs/python/.flake8 contrib/linux
 	. $(VIRTUALENV_DIR)/bin/activate; flake8 --config ./lint-configs/python/.flake8 contrib/chatops/
@@ -120,7 +132,7 @@ bandit: requirements .bandit
 	@echo
 	@echo "==================== bandit ===================="
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; bandit -r $(COMPONENTS) -lll
+	. $(VIRTUALENV_DIR)/bin/activate; bandit -r $(COMPONENTS_WITH_RUNNERS) -lll
 
 .PHONY: lint
 lint: requirements .lint
@@ -140,7 +152,7 @@ compile:
 .PHONY: .cleanpycs
 .cleanpycs:
 	@echo "Removing all .pyc files"
-	find $(COMPONENTS)  -name \*.pyc -type f -print0 | xargs -0 -I {} rm {}
+	find $(COMPONENTS_WITH_RUNNERS)  -name \*.pyc -type f -print0 | xargs -0 -I {} rm {}
 
 .PHONY: .st2client-dependencies-check
 .st2client-dependencies-check:

contrib/packs/actions/pack_mgmt/download.py

@@ -138,7 +138,7 @@ def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'):
 
         # We're trying to figure out which branch the ref is actually on,
         # since there's no direct way to check for this in git-python.
-        branches = repo.git.branch('-a', '--contains', gitref.hexsha)
+        branches = repo.git.branch('-a', '--contains', gitref.hexsha)  # pylint: disable=no-member
         branches = branches.replace('*', '').split()
 
         if active_branch.name not in branches or use_branch:
@@ -149,8 +149,8 @@ def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'):
         else:
             branch = repo.active_branch.name
 
-        repo.git.checkout(gitref.hexsha)
-        repo.git.branch('-f', branch, gitref.hexsha)
+        repo.git.checkout(gitref.hexsha)  # pylint: disable=no-member
+        repo.git.branch('-f', branch, gitref.hexsha)  # pylint: disable=no-member
         repo.git.checkout(branch)
 
         return temp_dir

st2api/st2api/controllers/resource.py

@@ -51,6 +51,7 @@ def split_id_value(value):
 
     return split
 
+
 DEFAULT_FILTER_TRANSFORM_FUNCTIONS = {
     # Support for filtering on multiple ids when a commona delimited string is provided
     # (e.g. ?id=1,2,3)

st2client/st2client/commands/action.py

@@ -105,6 +105,7 @@ def format_parameters(value):
 
     return value
 
+
 # String for indenting etc.
 WF_PREFIX = '+ '
 NON_WF_PREFIX = '  '

st2common/bin/migrations/v1.5/st2-migrate-datastore-to-include-scope-secret.py

@@ -67,5 +67,6 @@ def main():
     db_teardown()
     sys.exit(exit_code)
 
+
 if __name__ == '__main__':
     main()

st2common/bin/migrations/v2.1/st2-migrate-datastore-scopes.py

@@ -71,5 +71,6 @@ def main():
     db_teardown()
     sys.exit(exit_code)
 
+
 if __name__ == '__main__':
     main()

st2common/st2common/content/bootstrap.py

@@ -74,6 +74,8 @@ def register_opts():
         cfg.CONF.register_cli_opts(content_opts, group='register')
     except:
         sys.stderr.write('Failed registering opts.\n')
+
+
 register_opts()
 
 

st2common/st2common/log.py

@@ -161,6 +161,7 @@ def _audit(logger, msg, *args, **kwargs):
     if logger.isEnabledFor(logging.AUDIT):
         logger._log(logging.AUDIT, msg, args, **kwargs)
 
+
 logging.Logger.audit = _audit
 
 

st2common/st2common/models/db/action.py

@@ -95,6 +95,7 @@ def is_workflow(self):
         # pylint: disable=unsubscriptable-object
         return self.runner_type['name'] in WORKFLOW_RUNNER_TYPES
 
+
 # specialized access objects
 action_access = MongoDBAccess(ActionDB)
 

st2common/st2common/models/db/executionstate.py

@@ -48,6 +48,7 @@ class ActionExecutionStateDB(stormbase.StormFoundationDB):
         'indexes': ['query_module']
     }
 
+
 # specialized access objects
 actionexecstate_access = MongoDBAccess(ActionExecutionStateDB)
 

st2common/st2common/models/db/marker.py

@@ -52,4 +52,5 @@ class DumperMarkerDB(MarkerDB):
     """
     pass
 
+
 MODELS = [MarkerDB, DumperMarkerDB]

st2common/st2common/models/db/rbac.py

@@ -75,6 +75,7 @@ class PermissionGrantDB(stormbase.StormFoundationDB):
     resource_type = me.StringField(required=False)
     permission_types = me.ListField(field=me.StringField())
 
+
 # Specialized access objects
 role_access = MongoDBAccess(RoleDB)
 user_role_assignment_access = MongoDBAccess(UserRoleAssignmentDB)

st2common/st2common/models/db/rule.py

@@ -98,6 +98,7 @@ def __init__(self, *args, **values):
         self.ref = self.get_reference().ref
         self.uid = self.get_uid()
 
+
 rule_access = MongoDBAccess(RuleDB)
 rule_type_access = MongoDBAccess(RuleTypeDB)
 

st2common/st2common/models/db/rule_enforcement.py

@@ -65,6 +65,7 @@ def get_uid(self):
         uid = [self.RESOURCE_TYPE, str(self.id)]
         return ':'.join(uid)
 
+
 rule_enforcement_access = MongoDBAccess(RuleEnforcementDB)
 
 MODELS = [RuleEnforcementDB]

st2common/st2common/models/db/runner.py

@@ -67,6 +67,7 @@ def __init__(self, *args, **values):
         super(RunnerTypeDB, self).__init__(*args, **values)
         self.uid = self.get_uid()
 
+
 # specialized access objects
 runnertype_access = MongoDBAccess(RunnerTypeDB)
 

st2common/st2common/models/db/sensor.py

@@ -59,6 +59,7 @@ def __init__(self, *args, **values):
         self.ref = self.get_reference().ref
         self.uid = self.get_uid()
 
+
 sensor_type_access = MongoDBAccess(SensorTypeDB)
 
 MODELS = [SensorTypeDB]

st2common/st2common/models/db/trace.py

@@ -84,6 +84,7 @@ class TraceDB(stormbase.StormFoundationDB):
         ]
     }
 
+
 # specialized access objects
 trace_access = MongoDBAccess(TraceDB)
 

st2common/st2common/operators.py

@@ -185,6 +185,7 @@ def exists(value, criteria_pattern):
 def nexists(value, criteria_pattern):
     return value is None
 
+
 # operator match strings
 MATCH_WILDCARD = 'matchwildcard'
 MATCH_REGEX = 'matchregex'

st2common/tests/unit/test_db_model_uids.py

@@ -51,11 +51,11 @@ def test_get_uid(self):
         self.assertTrue(trigger_db.get_uid().startswith('trigger:tpack:tname:'))
 
         # Verify that same set of parameters always results in the same hash
-        parameters = {'a': 1, 'b': 2, 'c': [1, 2, 3], 'd': {'g': 1, 'h': 2}, 'b': u'unicode'}
+        parameters = {'a': 1, 'b': 'unicode', 'c': [1, 2, 3], 'd': {'g': 1, 'h': 2}}
         paramers_hash = json.dumps(parameters, sort_keys=True)
         paramers_hash = hashlib.md5(paramers_hash).hexdigest()
 
-        parameters = {'a': 1, 'b': 2, 'c': [1, 2, 3], 'b': u'unicode', 'd': {'g': 1, 'h': 2}}
+        parameters = {'a': 1, 'b': 'unicode', 'c': [1, 2, 3], 'd': {'g': 1, 'h': 2}}
         trigger_db = TriggerDB(name='tname', pack='tpack', parameters=parameters)
         self.assertEqual(trigger_db.get_uid(), 'trigger:tpack:tname:%s' % (paramers_hash))
 

st2reactor/st2reactor/garbage_collector/config.py

@@ -63,4 +63,5 @@ def _register_garbage_collector_opts():
     ]
     CONF.register_opts(ttl_opts, group='garbagecollector')
 
+
 register_opts()

test-requirements.txt

@@ -1,12 +1,12 @@
 coverage
 pep8==1.7.0
-flake8==3.0.4
-astroid==1.4.5
-pylint==1.5.5
+flake8==3.3.0
+astroid==1.4.9
+pylint==1.6.5
 pylint-plugin-utils>=0.2.3,<0.3
 bandit>=1.0.1,<1.1
 ipython
-mock>=1.0
+mock==2.0.0
 nose>=1.3.7
 tabulate
 unittest2

tools/diff-db-disk.py

@@ -270,5 +270,6 @@ def main():
     # Disconnect from db.
     db_teardown()
 
+
 if __name__ == '__main__':
     main()

tools/migrate_rules_to_include_pack.py

@@ -57,6 +57,7 @@ class RuleDB(stormbase.StormFoundationDB, stormbase.TagsMixin,
             'indexes': stormbase.TagsMixin.get_indices()
         }
 
+
 # specialized access objects
 rule_access_with_pack = MongoDBAccess(Migration.RuleDB)
 
@@ -82,6 +83,7 @@ class RuleDB(stormbase.StormBaseDB, stormbase.TagsMixin):
         'indexes': stormbase.TagsMixin.get_indices()
     }
 
+
 rule_access_without_pack = MongoDBAccess(RuleDB)
 
 

tools/visualize_action_chain.py

@@ -107,6 +107,7 @@ def main(metadata_path, output_path, print_source=False):
 
     print('Graph saved at %s' % (output_path + '.png'))
 
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Action chain visualization')
     parser.add_argument('--metadata-path', action='store', required=True,