Don't print statsd log messages unless debug / verbose mode is used

st2common/st2common/log.py

@@ -40,7 +40,10 @@
     'FormatNamedFileHandler',
     'ConfigurableSyslogHandler',
 
-    'LoggingStream'
+    'LoggingStream',
+
+    'ignore_lib2to3_log_messages',
+    'ignore_statsd_log_messages'
 ]
 
 logging.AUDIT = logging.CRITICAL + 10
@@ -202,3 +205,36 @@ def setup(config_file, redirect_stderr=True, excludes=None, disable_existing_log
         sys.stderr.write('ERROR: %s' % (msg))
 
         raise exc_cls(six.text_type(msg))
+
+
+def ignore_lib2to3_log_messages():
+    """
+    Work around to ignore "Generating grammar tables from" log messages which are logged under
+    INFO by default by libraries such as networkx which use 2to3.
+    """
+    import lib2to3.pgen2.driver
+
+    class MockLoggingModule(object):
+        def getLogger(self, *args, **kwargs):
+            return logging.getLogger('lib2to3')
+
+    lib2to3.pgen2.driver.logging = MockLoggingModule()
+    logging.getLogger('lib2to3').setLevel(logging.ERROR)
+
+
+def ignore_statsd_log_messages():
+    """
+    By default statsd client logs all the operations under INFO and that causes a lot of noise.
+
+    This pull request silences all the statsd INFO log messages.
+    """
+    import statsd.connection
+    import statsd.client
+
+    class MockLoggingModule(object):
+        def getLogger(self, *args, **kwargs):
+            return logging.getLogger('statsd')
+
+    statsd.connection.logging = MockLoggingModule()
+    statsd.client.logging = MockLoggingModule()
+    logging.getLogger('statsd').setLevel(logging.ERROR)

st2common/st2common/script_setup.py

@@ -25,7 +25,6 @@
 import logging as stdlib_logging
 
 from oslo_config import cfg
-
 from st2common import log as logging
 from st2common.database_setup import db_setup
 from st2common.database_setup import db_teardown
@@ -87,6 +86,12 @@ def setup(config, setup_db=True, register_mq_exchanges=True,
         for handler in handlers:
             handler.addFilter(LogLevelFilter(log_levels=exclude_log_levels))
 
+        # NOTE: statsd logger logs everything by default under INFO so we ignore those log
+        # messages unless verbose / debug mode is used
+        logging.ignore_statsd_log_messages()
+
+    logging.ignore_lib2to3_log_messages()
+
     # All other setup code which requires config to be parsed and logging to be correctly setup
     if setup_db:
         db_setup()

st2common/st2common/service_setup.py

@@ -98,6 +98,8 @@ def setup(service, config, setup_db=True, register_mq_exchanges=True,
 
     LOG.debug('Using logging config: %s', logging_config_path)
 
+    is_debug_enabled = (cfg.CONF.debug or cfg.CONF.system.debug)
+
     try:
         logging.setup(logging_config_path, redirect_stderr=cfg.CONF.log.redirect_stderr,
                       excludes=cfg.CONF.log.excludes)
@@ -110,14 +112,20 @@ def setup(service, config, setup_db=True, register_mq_exchanges=True,
         else:
             raise e
 
-    if cfg.CONF.debug or cfg.CONF.system.debug:
+    if not is_debug_enabled:
+        # NOTE: statsd logger logs everything by default under INFO so we ignore those log
+        # messages unless verbose / debug mode is used
+        logging.ignore_statsd_log_messages()
+
+    logging.ignore_lib2to3_log_messages()
+
+    if is_debug_enabled:
         enable_debugging()
 
     if cfg.CONF.profile:
         enable_profiling()
 
-    # All other setup which requires config to be parsed and logging to
-    # be correctly setup.
+    # All other setup which requires config to be parsed and logging to be correctly setup.
     if setup_db:
         db_setup()