Additional code metrics instrumentation

CHANGELOG.rst

@@ -83,6 +83,12 @@ Changed
   ``st2rulesengine`` service. This would make such issues very hard to troubleshoot because only
   way to find out about this failure would be to inspect the ``st2rulesengine`` service logs.
   (improvement) #4231
+* Improve code metric instrumentation and instrument code and various services with more metrics.
+  (improvement) #4310
+* Add new ``metrics.prefix`` config option. With this option user can specify an optional prefix
+  which is prepended to each metric key (name). This comes handy in scenarios where user wants to
+  submit metrics from multiple environments / deployments (e.g. testing, staging, dev) to the same
+  backend instance. (improvement) #4310
 
 Fixed
 ~~~~~

conf/metrics/carbon/storage-aggregation.conf

@@ -0,0 +1,52 @@
+# Aggregation methods for whisper files. Entries are scanned in order,
+# and first match wins. This file is scanned for changes every 60 seconds
+#
+#  [name]
+#  pattern = <regex>
+#  xFilesFactor = <float between 0 and 1>
+#  aggregationMethod = <average|sum|last|max|min>
+#
+#  name: Arbitrary unique name for the rule
+#  pattern: Regex pattern to match against the metric name
+#  xFilesFactor: Ratio of valid data points required for aggregation to the next retention to occur
+#  aggregationMethod: function to apply to data points for aggregation
+#
+[min]
+pattern = \.min$
+xFilesFactor = 0.1
+aggregationMethod = min
+
+[max]
+pattern = \.max$
+xFilesFactor = 0.1
+aggregationMethod = max
+
+[count]
+pattern = \.count$
+xFilesFactor = 0
+aggregationMethod = sum
+
+[count_legacy]
+pattern = ^stats_counts.*
+xFilesFactor = 0
+aggregationMethod = sum
+
+[lower]
+pattern = \.lower(_\d+)?$
+xFilesFactor = 0.1
+aggregationMethod = min
+
+[upper]
+pattern = \.upper(_\d+)?$
+xFilesFactor = 0.1
+aggregationMethod = max
+
+[sum]
+pattern = \.sum$
+xFilesFactor = 0
+aggregationMethod = sum
+
+[default_average]
+pattern = .*
+xFilesFactor = 0.5
+aggregationMethod = average

conf/metrics/carbon/storage-schemas.conf

@@ -0,0 +1,20 @@
+# Schema definitions for Whisper files. Entries are scanned in order,
+# and first match wins. This file is scanned for changes every 60 seconds.
+#
+#  [name]
+#  pattern = regex
+#  retentions = timePerPoint:timeToStore, timePerPoint:timeToStore, ...
+
+# Carbon's internal metrics. This entry should match what is specified in
+# CARBON_METRIC_PREFIX and CARBON_METRIC_INTERVAL settings
+[stats]
+pattern = ^stats.*
+retentions = 10s:1d,1m:7d,10m:1y
+
+[carbon]
+pattern = ^carbon\.
+retentions = 60:90d
+
+[default_1min_for_1day]
+pattern = .*
+retentions = 60s:1d

conf/metrics/statsd/localConfig.js

@@ -0,0 +1,19 @@
+// Sample statsd config for usage with metrics instrumentation
+{
+  // IP and port of a local or remote graphite instance to which statsd will
+  // submit metrics
+  graphiteHost: "127.0.0.1",
+  graphitePort: 2003,
+
+  // statsd listen IP and port
+  address: "0.0.0.0",
+  port: 8125,
+
+  // Enable debug mode for easier debugging, disable in production
+  debug: true,
+
+  // Disable legacy name prefix
+  graphite: {
+    legacyNamespace: false
+  }
+}

conf/st2.conf.sample

@@ -183,6 +183,8 @@ cluster_urls =  # comma separated list allowed here.
 [metrics]
 # Destination server to connect to if driver requires connection.
 host = 127.0.0.1
+# Optional prefix which is prepended to all the metric names. Comes handy when you want to submit metrics from various environment to the same metric backend instance.
+prefix = None
 # Driver type for metrics collection.
 driver = noop
 # Destination port to connect to if driver requires connection.

conf/st2.dev.conf

@@ -119,6 +119,6 @@ jitter_interval = 0
 enable_common_libs = True
 
 [metrics]
-driver = noop
+driver = echo
 host = 127.0.0.1
 port = 8125

st2actions/st2actions/container/base.py

@@ -33,7 +33,7 @@
 from st2common.util.action_db import (update_liveaction_status, get_liveaction_by_id)
 from st2common.util import param as param_utils
 from st2common.util.config_loader import ContentPackConfigLoader
-from st2common.metrics.base import CounterWithTimer, format_metrics_key
+from st2common.metrics.base import CounterWithTimer
 from st2common.util import jsonify
 
 from st2common.runners.base import get_runner
@@ -82,7 +82,7 @@ def dispatch(self, liveaction_db):
                 'in an unsupported status of "%s".' % liveaction_db.status
             )
 
-        with CounterWithTimer(key="st2.action.executions"):
+        with CounterWithTimer(key="action.executions"):
             liveaction_db = funcs[liveaction_db.status](runner)
 
         return liveaction_db.result
@@ -122,9 +122,10 @@ def _do_run(self, runner):
             extra = {'runner': runner, 'parameters': resolved_action_params}
             LOG.debug('Performing run for runner: %s' % (runner.runner_id), extra=extra)
 
-            with CounterWithTimer(key=format_metrics_key(action_db=runner.action, key='action')):
-                (status, result, context) = runner.run(action_params)
-                result = jsonify.try_loads(result)
+            with CounterWithTimer(key='action.executions'):
+                with CounterWithTimer(key='action.%s.executions' % (runner.action.ref)):
+                    (status, result, context) = runner.run(action_params)
+                    result = jsonify.try_loads(result)
 
             action_completed = status in action_constants.LIVEACTION_COMPLETED_STATES
 

st2api/st2api/app.py

@@ -22,6 +22,8 @@
 from st2common.middleware.cors import CorsMiddleware
 from st2common.middleware.request_id import RequestIDMiddleware
 from st2common.middleware.logging import LoggingMiddleware
+from st2common.middleware.instrumentation import RequestInstrumentationMiddleware
+from st2common.middleware.instrumentation import ResponseInstrumentationMiddleware
 from st2common.router import Router
 from st2common.util.monkey_patch import monkey_patch
 from st2common.constants.system import VERSION_STRING
@@ -75,6 +77,8 @@ def setup_app(config={}):
     app = ErrorHandlingMiddleware(app)
     app = CorsMiddleware(app)
     app = LoggingMiddleware(app, router)
+    app = ResponseInstrumentationMiddleware(app, service_name='api')
     app = RequestIDMiddleware(app)
+    app = RequestInstrumentationMiddleware(app, service_name='api')
 
     return app

st2auth/st2auth/app.py

@@ -20,6 +20,8 @@
 from st2common.middleware.cors import CorsMiddleware
 from st2common.middleware.request_id import RequestIDMiddleware
 from st2common.middleware.logging import LoggingMiddleware
+from st2common.middleware.instrumentation import RequestInstrumentationMiddleware
+from st2common.middleware.instrumentation import ResponseInstrumentationMiddleware
 from st2common.router import Router
 from st2common.util.monkey_patch import monkey_patch
 from st2common.constants.system import VERSION_STRING
@@ -69,6 +71,8 @@ def setup_app(config={}):
     app = ErrorHandlingMiddleware(app)
     app = CorsMiddleware(app)
     app = LoggingMiddleware(app, router)
+    app = ResponseInstrumentationMiddleware(app, service_name='auth')
     app = RequestIDMiddleware(app)
+    app = RequestInstrumentationMiddleware(app, service_name='auth')
 
     return app

st2common/setup.py

@@ -68,6 +68,7 @@
         'st2common.metrics.driver': [
             'statsd = st2common.metrics.drivers.statsd_driver:StatsdDriver',
             'noop = st2common.metrics.drivers.noop_driver:NoopDriver',
+            'echo = st2common.metrics.drivers.echo_driver:EchoDriver',
         ],
     }
 )

st2common/st2common/config.py

@@ -538,6 +538,11 @@ def register_opts(ignore_errors=False):
         cfg.IntOpt(
             'port', default=8125,
             help='Destination port to connect to if driver requires connection.'),
+        cfg.StrOpt(
+            'prefix', default=None,
+            help='Optional prefix which is prepended to all the metric names. Comes handy when '
+                 'you want to submit metrics from various environment to the same metric '
+                 'backend instance.')
     ]
 
     do_register_opts(metrics_opts, group='metrics', ignore_errors=ignore_errors)