First support for alert notifications; minor other features

config/config.go

@@ -39,6 +39,29 @@ func (c Config) String() string {
 
 // Validate checks an entire parsed Config for the validity of its fields.
 func (c Config) Validate() error {
+	ncNames := map[string]bool{}
+	for _, nc := range c.NotificationConfig {
+		if nc.Name == nil {
+			return fmt.Errorf("Missing name in notification config: %s", proto.MarshalTextString(nc))
+		}
+		for _, pdc := range nc.PagerdutyConfig {
+			if pdc.ServiceKey == nil {
+				return fmt.Errorf("Missing service key in PagerDuty notification config: %s", proto.MarshalTextString(pdc))
+			}
+		}
+		for _, ec := range nc.EmailConfig {
+			if ec.Email == nil {
+				return fmt.Errorf("Missing email address in email notification config: %s", proto.MarshalTextString(ec))
+			}
+		}
+
+		if _, ok := ncNames[nc.GetName()]; ok {
+			return fmt.Errorf("Notification config name not unique: %s", nc.GetName())
+		}
+
+		ncNames[nc.GetName()] = true
+	}
+
 	for _, a := range c.AggregationRule {
 		for _, f := range a.Filter {
 			if f.NameRe == nil {
@@ -48,7 +71,12 @@ func (c Config) Validate() error {
 				return fmt.Errorf("Missing value pattern (value_re) in filter definition: %s", proto.MarshalTextString(f))
 			}
 		}
+
+		if _, ok := ncNames[a.GetNotificationConfigName()]; !ok {
+			return fmt.Errorf("No such notification config: %s", a.GetNotificationConfigName())
+		}
 	}
+
 	return nil
 }
 
@@ -61,8 +89,9 @@ func (c Config) AggregationRules() manager.AggregationRules {
 			filters = append(filters, manager.NewFilter(filter.GetNameRe(), filter.GetValueRe()))
 		}
 		rules = append(rules, &manager.AggregationRule{
-			Filters:    filters,
-			RepeatRate: time.Duration(r.GetRepeatRateSeconds()) * time.Second,
+			Filters:                filters,
+			RepeatRate:             time.Duration(r.GetRepeatRateSeconds()) * time.Second,
+			NotificationConfigName: r.GetNotificationConfigName(),
 		})
 	}
 	return rules

config/config.proto

@@ -13,6 +13,29 @@
 
 package io.prometheus.alert_manager;
 
+// Configuration for notification via PagerDuty.
+message PagerDutyConfig {
+  // PagerDuty service key, see:
+  // http://developer.pagerduty.com/documentation/integration/events
+  optional string service_key = 1;
+}
+
+// Configuration for notification via mail.
+message EmailConfig {
+  // Email address to notify.
+  optional string email = 1;
+}
+
+// Notification configuration definition.
+message NotificationConfig {
+  // Name of this NotificationConfig. Referenced from AggregationRule.
+  optional string name = 1;
+  // Zero or more PagerDuty notification configurations.
+  repeated PagerDutyConfig pagerduty_config = 2;
+  // Zero or more email notification configurations.
+  repeated EmailConfig email_config = 3;
+}
+
 // A regex-based label filter used in aggregations.
 message Filter {
   // The regex matching the label name.
@@ -21,11 +44,21 @@ message Filter {
   optional string value_re = 2;
 }
 
+// Grouping and notification setting definitions for alerts.
 message AggregationRule {
+  // Filters that define which alerts are matched by this AggregationRule.
   repeated Filter filter = 1;
+  // How many seconds to wait before resending a notification for a specific alert.
   optional int32 repeat_rate_seconds = 2 [default = 7200];
+  // Notification configuration to use for this AggregationRule, referenced by
+  // their name.
+  optional string notification_config_name = 3;
 }
 
+// Global alert manager configuration.
 message AlertManagerConfig {
+  // Aggregation rule definitions.
   repeated AggregationRule aggregation_rule = 1;
+  // Notification configuration definitions.
+  repeated NotificationConfig notification_config = 2;
 }

config/config_test.go

@@ -48,13 +48,25 @@ func TestConfigs(t *testing.T) {
 		}, {
 			inputFile: "sample.conf.input",
 		}, {
-			inputFile:   "missing_name_re.conf.input",
+			inputFile:   "missing_filter_name_re.conf.input",
 			shouldFail:  true,
 			errContains: "Missing name pattern",
 		}, {
 			inputFile:   "invalid_proto_format.conf.input",
 			shouldFail:  true,
 			errContains: "unknown field name",
+		}, {
+			inputFile:   "duplicate_nc_name.conf.input",
+			shouldFail:  true,
+			errContains: "not unique",
+		}, {
+			inputFile:   "nonexistent_nc_name.conf.input",
+			shouldFail:  true,
+			errContains: "No such notification config",
+		}, {
+			inputFile:   "missing_nc_name.conf.input",
+			shouldFail:  true,
+			errContains: "Missing name",
 		},
 	}
 

config/fixtures/duplicate_nc_name.conf.input

@@ -0,0 +1,28 @@
+notification_config {
+  name: "alertmanager_test"
+  pagerduty_config {
+    service_key: "supersecretapikey"
+  }
+  email_config {
+    email: "test@testservice.org"
+  }
+}
+
+notification_config {
+  name: "alertmanager_test"
+  pagerduty_config {
+    service_key: "supersecretapikey"
+  }
+  email_config {
+    email: "test@testservice.org"
+  }
+}
+
+aggregation_rule {
+  filter {
+    name_re: "service"
+    value_re: "test"
+  }
+  repeat_rate_seconds: 3600
+  notification_config_name: "alertmanager_test"
+}

config/fixtures/missing_filter_name_re.conf.input

@@ -0,0 +1,6 @@
+aggregation_rule {
+  filter {
+    value_re: "test"
+  }
+  repeat_rate_seconds: 3600
+}

config/fixtures/missing_nc_name.conf.input

@@ -0,0 +1,27 @@
+notification_config {
+  pagerduty_config {
+    service_key: "supersecretapikey"
+  }
+  email_config {
+    email: "test@testservice.org"
+  }
+}
+
+notification_config {
+  name: "alertmanager_test"
+  pagerduty_config {
+    service_key: "supersecretapikey"
+  }
+  email_config {
+    email: "test@testservice.org"
+  }
+}
+
+aggregation_rule {
+  filter {
+    name_re: "service"
+    value_re: "test"
+  }
+  repeat_rate_seconds: 3600
+  notification_config_name: "alertmanager_test"
+}

config/fixtures/nonexistent_nc_name.conf.input

@@ -0,0 +1,18 @@
+notification_config {
+  name: "alertmanager_test"
+  pagerduty_config {
+    service_key: "supersecretapikey"
+  }
+  email_config {
+    email: "test@testservice.org"
+  }
+}
+
+aggregation_rule {
+  filter {
+    name_re: "service"
+    value_re: "test"
+  }
+  repeat_rate_seconds: 3600
+  notification_config_name: "alertmanager_test2"
+}

config/fixtures/sample.conf.input

@@ -1,18 +1,18 @@
-aggregation_rule {
-  filter {
-    name_re: "service"
-    value_re: "discovery"
+notification_config {
+  name: "alertmanager_test"
+  pagerduty_config {
+    service_key: "supersecretapikey"
   }
-  filter {
-    name_re: "zone"
-    value_re: "aa"
+  email_config {
+    email: "test@testservice.org"
   }
-  repeat_rate_seconds: 3600
 }
 
 aggregation_rule {
   filter {
     name_re: "service"
     value_re: "test"
   }
+  repeat_rate_seconds: 3600
+  notification_config_name: "alertmanager_test"
 }

main.go

@@ -38,9 +38,10 @@ func main() {
 	suppressor := manager.NewSuppressor()
 	defer suppressor.Close()
 
-	summarizer := manager.NewSummaryDispatcher()
+	notifier := manager.NewNotifier(conf.NotificationConfig)
+	defer notifier.Close()
 
-	aggregator := manager.NewAggregator(summarizer)
+	aggregator := manager.NewAggregator(notifier)
 	defer aggregator.Close()
 
 	webService := &web.WebService{
@@ -52,16 +53,17 @@ func main() {
 
 		// Template-based page handlers.
 		AlertsHandler: &web.AlertsHandler{
-			Aggregator: aggregator,
+			Aggregator:              aggregator,
+			IsInhibitedInterrogator: suppressor,
 		},
 		SilencesHandler: &web.SilencesHandler{
 			Suppressor: suppressor,
 		},
 	}
 	go webService.ServeForever()
 
-  aggregator.SetRules(conf.AggregationRules())
+	aggregator.SetRules(conf.AggregationRules())
 
 	log.Println("Running summary dispatcher...")
-	summarizer.Dispatch(suppressor)
+	notifier.Dispatch(suppressor)
 }