diff --git a/config/notifiers.go b/config/notifiers.go index 831c45af2a..3608b5719c 100644 --- a/config/notifiers.go +++ b/config/notifiers.go @@ -617,6 +617,10 @@ type IncidentioConfig struct { // Timeout is the maximum time allowed to invoke incident.io. Setting this to 0 // does not impose a timeout. Timeout time.Duration `yaml:"timeout" json:"timeout"` + + // Metadata is a set of arbitrary key/value pairs to include with alerts. + // Values support Go template syntax. + Metadata map[string]string `yaml:"metadata,omitempty" json:"metadata,omitempty"` } // UnmarshalYAML implements the yaml.Unmarshaler interface. diff --git a/notify/incidentio/incidentio.go b/notify/incidentio/incidentio.go index b985853eab..6b16cf5e6b 100644 --- a/notify/incidentio/incidentio.go +++ b/notify/incidentio/incidentio.go @@ -96,9 +96,10 @@ type Message struct { *template.Data // The protocol version. - Version string `json:"version"` - GroupKey string `json:"groupKey"` - TruncatedAlerts uint64 `json:"truncatedAlerts"` + Version string `json:"version"` + GroupKey string `json:"groupKey"` + TruncatedAlerts uint64 `json:"truncatedAlerts"` + Metadata map[string]string `json:"metadata,omitempty"` } func truncateAlerts(maxAlerts uint64, alerts []*types.Alert) ([]*types.Alert, uint64) { @@ -164,11 +165,28 @@ func (n *Notifier) Notify(ctx context.Context, alerts ...*types.Alert) (bool, er n.logger.Debug("incident.io notification", "groupKey", groupKey) + // Render metadata templates + var metadata map[string]string + if len(n.conf.Metadata) > 0 { + var tmplErr error + tmpl := notify.TmplText(n.tmpl, data, &tmplErr) + + metadata = make(map[string]string, len(n.conf.Metadata)) + for k, v := range n.conf.Metadata { + metadata[k] = tmpl(v) + } + + if tmplErr != nil { + return false, fmt.Errorf("failed to render metadata templates: %w", tmplErr) + } + } + msg := &Message{ Version: "1", Data: data, GroupKey: groupKey.String(), TruncatedAlerts: numTruncated, + Metadata: metadata, } buf, err := n.encodeMessage(msg) diff --git a/notify/incidentio/incidentio_test.go b/notify/incidentio/incidentio_test.go index 65557a6199..48eae75252 100644 --- a/notify/incidentio/incidentio_test.go +++ b/notify/incidentio/incidentio_test.go @@ -476,3 +476,207 @@ func TestIncidentIOPayloadTruncationWithLabelTruncation(t *testing.T) { } } } + +func TestIncidentIOMetadataEmpty(t *testing.T) { + // When no metadata is configured, the field should be omitted from JSON. + var receivedBody []byte + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + var err error + receivedBody, err = io.ReadAll(r.Body) + require.NoError(t, err) + w.WriteHeader(http.StatusOK) + }, + )) + defer server.Close() + + u, err := url.Parse(server.URL) + require.NoError(t, err) + + notifier, err := New( + &config.IncidentioConfig{ + URL: &config.URL{URL: u}, + HTTPConfig: &commoncfg.HTTPClientConfig{}, + AlertSourceToken: "test-token", + }, + test.CreateTmpl(t), + promslog.NewNopLogger(), + ) + require.NoError(t, err) + + ctx := context.Background() + ctx = notify.WithGroupKey(ctx, "1") + + alert := &types.Alert{ + Alert: model.Alert{ + Labels: model.LabelSet{"alertname": "TestAlert"}, + StartsAt: time.Now(), + EndsAt: time.Now().Add(time.Hour), + }, + } + + retry, err := notifier.Notify(ctx, alert) + require.NoError(t, err) + require.False(t, retry) + + // Verify metadata field is not present in JSON + var rawMsg map[string]json.RawMessage + require.NoError(t, json.Unmarshal(receivedBody, &rawMsg)) + _, hasMetadata := rawMsg["metadata"] + require.False(t, hasMetadata, "metadata field should be omitted when not configured") +} + +func TestIncidentIOMetadataStatic(t *testing.T) { + // Static metadata values (no templates) should be passed through as-is. + var receivedMsg Message + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + require.NoError(t, json.NewDecoder(r.Body).Decode(&receivedMsg)) + w.WriteHeader(http.StatusOK) + }, + )) + defer server.Close() + + u, err := url.Parse(server.URL) + require.NoError(t, err) + + notifier, err := New( + &config.IncidentioConfig{ + URL: &config.URL{URL: u}, + HTTPConfig: &commoncfg.HTTPClientConfig{}, + AlertSourceToken: "test-token", + Metadata: map[string]string{ + "environment": "production", + "team": "sre", + "region": "us-east-1", + }, + }, + test.CreateTmpl(t), + promslog.NewNopLogger(), + ) + require.NoError(t, err) + + ctx := context.Background() + ctx = notify.WithGroupKey(ctx, "1") + + alert := &types.Alert{ + Alert: model.Alert{ + Labels: model.LabelSet{"alertname": "TestAlert"}, + StartsAt: time.Now(), + EndsAt: time.Now().Add(time.Hour), + }, + } + + retry, err := notifier.Notify(ctx, alert) + require.NoError(t, err) + require.False(t, retry) + + require.Equal(t, map[string]string{ + "environment": "production", + "team": "sre", + "region": "us-east-1", + }, receivedMsg.Metadata) +} + +func TestIncidentIOMetadataTemplated(t *testing.T) { + // Metadata values using Go templates should be rendered with alert data. + var receivedMsg Message + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + require.NoError(t, json.NewDecoder(r.Body).Decode(&receivedMsg)) + w.WriteHeader(http.StatusOK) + }, + )) + defer server.Close() + + u, err := url.Parse(server.URL) + require.NoError(t, err) + + notifier, err := New( + &config.IncidentioConfig{ + URL: &config.URL{URL: u}, + HTTPConfig: &commoncfg.HTTPClientConfig{}, + AlertSourceToken: "test-token", + Metadata: map[string]string{ + "severity": "{{ .CommonLabels.severity }}", + "alert_name": "{{ .CommonLabels.alertname }}", + "alert_count": "{{ len .Alerts }}", + "status": "{{ .Status }}", + }, + }, + test.CreateTmpl(t), + promslog.NewNopLogger(), + ) + require.NoError(t, err) + + ctx := context.Background() + ctx = notify.WithGroupKey(ctx, "1") + + alerts := []*types.Alert{ + { + Alert: model.Alert{ + Labels: model.LabelSet{ + "alertname": "HighLatency", + "severity": "critical", + }, + StartsAt: time.Now(), + EndsAt: time.Now().Add(time.Hour), + }, + }, + { + Alert: model.Alert{ + Labels: model.LabelSet{ + "alertname": "HighLatency", + "severity": "critical", + }, + StartsAt: time.Now(), + EndsAt: time.Now().Add(time.Hour), + }, + }, + } + + retry, err := notifier.Notify(ctx, alerts...) + require.NoError(t, err) + require.False(t, retry) + + require.Equal(t, "critical", receivedMsg.Metadata["severity"]) + require.Equal(t, "HighLatency", receivedMsg.Metadata["alert_name"]) + require.Equal(t, "2", receivedMsg.Metadata["alert_count"]) + require.Equal(t, "firing", receivedMsg.Metadata["status"]) +} + +func TestIncidentIOMetadataTemplateError(t *testing.T) { + // Invalid template references should return an error with no retry. + u, err := url.Parse("https://example.com") + require.NoError(t, err) + + notifier, err := New( + &config.IncidentioConfig{ + URL: &config.URL{URL: u}, + HTTPConfig: &commoncfg.HTTPClientConfig{}, + AlertSourceToken: "test-token", + Metadata: map[string]string{ + "bad": "{{ .NonExistentMethod }}", + }, + }, + test.CreateTmpl(t), + promslog.NewNopLogger(), + ) + require.NoError(t, err) + + ctx := context.Background() + ctx = notify.WithGroupKey(ctx, "1") + + alert := &types.Alert{ + Alert: model.Alert{ + Labels: model.LabelSet{"alertname": "TestAlert"}, + StartsAt: time.Now(), + EndsAt: time.Now().Add(time.Hour), + }, + } + + retry, err := notifier.Notify(ctx, alert) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to render metadata templates") + require.False(t, retry, "should not retry on template rendering errors") +}