From 3c5b44256d746f605b786cbd03fca5cf69357589 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 23 Aug 2019 15:22:22 +0000 Subject: [PATCH] Fix duplicate alertmanager notifications We need to wire up the notification log to gossip to share state with peers. Also avoid creating two throwaway metrics registries, since one will do. Signed-off-by: Bryan Boreham --- pkg/alertmanager/alertmanager.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pkg/alertmanager/alertmanager.go b/pkg/alertmanager/alertmanager.go index c7deed935b9..3ccb3f28aaf 100644 --- a/pkg/alertmanager/alertmanager.go +++ b/pkg/alertmanager/alertmanager.go @@ -65,6 +65,11 @@ func New(cfg *Config) (*Alertmanager, error) { stop: make(chan struct{}), } + // TODO(cortex): Build a registry that can merge metrics from multiple users. + // For now, these metrics are ignored, as we can't register the same + // metric twice with a single registry. + localRegistry := prometheus.NewRegistry() + am.wg.Add(1) nflogID := fmt.Sprintf("nflog:%s", cfg.UserID) var err error @@ -72,23 +77,19 @@ func New(cfg *Config) (*Alertmanager, error) { nflog.WithRetention(cfg.Retention), nflog.WithSnapshot(filepath.Join(cfg.DataDir, nflogID)), nflog.WithMaintenance(notificationLogMaintenancePeriod, am.stop, am.wg.Done), - // TODO(cortex): Build a registry that can merge metrics from multiple users. - // For now, these metrics are ignored, as we can't register the same - // metric twice with a single registry. - nflog.WithMetrics(prometheus.NewRegistry()), + nflog.WithMetrics(localRegistry), nflog.WithLogger(log.With(am.logger, "component", "nflog")), ) if err != nil { return nil, fmt.Errorf("failed to create notification log: %v", err) } + if cfg.Peer != nil { + c := cfg.Peer.AddState("nfl:"+cfg.UserID, am.nflog, localRegistry) + am.nflog.SetBroadcast(c.Broadcast) + } am.marker = types.NewMarker() - // TODO(cortex): Build a registry that can merge metrics from multiple users. - // For now, these metrics are ignored, as we can't register the same - // metric twice with a single registry. - localRegistry := prometheus.NewRegistry() - silencesID := fmt.Sprintf("silences:%s", cfg.UserID) am.silences, err = silence.New(silence.Options{ SnapshotFile: filepath.Join(cfg.DataDir, silencesID),