From 48d7f25d2d0fab873967c7e678f2c067f9e42e99 Mon Sep 17 00:00:00 2001 From: Roger Steneteg Date: Tue, 19 Feb 2019 12:29:31 -0600 Subject: [PATCH 1/2] adding alert templates support for AlertManager Signed-off-by: Roger Steneteg --- pkg/alertmanager/alertmanager.go | 12 ++++++---- pkg/alertmanager/multitenant.go | 39 +++++++++++++++++++++++++++++--- pkg/configs/api/api.go | 22 ++++++++++++++---- pkg/configs/api/api_test.go | 11 --------- pkg/configs/configs.go | 4 ++++ 5 files changed, 65 insertions(+), 23 deletions(-) diff --git a/pkg/alertmanager/alertmanager.go b/pkg/alertmanager/alertmanager.go index 2d635ce7fe3..2bdeb225db2 100644 --- a/pkg/alertmanager/alertmanager.go +++ b/pkg/alertmanager/alertmanager.go @@ -149,18 +149,20 @@ func New(cfg *Config) (*Alertmanager, error) { } // ApplyConfig applies a new configuration to an Alertmanager. -func (am *Alertmanager) ApplyConfig(conf *config.Config) error { +func (am *Alertmanager) ApplyConfig(userID string, conf *config.Config) error { var ( tmpl *template.Template pipeline notify.Stage ) - // TODO(cortex): How to support template files? - if len(conf.Templates) != 0 { - return fmt.Errorf("template files are not yet supported") + templateFiles := make([]string, len(conf.Templates), len(conf.Templates)) + if len(conf.Templates) > 0 { + for i, t := range conf.Templates { + templateFiles[i] = filepath.Join(am.cfg.DataDir, "templates", userID, t) + } } - tmpl, err := template.FromGlobs() + tmpl, err := template.FromGlobs(templateFiles...) if err != nil { return err } diff --git a/pkg/alertmanager/multitenant.go b/pkg/alertmanager/multitenant.go index 5e2ca05bb5a..24fef2c3354 100644 --- a/pkg/alertmanager/multitenant.go +++ b/pkg/alertmanager/multitenant.go @@ -9,6 +9,7 @@ import ( "net" "net/http" "os" + "path/filepath" "sort" "strconv" "strings" @@ -421,12 +422,44 @@ func (am *MultitenantAlertmanager) transformConfig(userID string, amConfig *amco return amConfig, nil } +func (am *MultitenantAlertmanager) createTemplatesFile(userID, fn, content string) (bool, error) { + dir := filepath.Join(am.cfg.DataDir, "templates", userID, filepath.Dir(fn)) + err := os.MkdirAll(dir, 0755) + if err != nil { + return false, fmt.Errorf("unable to create Alertmanager templates directory %q: %s", dir, err) + } + + file := filepath.Join(dir, fn) + // Check if the template file already exists and if it has changed + if tmpl, err := ioutil.ReadFile(file); err == nil && string(tmpl) == content { + return false, nil + } + + if err := ioutil.WriteFile(file, []byte(content), 0644); err != nil { + return false, fmt.Errorf("unable to create Alertmanager template file %q: %s", file, err) + } + + return true, nil +} + // setConfig applies the given configuration to the alertmanager for `userID`, // creating an alertmanager if it doesn't already exist. func (am *MultitenantAlertmanager) setConfig(userID string, config configs.Config) error { _, hasExisting := am.alertmanagers[userID] var amConfig *amconfig.Config var err error + var hasTemplateChanges bool + + for fn, content := range config.TemplateFiles { + hasChanged, err := am.createTemplatesFile(userID, fn, content) + if err != nil { + return err + } + + if hasChanged { + hasTemplateChanges = true + } + } if config.AlertmanagerConfig == "" { if am.fallbackConfig == "" { @@ -462,9 +495,9 @@ func (am *MultitenantAlertmanager) setConfig(userID string, config configs.Confi am.alertmanagersMtx.Lock() am.alertmanagers[userID] = newAM am.alertmanagersMtx.Unlock() - } else if am.cfgs[userID].AlertmanagerConfig != config.AlertmanagerConfig { + } else if am.cfgs[userID].AlertmanagerConfig != config.AlertmanagerConfig || hasTemplateChanges { // If the config changed, apply the new one. - err := am.alertmanagers[userID].ApplyConfig(amConfig) + err := am.alertmanagers[userID].ApplyConfig(userID, amConfig) if err != nil { return fmt.Errorf("unable to apply Alertmanager config for user %v: %v", userID, err) } @@ -486,7 +519,7 @@ func (am *MultitenantAlertmanager) newAlertmanager(userID string, amConfig *amco return nil, fmt.Errorf("unable to start Alertmanager for user %v: %v", userID, err) } - if err := newAM.ApplyConfig(amConfig); err != nil { + if err := newAM.ApplyConfig(userID, amConfig); err != nil { return nil, fmt.Errorf("unable to apply initial config for user %v: %v", userID, err) } return newAM, nil diff --git a/pkg/configs/api/api.go b/pkg/configs/api/api.go index 462b0d7013f..bfe34d4748a 100644 --- a/pkg/configs/api/api.go +++ b/pkg/configs/api/api.go @@ -8,6 +8,7 @@ import ( "net/http" "strconv" + "github.com/alecthomas/template" "github.com/go-kit/kit/log/level" "github.com/gorilla/mux" amconfig "github.com/prometheus/alertmanager/config" @@ -57,6 +58,8 @@ func (a *API) RegisterRoutes(r *mux.Router) { // be used. {"get_rules", "GET", "/api/prom/configs/rules", a.getConfig}, {"set_rules", "POST", "/api/prom/configs/rules", a.setConfig}, + {"get_templates", "GET", "/api/prom/configs/templates", a.getConfig}, + {"set_templates", "POST", "/api/prom/configs/templates", a.setConfig}, {"get_alertmanager_config", "GET", "/api/prom/configs/alertmanager", a.getConfig}, {"set_alertmanager_config", "POST", "/api/prom/configs/alertmanager", a.setConfig}, {"validate_alertmanager_config", "POST", "/api/prom/configs/alertmanager/validate", a.validateAlertmanagerConfig}, @@ -124,6 +127,11 @@ func (a *API) setConfig(w http.ResponseWriter, r *http.Request) { http.Error(w, fmt.Sprintf("Invalid rules: %v", err), http.StatusBadRequest) return } + if err := validateTemplateFiles(cfg); err != nil { + level.Error(logger).Log("msg", "invalid templates", "err", err) + http.Error(w, fmt.Sprintf("Invalid templates: %v", err), http.StatusBadRequest) + return + } if err := a.db.SetConfig(userID, cfg); err != nil { // XXX: Untested level.Error(logger).Log("msg", "error storing config", "err", err) @@ -162,10 +170,6 @@ func validateAlertmanagerConfig(cfg string) error { return err } - if len(amCfg.Templates) != 0 { - return fmt.Errorf("template files are not supported in Cortex yet") - } - for _, recv := range amCfg.Receivers { if len(recv.EmailConfigs) != 0 { return fmt.Errorf("email notifications are not supported in Cortex yet") @@ -180,6 +184,16 @@ func validateRulesFiles(c configs.Config) error { return err } +func validateTemplateFiles(c configs.Config) error { + for fn, content := range c.TemplateFiles { + if _, err := template.New(fn).Parse(content); err != nil { + return err + } + } + + return nil +} + // ConfigsView renders multiple configurations, mapping userID to configs.View. // Exposed only for tests. type ConfigsView struct { diff --git a/pkg/configs/api/api_test.go b/pkg/configs/api/api_test.go index 116ce5303e1..45abef4d3b6 100644 --- a/pkg/configs/api/api_test.go +++ b/pkg/configs/api/api_test.go @@ -233,17 +233,6 @@ var amCfgValidationTests = []struct { errContains: "yaml", }, { config: ` - route: - receiver: noop - templates: - - "/path/to/file" - - receivers: - - name: noop`, - shouldFail: true, - errContains: "template files are not supported in Cortex yet", - }, { - config: ` global: smtp_smarthost: localhost:25 smtp_from: alertmanager@example.org diff --git a/pkg/configs/configs.go b/pkg/configs/configs.go index c5dab0d96b0..fcf9c593125 100644 --- a/pkg/configs/configs.go +++ b/pkg/configs/configs.go @@ -72,6 +72,7 @@ func (v *RuleFormatVersion) UnmarshalJSON(data []byte) error { type Config struct { // RulesFiles maps from a rules filename to file contents. RulesConfig RulesConfig + TemplateFiles map[string]string AlertmanagerConfig string } @@ -81,6 +82,7 @@ type Config struct { type configCompat struct { RulesFiles map[string]string `json:"rules_files"` RuleFormatVersion RuleFormatVersion `json:"rule_format_version"` + TemplateFiles map[string]string `json:"template_files"` AlertmanagerConfig string `json:"alertmanager_config"` } @@ -89,6 +91,7 @@ func (c Config) MarshalJSON() ([]byte, error) { compat := &configCompat{ RulesFiles: c.RulesConfig.Files, RuleFormatVersion: c.RulesConfig.FormatVersion, + TemplateFiles: c.TemplateFiles, AlertmanagerConfig: c.AlertmanagerConfig, } @@ -106,6 +109,7 @@ func (c *Config) UnmarshalJSON(data []byte) error { Files: compat.RulesFiles, FormatVersion: compat.RuleFormatVersion, }, + TemplateFiles: compat.TemplateFiles, AlertmanagerConfig: compat.AlertmanagerConfig, } return nil From 71c31af0c5915706972e72d05326bfff8104f8fb Mon Sep 17 00:00:00 2001 From: Roger Steneteg Date: Fri, 22 Feb 2019 10:00:45 -0600 Subject: [PATCH 2/2] switched to the html/template package Signed-off-by: Roger Steneteg --- pkg/configs/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/configs/api/api.go b/pkg/configs/api/api.go index bfe34d4748a..74a1c7c5fcd 100644 --- a/pkg/configs/api/api.go +++ b/pkg/configs/api/api.go @@ -4,11 +4,11 @@ import ( "database/sql" "encoding/json" "fmt" + "html/template" "io/ioutil" "net/http" "strconv" - "github.com/alecthomas/template" "github.com/go-kit/kit/log/level" "github.com/gorilla/mux" amconfig "github.com/prometheus/alertmanager/config"