From 10f300024ef17c5ba0b1e2a8695e5d07ce7e3f01 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Thu, 13 Jun 2019 23:42:46 -0400 Subject: [PATCH 01/60] basic version of gcs backed ruler Signed-off-by: Jacob Lisi --- pkg/alertmanager/multitenant.go | 11 +- pkg/chunk/gcp/gcs_config_client.go | 212 ++++++++++++++++++++++++++++ pkg/chunk/gcp/gcs_object_client.go | 4 + pkg/chunk/gcp/instrumentation.go | 16 ++- pkg/configs/client/client.go | 86 +++++------ pkg/configs/client/config.go | 12 +- pkg/configs/configs.go | 18 +++ pkg/configs/db/db.go | 54 ++++--- pkg/configs/db/dbtest/unit.go | 2 +- pkg/configs/db/memory/memory.go | 45 +++++- pkg/configs/db/postgres/postgres.go | 125 ++++++++++++---- pkg/configs/db/timed.go | 23 +++ pkg/configs/db/traced.go | 10 ++ pkg/cortex/modules.go | 4 +- pkg/ruler/api_test.go | 40 ++---- pkg/ruler/ruler.go | 6 +- pkg/ruler/ruler_test.go | 7 +- pkg/ruler/scheduler.go | 40 ++---- 18 files changed, 514 insertions(+), 201 deletions(-) create mode 100644 pkg/chunk/gcp/gcs_config_client.go diff --git a/pkg/alertmanager/multitenant.go b/pkg/alertmanager/multitenant.go index 608377893d7..e49c7b9f4f3 100644 --- a/pkg/alertmanager/multitenant.go +++ b/pkg/alertmanager/multitenant.go @@ -24,6 +24,7 @@ import ( "github.com/cortexproject/cortex/pkg/configs" configs_client "github.com/cortexproject/cortex/pkg/configs/client" + "github.com/cortexproject/cortex/pkg/configs/db" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" ) @@ -214,7 +215,7 @@ func (cfg *MultitenantAlertmanagerConfig) RegisterFlags(f *flag.FlagSet) { type MultitenantAlertmanager struct { cfg *MultitenantAlertmanagerConfig - configsAPI configs_client.Client + configsAPI db.Poller // The fallback config is stored as a string and parsed every time it's needed // because we mutate the parsed results and don't want those changes to take @@ -350,16 +351,12 @@ func (am *MultitenantAlertmanager) updateConfigs(now time.Time) error { // poll the configuration server. Not re-entrant. func (am *MultitenantAlertmanager) poll() (map[string]configs.View, error) { - configID := am.latestConfig - cfgs, err := am.configsAPI.GetAlerts(context.Background(), configID) + cfgs, err := am.configsAPI.GetAlerts(context.Background()) if err != nil { level.Warn(util.Logger).Log("msg", "MultitenantAlertmanager: configs server poll failed", "err", err) return nil, err } - am.latestMutex.Lock() - am.latestConfig = cfgs.GetLatestConfigID() - am.latestMutex.Unlock() - return cfgs.Configs, nil + return cfgs, nil } func (am *MultitenantAlertmanager) addNewConfigs(cfgs map[string]configs.View) { diff --git a/pkg/chunk/gcp/gcs_config_client.go b/pkg/chunk/gcp/gcs_config_client.go new file mode 100644 index 00000000000..1da3f483b37 --- /dev/null +++ b/pkg/chunk/gcp/gcs_config_client.go @@ -0,0 +1,212 @@ +package gcp + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "strings" + "time" + + "cloud.google.com/go/storage" + "google.golang.org/api/iterator" + + "github.com/cortexproject/cortex/pkg/configs" + "github.com/pkg/errors" +) + +const ( + rulePrefix = "rules/" +) + +type ConfigClient struct { + cfg ConfigDBConfig + + client *storage.Client + bucket *storage.BucketHandle + + lastPolled time.Time +} + +// ConfigDBConfig is config for the GCS Chunk Client. +type ConfigDBConfig struct { + BucketName string `yaml:"bucket_name"` +} + +// RegisterFlags registers flags. +func (cfg ConfigDBConfig) RegisterFlags(f *flag.FlagSet) { + f.StringVar(&cfg.BucketName, "gcs.configdb.bucketname", "", "Name of GCS bucket rule and alert configurations in") +} + +// NewConfigClient makes a new chunk.ObjectClient that writes chunks to GCS. +func NewConfigClient(ctx context.Context, cfg ConfigDBConfig) (*ConfigClient, error) { + option, err := gcsInstrumentation(ctx, "configs") + if err != nil { + return nil, err + } + + client, err := storage.NewClient(ctx, option) + if err != nil { + return nil, err + } + + bucket := client.Bucket(cfg.BucketName) + if bucket == nil { + return nil, fmt.Errorf("bucket %v does not exist", cfg.BucketName) + } + return &ConfigClient{ + cfg: cfg, + client: client, + bucket: bucket, + lastPolled: time.Unix(0, 0), + }, nil +} + +func (g *ConfigClient) GetRulesConfig(ctx context.Context, userID string) (configs.VersionedRulesConfig, error) { + return g.getRulesConfig(ctx, rulePrefix+userID) +} + +func (g *ConfigClient) getRulesConfig(ctx context.Context, ruleObj string) (configs.VersionedRulesConfig, error) { + reader, err := g.bucket.Object(ruleObj).NewReader(ctx) + if err != nil { + return configs.VersionedRulesConfig{}, err + } + defer reader.Close() + + buf, err := ioutil.ReadAll(reader) + if err != nil { + return configs.VersionedRulesConfig{}, err + } + + config := configs.VersionedRulesConfig{} + err = json.Unmarshal(buf, &config) + if err != nil { + return configs.VersionedRulesConfig{}, err + } + + config.ID = configs.ID(reader.Attrs.Generation) + return config, nil +} + +func (g *ConfigClient) SetRulesConfig(ctx context.Context, userID string, oldConfig configs.RulesConfig, newConfig configs.RulesConfig) (bool, error) { + current, err := g.GetRulesConfig(ctx, userID) + if err != nil { + return false, err + } + + // The supplied oldConfig must match the current config. If no config + // exists, then oldConfig must be nil. Otherwise, it must exactly + // equal the existing config. + if oldConfig.Equal(current.Config) { + return false, errors.New("old config provided does not match what is currently stored") + } + + cfgBytes, err := json.Marshal(newConfig) + if err != nil { + return false, err + } + + writer := g.bucket.Object(rulePrefix + userID).If(storage.Conditions{GenerationMatch: int64(current.ID)}).NewWriter(ctx) + if _, err := writer.Write(cfgBytes); err != nil { + return false, err + } + if err := writer.Close(); err != nil { + return true, err + } + + return true, nil +} + +func (g *ConfigClient) GetAllRulesConfigs(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { + objs := g.bucket.Objects(ctx, &storage.Query{ + Prefix: rulePrefix, + }) + + ruleMap := map[string]configs.VersionedRulesConfig{} + for { + objAttrs, err := objs.Next() + + if err == iterator.Done { + break + } + + if err != nil { + return nil, err + } + + rls, err := g.getRulesConfig(ctx, objAttrs.Name) + if err != nil { + return nil, err + } + ruleMap[strings.TrimPrefix(objAttrs.Name, rulePrefix)] = rls + } + + return ruleMap, nil +} + +func (g *ConfigClient) GetRulesConfigs(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { + panic("not implemented") +} + +func (g *ConfigClient) GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { + objs := g.bucket.Objects(ctx, &storage.Query{ + Prefix: rulePrefix, + }) + + ruleMap := map[string]configs.VersionedRulesConfig{} + for { + objAttrs, err := objs.Next() + + if err == iterator.Done { + break + } + + if err != nil { + return nil, err + } + + if objAttrs.Updated.After(g.lastPolled) { + rls, err := g.getRulesConfig(ctx, objAttrs.Name) + if err != nil { + return nil, err + } + ruleMap[strings.TrimPrefix(objAttrs.Name, rulePrefix)] = rls + } + } + + g.lastPolled = time.Now() + return ruleMap, nil +} + +func (g *ConfigClient) GetConfig(ctx context.Context, userID string) (configs.View, error) { + panic("not implemented") +} + +func (g *ConfigClient) SetConfig(ctx context.Context, userID string, cfg configs.Config) error { + panic("not implemented") +} + +func (g *ConfigClient) GetAllConfigs(ctx context.Context) (map[string]configs.View, error) { + panic("not implemented") +} + +func (g *ConfigClient) GetConfigs(ctx context.Context, since configs.ID) (map[string]configs.View, error) { + panic("not implemented") +} + +func (g *ConfigClient) DeactivateConfig(ctx context.Context, userID string) error { + panic("not implemented") +} + +func (g *ConfigClient) RestoreConfig(ctx context.Context, userID string) error { + panic("not implemented") +} + +func (g *ConfigClient) Close() error { + panic("not implemented") +} + +func (g *ConfigClient) GetAlerts(ctx context.Context) (map[string]configs.View, error) { + panic("not implemented") +} diff --git a/pkg/chunk/gcp/gcs_object_client.go b/pkg/chunk/gcp/gcs_object_client.go index c07f25ac94c..5e93fa8393c 100644 --- a/pkg/chunk/gcp/gcs_object_client.go +++ b/pkg/chunk/gcp/gcs_object_client.go @@ -36,7 +36,11 @@ func (cfg *GCSConfig) RegisterFlags(f *flag.FlagSet) { // NewGCSObjectClient makes a new chunk.ObjectClient that writes chunks to GCS. func NewGCSObjectClient(ctx context.Context, cfg GCSConfig, schemaCfg chunk.SchemaConfig) (chunk.ObjectClient, error) { +<<<<<<< HEAD option, err := gcsInstrumentation(ctx, storage.ScopeReadWrite) +======= + option, err := gcsInstrumentation(ctx, "chunk") +>>>>>>> c48b40de... basic version of gcs backed ruler if err != nil { return nil, err } diff --git a/pkg/chunk/gcp/instrumentation.go b/pkg/chunk/gcp/instrumentation.go index 1347ee69cef..b49c9ff14cf 100644 --- a/pkg/chunk/gcp/instrumentation.go +++ b/pkg/chunk/gcp/instrumentation.go @@ -36,7 +36,7 @@ var ( // GCS latency seems to range from a few ms to a few secs and is // important. So use 6 buckets from 5ms to 5s. Buckets: prometheus.ExponentialBuckets(0.005, 4, 6), - }, []string{"operation", "status_code"}) + }, []string{"operation", "status_code", "type"}) ) func bigtableInstrumentation() ([]grpc.UnaryClientInterceptor, []grpc.StreamClientInterceptor) { @@ -50,15 +50,16 @@ func bigtableInstrumentation() ([]grpc.UnaryClientInterceptor, []grpc.StreamClie } } -func gcsInstrumentation(ctx context.Context, scope string) (option.ClientOption, error) { +func gcsInstrumentation(ctx context.Context, clientType, scope string) (option.ClientOption, error) { transport, err := google_http.NewTransport(ctx, http.DefaultTransport, option.WithScopes(scope)) if err != nil { return nil, err } client := &http.Client{ Transport: instrumentedTransport{ - observer: gcsRequestDuration, - next: transport, + observer: gcsRequestDuration, + next: transport, + clientType: clientType, }, } return option.WithHTTPClient(client), nil @@ -73,15 +74,16 @@ func toOptions(opts []grpc.DialOption) []option.ClientOption { } type instrumentedTransport struct { - observer prometheus.ObserverVec - next http.RoundTripper + observer prometheus.ObserverVec + next http.RoundTripper + clientType string } func (i instrumentedTransport) RoundTrip(req *http.Request) (*http.Response, error) { start := time.Now() resp, err := i.next.RoundTrip(req) if err == nil { - i.observer.WithLabelValues(req.Method, strconv.Itoa(resp.StatusCode)).Observe(time.Since(start).Seconds()) + i.observer.WithLabelValues(req.Method, strconv.Itoa(resp.StatusCode), i.clientType).Observe(time.Since(start).Seconds()) } return resp, err } diff --git a/pkg/configs/client/client.go b/pkg/configs/client/client.go index a5528b29ecc..ddffc28cfe3 100644 --- a/pkg/configs/client/client.go +++ b/pkg/configs/client/client.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/url" + "sync" "time" "github.com/cortexproject/cortex/pkg/configs" @@ -14,49 +15,43 @@ import ( "github.com/go-kit/kit/log/level" ) -// Client is what the ruler and altermanger needs from a config store to process rules. -type Client interface { - // GetRules returns all Cortex configurations from a configs API server - // that have been updated after the given configs.ID was last updated. - GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) - - // GetAlerts fetches all the alerts that have changes since since. - GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) -} - // New creates a new ConfigClient. -func New(cfg Config) (Client, error) { +func New(cfg Config) (db.Poller, error) { // All of this falderal is to allow for a smooth transition away from // using the configs server and toward directly connecting to the database. // See https://github.com/cortexproject/cortex/issues/619 if cfg.ConfigsAPIURL.URL != nil { return instrumented{ - next: configsClient{ + next: &configsClient{ URL: cfg.ConfigsAPIURL.URL, Timeout: cfg.ClientTimeout, }, }, nil } - db, err := db.New(cfg.DBConfig) + poller, err := db.New(cfg.DBConfig) if err != nil { return nil, err } - return instrumented{ - next: dbStore{ - db: db, - }, - }, nil + return poller, nil } // configsClient allows retrieving recording and alerting rules from the configs server. type configsClient struct { URL *url.URL Timeout time.Duration + + sync.RWMutex + latestRules configs.ID + latestAlerts configs.ID } // GetRules implements ConfigClient. -func (c configsClient) GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { +func (c *configsClient) GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { + c.RLock() + since := c.latestRules + c.RUnlock() + suffix := "" if since != 0 { suffix = fmt.Sprintf("?since=%d", since) @@ -73,17 +68,35 @@ func (c configsClient) GetRules(ctx context.Context, since configs.ID) (map[stri configs[id] = *cfg } } + + c.Lock() + c.latestRules = response.GetLatestConfigID() + c.Unlock() + return configs, nil } // GetAlerts implements ConfigClient. -func (c configsClient) GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) { +func (c *configsClient) GetAlerts(ctx context.Context) (map[string]configs.View, error) { + c.RLock() + since := c.latestAlerts + c.RUnlock() + suffix := "" if since != 0 { suffix = fmt.Sprintf("?since=%d", since) } endpoint := fmt.Sprintf("%s/private/api/prom/configs/alertmanager%s", c.URL.String(), suffix) - return doRequest(endpoint, c.Timeout, since) + response, err := doRequest(endpoint, c.Timeout, since) + if err != nil { + return nil, err + } + + c.Lock() + c.latestAlerts = response.GetLatestConfigID() + c.Unlock() + + return response.Configs, nil } func doRequest(endpoint string, timeout time.Duration, since configs.ID) (*ConfigsResponse, error) { @@ -113,37 +126,6 @@ func doRequest(endpoint string, timeout time.Duration, since configs.ID) (*Confi return &config, nil } -type dbStore struct { - db db.DB -} - -// GetRules implements ConfigClient. -func (d dbStore) GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { - if since == 0 { - return d.db.GetAllRulesConfigs(ctx) - } - return d.db.GetRulesConfigs(ctx, since) -} - -// GetAlerts implements ConfigClient. -func (d dbStore) GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) { - var resp map[string]configs.View - var err error - if since == 0 { - resp, err = d.db.GetAllConfigs(ctx) - - } - resp, err = d.db.GetConfigs(ctx, since) - if err != nil { - return nil, err - } - - return &ConfigsResponse{ - since: since, - Configs: resp, - }, nil -} - // ConfigsResponse is a response from server for GetConfigs. type ConfigsResponse struct { // The version since which these configs were changed diff --git a/pkg/configs/client/config.go b/pkg/configs/client/config.go index 3850b967c68..677472f2b3a 100644 --- a/pkg/configs/client/config.go +++ b/pkg/configs/client/config.go @@ -46,24 +46,24 @@ func init() { } type instrumented struct { - next Client + next db.Poller } -func (i instrumented) GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { +func (i instrumented) GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { var cfgs map[string]configs.VersionedRulesConfig err := instrument.CollectedRequest(context.Background(), "Configs.GetConfigs", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { var err error - cfgs, err = i.next.GetRules(ctx, since) // Warning: this will produce an incorrect result if the configID ever overflows + cfgs, err = i.next.GetRules(ctx) // Warning: this will produce an incorrect result if the configID ever overflows return err }) return cfgs, err } -func (i instrumented) GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) { - var cfgs *ConfigsResponse +func (i instrumented) GetAlerts(ctx context.Context) (map[string]configs.View, error) { + var cfgs map[string]configs.View err := instrument.CollectedRequest(context.Background(), "Configs.GetConfigs", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { var err error - cfgs, err = i.next.GetAlerts(ctx, since) // Warning: this will produce an incorrect result if the configID ever overflows + cfgs, err = i.next.GetAlerts(ctx) return err }) return cfgs, err diff --git a/pkg/configs/configs.go b/pkg/configs/configs.go index fcf9c593125..8e553407830 100644 --- a/pkg/configs/configs.go +++ b/pkg/configs/configs.go @@ -296,3 +296,21 @@ type VersionedRulesConfig struct { func (vr VersionedRulesConfig) IsDeleted() bool { return !vr.DeletedAt.IsZero() } + +func GetLatestConfigID(cfgs map[string]View, latest ID) ID { + for _, config := range cfgs { + if config.ID > latest { + latest = config.ID + } + } + return latest +} + +func GetLatestRulesID(cfgs map[string]VersionedRulesConfig, latest ID) ID { + for _, config := range cfgs { + if config.ID > latest { + latest = config.ID + } + } + return latest +} diff --git a/pkg/configs/db/db.go b/pkg/configs/db/db.go index b04fcd36978..b5a371ea295 100644 --- a/pkg/configs/db/db.go +++ b/pkg/configs/db/db.go @@ -4,9 +4,8 @@ import ( "context" "flag" "fmt" - "io/ioutil" - "net/url" + "github.com/cortexproject/cortex/pkg/chunk/gcp" "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/configs/db/memory" "github.com/cortexproject/cortex/pkg/configs/db/postgres" @@ -14,9 +13,9 @@ import ( // Config configures the database. type Config struct { - URI string - MigrationsDir string - PasswordFile string + Type string + PostgresConfig postgres.Config + GCSConfig gcp.ConfigDBConfig // Allow injection of mock DBs for unit testing. Mock DB @@ -24,13 +23,15 @@ type Config struct { // RegisterFlags adds the flags required to configure this to the given FlagSet. func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - flag.StringVar(&cfg.URI, "database.uri", "postgres://postgres@configs-db.weave.local/configs?sslmode=disable", "URI where the database can be found (for dev you can use memory://)") - flag.StringVar(&cfg.MigrationsDir, "database.migrations", "", "Path where the database migration files can be found") - flag.StringVar(&cfg.PasswordFile, "database.password-file", "", "File containing password (username goes in URI)") + flag.StringVar(&cfg.Type, "configdb.type", "postgres", "Config database backend to utilize, (postgres, memory, gcp)") + cfg.PostgresConfig.RegisterFlags(f) + cfg.GCSConfig.RegisterFlags(f) } // DB is the interface for the database. type DB interface { + Poller + // GetRulesConfig gets the user's ruler config GetRulesConfig(ctx context.Context, userID string) (configs.VersionedRulesConfig, error) @@ -58,36 +59,31 @@ type DB interface { Close() error } +// Poller is the interface for getting recently updated rules from the database +type Poller interface { + GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) + GetAlerts(ctx context.Context) (map[string]configs.View, error) +} + // New creates a new database. func New(cfg Config) (DB, error) { if cfg.Mock != nil { return cfg.Mock, nil } - u, err := url.Parse(cfg.URI) - if err != nil { - return nil, err - } - - if len(cfg.PasswordFile) != 0 { - if u.User == nil { - return nil, fmt.Errorf("--database.password-file requires username in --database.uri") - } - passwordBytes, err := ioutil.ReadFile(cfg.PasswordFile) - if err != nil { - return nil, fmt.Errorf("Could not read database password file: %v", err) - } - u.User = url.UserPassword(u.User.Username(), string(passwordBytes)) - } - - var d DB - switch u.Scheme { + var ( + d DB + err error + ) + switch cfg.Type { case "memory": - d, err = memory.New(u.String(), cfg.MigrationsDir) + d, err = memory.New() case "postgres": - d, err = postgres.New(u.String(), cfg.MigrationsDir) + d, err = postgres.New(cfg.PostgresConfig) + case "gcs": + d, err = gcp.NewConfigClient(context.TODO(), cfg.GCSConfig) default: - return nil, fmt.Errorf("Unknown database type: %s", u.Scheme) + return nil, fmt.Errorf("Unknown database type: %s", cfg.Type) } if err != nil { return nil, err diff --git a/pkg/configs/db/dbtest/unit.go b/pkg/configs/db/dbtest/unit.go index ec03bb0b064..265565a217b 100644 --- a/pkg/configs/db/dbtest/unit.go +++ b/pkg/configs/db/dbtest/unit.go @@ -15,7 +15,7 @@ import ( func Setup(t *testing.T) db.DB { require.NoError(t, logging.Setup("debug")) database, err := db.New(db.Config{ - URI: "memory://", + Type: "memory", }) require.NoError(t, err) return database diff --git a/pkg/configs/db/memory/memory.go b/pkg/configs/db/memory/memory.go index c2f8bbe2c0c..b8ce83b0f1f 100644 --- a/pkg/configs/db/memory/memory.go +++ b/pkg/configs/db/memory/memory.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "sync" "time" "github.com/cortexproject/cortex/pkg/configs" @@ -13,13 +14,17 @@ import ( type DB struct { cfgs map[string]configs.View id uint + + sync.RWMutex + latestPoll configs.ID } // New creates a new in-memory database -func New(_, _ string) (*DB, error) { +func New() (*DB, error) { return &DB{ - cfgs: map[string]configs.View{}, - id: 0, + cfgs: map[string]configs.View{}, + id: 0, + latestPoll: -1, }, nil } @@ -143,3 +148,37 @@ func (d *DB) GetRulesConfigs(ctx context.Context, since configs.ID) (map[string] } return cfgs, nil } + +func (d *DB) GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { + d.RLock() + since := d.latestPoll + d.RUnlock() + + rls, err := d.GetRulesConfigs(ctx, since) + if err != nil { + return nil, err + } + + d.Lock() + d.latestPoll = configs.GetLatestRulesID(rls, since) + d.Unlock() + + return rls, nil +} + +func (d *DB) GetAlerts(ctx context.Context) (map[string]configs.View, error) { + d.RLock() + since := d.latestPoll + d.RUnlock() + + cfgs, err := d.GetConfigs(ctx, since) + if err != nil { + return nil, err + } + + d.Lock() + d.latestPoll = configs.GetLatestConfigID(cfgs, since) + d.Unlock() + + return cfgs, nil +} diff --git a/pkg/configs/db/postgres/postgres.go b/pkg/configs/db/postgres/postgres.go index 12f8e9b6aa5..27abd4815a2 100644 --- a/pkg/configs/db/postgres/postgres.go +++ b/pkg/configs/db/postgres/postgres.go @@ -4,7 +4,11 @@ import ( "context" "database/sql" "encoding/json" + "flag" "fmt" + "io/ioutil" + "net/url" + "sync" "time" "github.com/Masterminds/squirrel" @@ -34,10 +38,27 @@ var ( } ) +type Config struct { + URI string + MigrationsDir string + PasswordFile string +} + +// RegisterFlags adds the flags required to configure this to the given FlagSet. +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + flag.StringVar(&cfg.URI, "database.uri", "postgres://postgres@configs-db.weave.local/configs?sslmode=disable", "URI where the database can be found") + flag.StringVar(&cfg.MigrationsDir, "database.migrations", "", "Path where the database migration files can be found") + flag.StringVar(&cfg.PasswordFile, "database.password-file", "", "File containing password (username goes in URI)") +} + // DB is a postgres db, for dev and production type DB struct { dbProxy squirrel.StatementBuilderType + + sync.RWMutex + latestAlertPoll configs.ID + latestRulePoll configs.ID } type dbProxy interface { @@ -63,35 +84,53 @@ func dbWait(db *sql.DB) error { } // New creates a new postgres DB -func New(uri, migrationsDir string) (DB, error) { - db, err := sql.Open("postgres", uri) +func New(cfg Config) (*DB, error) { + uri, err := url.Parse(cfg.URI) + if err != nil { + return nil, err + } + + if len(cfg.PasswordFile) != 0 { + if uri.User == nil { + return nil, fmt.Errorf("--database.password-file requires username in --database.uri") + } + passwordBytes, err := ioutil.ReadFile(cfg.PasswordFile) + if err != nil { + return nil, fmt.Errorf("Could not read database password file: %v", err) + } + uri.User = url.UserPassword(uri.User.Username(), string(passwordBytes)) + } + + db, err := sql.Open("postgres", uri.String()) if err != nil { - return DB{}, errors.Wrap(err, "cannot open postgres db") + return nil, errors.Wrap(err, "cannot open postgres db") } if err := dbWait(db); err != nil { - return DB{}, errors.Wrap(err, "cannot establish db connection") + return nil, errors.Wrap(err, "cannot establish db connection") } - if migrationsDir != "" { + if cfg.MigrationsDir != "" { level.Info(util.Logger).Log("msg", "running database migrations...") - if errs, ok := migrate.UpSync(uri, migrationsDir); !ok { + if errs, ok := migrate.UpSync(uri.String(), cfg.MigrationsDir); !ok { for _, err := range errs { level.Error(util.Logger).Log("err", err) } - return DB{}, errors.New("database migrations failed") + return nil, errors.New("database migrations failed") } } - return DB{ + return &DB{ dbProxy: db, StatementBuilderType: statementBuilder(db), + latestAlertPoll: -1, + latestRulePoll: -1, }, err } var statementBuilder = squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar).RunWith -func (d DB) findConfigs(filter squirrel.Sqlizer) (map[string]configs.View, error) { +func (d *DB) findConfigs(filter squirrel.Sqlizer) (map[string]configs.View, error) { rows, err := d.Select("id", "owner_id", "config", "deleted_at"). Options("DISTINCT ON (owner_id)"). From("configs"). @@ -123,7 +162,7 @@ func (d DB) findConfigs(filter squirrel.Sqlizer) (map[string]configs.View, error } // GetConfig gets a configuration. -func (d DB) GetConfig(ctx context.Context, userID string) (configs.View, error) { +func (d *DB) GetConfig(ctx context.Context, userID string) (configs.View, error) { var cfgView configs.View var cfgBytes []byte var deletedAt pq.NullTime @@ -142,7 +181,7 @@ func (d DB) GetConfig(ctx context.Context, userID string) (configs.View, error) } // SetConfig sets a configuration. -func (d DB) SetConfig(ctx context.Context, userID string, cfg configs.Config) error { +func (d *DB) SetConfig(ctx context.Context, userID string, cfg configs.Config) error { if !cfg.RulesConfig.FormatVersion.IsValid() { return fmt.Errorf("invalid rule format version %v", cfg.RulesConfig.FormatVersion) } @@ -159,12 +198,12 @@ func (d DB) SetConfig(ctx context.Context, userID string, cfg configs.Config) er } // GetAllConfigs gets all of the configs. -func (d DB) GetAllConfigs(ctx context.Context) (map[string]configs.View, error) { +func (d *DB) GetAllConfigs(ctx context.Context) (map[string]configs.View, error) { return d.findConfigs(allConfigs) } // GetConfigs gets all of the configs that have changed recently. -func (d DB) GetConfigs(ctx context.Context, since configs.ID) (map[string]configs.View, error) { +func (d *DB) GetConfigs(ctx context.Context, since configs.ID) (map[string]configs.View, error) { return d.findConfigs(squirrel.And{ allConfigs, squirrel.Gt{"id": since}, @@ -172,7 +211,7 @@ func (d DB) GetConfigs(ctx context.Context, since configs.ID) (map[string]config } // GetRulesConfig gets the latest alertmanager config for a user. -func (d DB) GetRulesConfig(ctx context.Context, userID string) (configs.VersionedRulesConfig, error) { +func (d *DB) GetRulesConfig(ctx context.Context, userID string) (configs.VersionedRulesConfig, error) { current, err := d.GetConfig(ctx, userID) if err != nil { return configs.VersionedRulesConfig{}, err @@ -185,9 +224,9 @@ func (d DB) GetRulesConfig(ctx context.Context, userID string) (configs.Versione } // SetRulesConfig sets the current alertmanager config for a user. -func (d DB) SetRulesConfig(ctx context.Context, userID string, oldConfig, newConfig configs.RulesConfig) (bool, error) { +func (d *DB) SetRulesConfig(ctx context.Context, userID string, oldConfig, newConfig configs.RulesConfig) (bool, error) { updated := false - err := d.Transaction(func(tx DB) error { + err := d.Transaction(func(tx *DB) error { current, err := d.GetConfig(ctx, userID) if err != nil && err != sql.ErrNoRows { return err @@ -210,7 +249,7 @@ func (d DB) SetRulesConfig(ctx context.Context, userID string, oldConfig, newCon // findRulesConfigs helps GetAllRulesConfigs and GetRulesConfigs retrieve the // set of all active rules configurations across all our users. -func (d DB) findRulesConfigs(filter squirrel.Sqlizer) (map[string]configs.VersionedRulesConfig, error) { +func (d *DB) findRulesConfigs(filter squirrel.Sqlizer) (map[string]configs.VersionedRulesConfig, error) { rows, err := d.Select("id", "owner_id", "config ->> 'rules_files'", "config ->> 'rule_format_version'", "deleted_at"). Options("DISTINCT ON (owner_id)"). From("configs"). @@ -261,12 +300,12 @@ func (d DB) findRulesConfigs(filter squirrel.Sqlizer) (map[string]configs.Versio } // GetAllRulesConfigs gets all alertmanager configs for all users. -func (d DB) GetAllRulesConfigs(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { +func (d *DB) GetAllRulesConfigs(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { return d.findRulesConfigs(allConfigs) } // GetRulesConfigs gets all the alertmanager configs that have changed since a given config. -func (d DB) GetRulesConfigs(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { +func (d *DB) GetRulesConfigs(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { return d.findRulesConfigs(squirrel.And{ allConfigs, squirrel.Gt{"id": since}, @@ -276,7 +315,7 @@ func (d DB) GetRulesConfigs(ctx context.Context, since configs.ID) (map[string]c // SetDeletedAtConfig sets a deletedAt for configuration // by adding a single new row with deleted_at set // the same as SetConfig is actually insert -func (d DB) SetDeletedAtConfig(ctx context.Context, userID string, deletedAt pq.NullTime, cfg configs.Config) error { +func (d *DB) SetDeletedAtConfig(ctx context.Context, userID string, deletedAt pq.NullTime, cfg configs.Config) error { cfgBytes, err := json.Marshal(cfg) if err != nil { return err @@ -289,7 +328,7 @@ func (d DB) SetDeletedAtConfig(ctx context.Context, userID string, deletedAt pq. } // DeactivateConfig deactivates a configuration. -func (d DB) DeactivateConfig(ctx context.Context, userID string) error { +func (d *DB) DeactivateConfig(ctx context.Context, userID string) error { cfg, err := d.GetConfig(ctx, userID) if err != nil { return err @@ -298,7 +337,7 @@ func (d DB) DeactivateConfig(ctx context.Context, userID string) error { } // RestoreConfig restores configuration. -func (d DB) RestoreConfig(ctx context.Context, userID string) error { +func (d *DB) RestoreConfig(ctx context.Context, userID string) error { cfg, err := d.GetConfig(ctx, userID) if err != nil { return err @@ -308,7 +347,7 @@ func (d DB) RestoreConfig(ctx context.Context, userID string) error { // Transaction runs the given function in a postgres transaction. If fn returns // an error the txn will be rolled back. -func (d DB) Transaction(f func(DB) error) error { +func (d *DB) Transaction(f func(*DB) error) error { if _, ok := d.dbProxy.(*sql.Tx); ok { // Already in a nested transaction return f(d) @@ -318,7 +357,7 @@ func (d DB) Transaction(f func(DB) error) error { if err != nil { return err } - err = f(DB{ + err = f(&DB{ dbProxy: tx, StatementBuilderType: statementBuilder(tx), }) @@ -333,7 +372,7 @@ func (d DB) Transaction(f func(DB) error) error { } // Close finishes using the db -func (d DB) Close() error { +func (d *DB) Close() error { if db, ok := d.dbProxy.(interface { Close() error }); ok { @@ -341,3 +380,39 @@ func (d DB) Close() error { } return nil } + +// GetRules returns all the rule groups added since the previous poll +func (d *DB) GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { + d.RLock() + since := d.latestRulePoll + d.RUnlock() + + rls, err := d.GetRulesConfigs(ctx, since) + if err != nil { + return nil, err + } + + d.Lock() + d.latestRulePoll = configs.GetLatestRulesID(rls, since) + d.Unlock() + + return rls, nil +} + +// GetAlerts returns all the alerts configs added since the previous poll +func (d *DB) GetAlerts(ctx context.Context) (map[string]configs.View, error) { + d.RLock() + since := d.latestAlertPoll + d.RUnlock() + + cfgs, err := d.GetConfigs(ctx, since) + if err != nil { + return nil, err + } + + d.Lock() + d.latestAlertPoll = configs.GetLatestConfigID(cfgs, since) + d.Unlock() + + return cfgs, nil +} diff --git a/pkg/configs/db/timed.go b/pkg/configs/db/timed.go index 54826582dfb..efbeeaa1dd8 100644 --- a/pkg/configs/db/timed.go +++ b/pkg/configs/db/timed.go @@ -138,3 +138,26 @@ func (t timed) GetRulesConfigs(ctx context.Context, since configs.ID) (map[strin return cfgs, err } + +func (t timed) GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { + var cfgs map[string]configs.VersionedRulesConfig + err := instrument.CollectedRequest(ctx, "DB.GetRules", databaseRequestDuration, instrument.ErrorCode, func(ctx context.Context) error { + var err error + cfgs, err = t.d.GetRules(ctx) + return err + }) + + return cfgs, err +} + +func (t timed) GetAlerts(ctx context.Context) (map[string]configs.View, error) { + var ( + cfgs map[string]configs.View + ) + err := instrument.CollectedRequest(ctx, "DB.GetAlerts", databaseRequestDuration, instrument.ErrorCode, func(ctx context.Context) error { + var err error + cfgs, err = t.d.GetAlerts(ctx) + return err + }) + return cfgs, err +} diff --git a/pkg/configs/db/traced.go b/pkg/configs/db/traced.go index a328b64adaa..6d249f3a32a 100644 --- a/pkg/configs/db/traced.go +++ b/pkg/configs/db/traced.go @@ -72,3 +72,13 @@ func (t traced) GetRulesConfigs(ctx context.Context, since configs.ID) (cfgs map defer func() { t.trace("GetConfigs", since, cfgs, err) }() return t.d.GetRulesConfigs(ctx, since) } + +func (t traced) GetRules(ctx context.Context) (cfgs map[string]configs.VersionedRulesConfig, err error) { + defer func() { t.trace("GetRules", cfgs, err) }() + return t.d.GetRules(ctx) +} + +func (t traced) GetAlerts(ctx context.Context) (cfgs map[string]configs.View, err error) { + defer func() { t.trace("GetAlerts", cfgs, err) }() + return t.d.GetAlerts(ctx) +} diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 51d224a7f74..10c5a55c7c5 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -320,12 +320,12 @@ func (t *Cortex) initRuler(cfg *Config) (err error) { cfg.Ruler.LifecyclerConfig.ListenPort = &cfg.Server.GRPCListenPort queryable, engine := querier.New(cfg.Querier, t.distributor, t.store) - rulesAPI, err := config_client.New(cfg.ConfigStore) + poller, err := config_client.New(cfg.ConfigStore) if err != nil { return err } - t.ruler, err = ruler.NewRuler(cfg.Ruler, engine, queryable, t.distributor, rulesAPI) + t.ruler, err = ruler.NewRuler(cfg.Ruler, engine, queryable, t.distributor, poller) if err != nil { return } diff --git a/pkg/ruler/api_test.go b/pkg/ruler/api_test.go index 3d8e4f51c26..8db25989993 100644 --- a/pkg/ruler/api_test.go +++ b/pkg/ruler/api_test.go @@ -16,7 +16,6 @@ import ( "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/configs/api" - "github.com/cortexproject/cortex/pkg/configs/client" "github.com/cortexproject/cortex/pkg/configs/db" "github.com/cortexproject/cortex/pkg/configs/db/dbtest" "github.com/weaveworks/common/user" @@ -27,10 +26,10 @@ const ( ) var ( - app *API - database db.DB - counter int - privateAPI client.Client + app *API + database db.DB + counter int + poller db.Poller ) // setup sets up the environment for the tests. @@ -39,12 +38,7 @@ func setup(t *testing.T) { app = NewAPI(database) counter = 0 var err error - privateAPI, err = client.New(client.Config{ - DBConfig: db.Config{ - URI: "mock", // trigger client.NewConfigClient to use the mock DB. - Mock: database, - }, - }) + poller = database require.NoError(t, err) } @@ -348,7 +342,7 @@ func Test_GetAllConfigs_Empty(t *testing.T) { setup(t) defer cleanup(t) - configs, err := privateAPI.GetRules(context.Background(), 0) + configs, err := poller.GetRules(context.Background()) assert.NoError(t, err, "error getting configs") assert.Equal(t, 0, len(configs)) } @@ -362,7 +356,7 @@ func Test_GetAllConfigs(t *testing.T) { config := makeRulerConfig(configs.RuleFormatV2) view := post(t, userID, configs.RulesConfig{}, config) - found, err := privateAPI.GetRules(context.Background(), 0) + found, err := poller.GetRules(context.Background()) assert.NoError(t, err, "error getting configs") assert.Equal(t, map[string]configs.VersionedRulesConfig{ userID: view, @@ -380,29 +374,13 @@ func Test_GetAllConfigs_Newest(t *testing.T) { config2 := post(t, userID, config1.Config, makeRulerConfig(configs.RuleFormatV2)) lastCreated := post(t, userID, config2.Config, makeRulerConfig(configs.RuleFormatV2)) - found, err := privateAPI.GetRules(context.Background(), 0) + found, err := poller.GetRules(context.Background()) assert.NoError(t, err, "error getting configs") assert.Equal(t, map[string]configs.VersionedRulesConfig{ userID: lastCreated, }, found) } -func Test_GetConfigs_IncludesNewerConfigsAndExcludesOlder(t *testing.T) { - setup(t) - defer cleanup(t) - - post(t, makeUserID(), configs.RulesConfig{}, makeRulerConfig(configs.RuleFormatV2)) - config2 := post(t, makeUserID(), configs.RulesConfig{}, makeRulerConfig(configs.RuleFormatV2)) - userID3 := makeUserID() - config3 := post(t, userID3, configs.RulesConfig{}, makeRulerConfig(configs.RuleFormatV2)) - - found, err := privateAPI.GetRules(context.Background(), config2.ID) - assert.NoError(t, err, "error getting configs") - assert.Equal(t, map[string]configs.VersionedRulesConfig{ - userID3: config3, - }, found) -} - // postAlertmanagerConfig posts an alertmanager config to the alertmanager configs API. func postAlertmanagerConfig(t *testing.T, userID, configFile string) { config := configs.Config{ @@ -441,7 +419,7 @@ func Test_AlertmanagerConfig_NotInAllConfigs(t *testing.T) { - name: noop`) postAlertmanagerConfig(t, makeUserID(), config) - found, err := privateAPI.GetRules(context.Background(), 0) + found, err := poller.GetRules(context.Background()) assert.NoError(t, err, "error getting configs") assert.Equal(t, map[string]configs.VersionedRulesConfig{}, found) } diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 790d0a12f00..6cc1bdc6950 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -22,7 +22,7 @@ import ( "golang.org/x/net/context" "golang.org/x/net/context/ctxhttp" - "github.com/cortexproject/cortex/pkg/configs/client" + "github.com/cortexproject/cortex/pkg/configs/db" "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/util" @@ -130,7 +130,7 @@ type Ruler struct { } // NewRuler creates a new ruler from a distributor and chunk store. -func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d *distributor.Distributor, rulesAPI client.Client) (*Ruler, error) { +func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d *distributor.Distributor, poller db.Poller) (*Ruler, error) { if cfg.NumWorkers <= 0 { return nil, fmt.Errorf("must have at least 1 worker, got %d", cfg.NumWorkers) } @@ -151,7 +151,7 @@ func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d workerWG: &sync.WaitGroup{}, } - ruler.scheduler = newScheduler(rulesAPI, cfg.EvaluationInterval, cfg.EvaluationInterval, ruler.newGroup) + ruler.scheduler = newScheduler(poller, cfg.EvaluationInterval, cfg.EvaluationInterval, ruler.newGroup) // If sharding is enabled, create/join a ring to distribute tokens to // the ruler diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index 51725da5c7b..ee039439dd5 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -12,7 +12,6 @@ import ( "github.com/prometheus/prometheus/promql" "github.com/cortexproject/cortex/pkg/configs" - client_config "github.com/cortexproject/cortex/pkg/configs/client" "github.com/cortexproject/cortex/pkg/querier" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ring/kv/codec" @@ -25,12 +24,12 @@ import ( type mockRuleStore struct{} -func (m *mockRuleStore) GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { +func (m *mockRuleStore) GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { return map[string]configs.VersionedRulesConfig{}, nil } -func (m *mockRuleStore) GetAlerts(ctx context.Context, since configs.ID) (*client_config.ConfigsResponse, error) { - return nil, nil +func (m *mockRuleStore) GetAlerts(ctx context.Context) (map[string]configs.View, error) { + return map[string]configs.View{}, nil } func defaultRulerConfig() Config { diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index 3a9ff8ec704..51dcd2bd117 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -17,7 +17,7 @@ import ( "github.com/prometheus/prometheus/rules" "github.com/cortexproject/cortex/pkg/configs" - config_client "github.com/cortexproject/cortex/pkg/configs/client" + "github.com/cortexproject/cortex/pkg/configs/db" "github.com/cortexproject/cortex/pkg/util" ) @@ -85,7 +85,7 @@ type userConfig struct { type groupFactory func(userID string, groupName string, rls []rules.Rule) (*group, error) type scheduler struct { - ruleStore config_client.Client + poller db.Poller evaluationInterval time.Duration // how often we re-evaluate each rule set q *SchedulingQueue @@ -99,9 +99,9 @@ type scheduler struct { } // newScheduler makes a new scheduler. -func newScheduler(ruleStore config_client.Client, evaluationInterval, pollInterval time.Duration, groupFn groupFactory) *scheduler { +func newScheduler(poller db.Poller, evaluationInterval, pollInterval time.Duration, groupFn groupFactory) *scheduler { return &scheduler{ - ruleStore: ruleStore, + poller: poller, evaluationInterval: evaluationInterval, pollInterval: pollInterval, q: NewSchedulingQueue(clockwork.NewRealClock()), @@ -166,33 +166,14 @@ func (s *scheduler) updateConfigs(now time.Time) error { // poll the configuration server. Not re-entrant. func (s *scheduler) poll() (map[string]configs.VersionedRulesConfig, error) { - s.Lock() - configID := s.latestConfig - s.Unlock() - - cfgs, err := s.ruleStore.GetRules(context.Background(), configID) // Warning: this will produce an incorrect result if the configID ever overflows + cfgs, err := s.poller.GetRules(context.Background()) // Warning: this will produce an incorrect result if the configID ever overflows if err != nil { level.Warn(util.Logger).Log("msg", "scheduler: configs server poll failed", "err", err) return nil, err } - s.Lock() - s.latestConfig = getLatestConfigID(cfgs, configID) - s.Unlock() return cfgs, nil } -// getLatestConfigID gets the latest configs ID. -// max [latest, max (map getID cfgs)] -func getLatestConfigID(cfgs map[string]configs.VersionedRulesConfig, latest configs.ID) configs.ID { - ret := latest - for _, config := range cfgs { - if config.ID > ret { - ret = config.ID - } - } - return ret -} - // computeNextEvalTime Computes when a user's rules should be next evaluated, based on how far we are through an evaluation cycle func (s *scheduler) computeNextEvalTime(hasher hash.Hash64, now time.Time, userID string) time.Time { intervalNanos := float64(s.evaluationInterval.Nanoseconds()) @@ -214,12 +195,9 @@ func (s *scheduler) addNewConfigs(now time.Time, cfgs map[string]configs.Version // TODO: instrument how many configs we have, both valid & invalid. level.Debug(util.Logger).Log("msg", "adding configurations", "num_configs", len(cfgs)) hasher := fnv.New64a() - s.Lock() - generation := s.latestConfig - s.Unlock() for userID, config := range cfgs { - s.addUserConfig(now, hasher, generation, userID, config) + s.addUserConfig(now, hasher, userID, config) } configUpdates.Add(float64(len(cfgs))) @@ -229,7 +207,7 @@ func (s *scheduler) addNewConfigs(now time.Time, cfgs map[string]configs.Version totalConfigs.Set(float64(lenCfgs)) } -func (s *scheduler) addUserConfig(now time.Time, hasher hash.Hash64, generation configs.ID, userID string, config configs.VersionedRulesConfig) { +func (s *scheduler) addUserConfig(now time.Time, hasher hash.Hash64, userID string, config configs.VersionedRulesConfig) { rulesByGroup, err := config.Config.Parse() if err != nil { // XXX: This means that if a user has a working configuration and @@ -248,7 +226,7 @@ func (s *scheduler) addUserConfig(now time.Time, hasher hash.Hash64, generation s.Unlock() return } - s.cfgs[userID] = userConfig{rules: rulesByGroup, generation: generation} + s.cfgs[userID] = userConfig{rules: rulesByGroup, generation: config.ID} s.Unlock() ringHasher := fnv.New32a() @@ -267,7 +245,7 @@ func (s *scheduler) addUserConfig(now time.Time, hasher hash.Hash64, generation ringHasher.Reset() ringHasher.Write([]byte(userID + ":" + group)) hash := ringHasher.Sum32() - workItems = append(workItems, workItem{userID, group, hash, g, evalTime, generation}) + workItems = append(workItems, workItem{userID, group, hash, g, evalTime, config.ID}) } for _, i := range workItems { totalRuleGroups.Inc() From c7aafafff0b7431fef49b79e838e8f0ed588733a Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 14 Jun 2019 00:52:52 -0400 Subject: [PATCH 02/60] fix gcs configdb config function receiver to be pointer Signed-off-by: Jacob Lisi --- pkg/chunk/gcp/gcs_config_client.go | 7 ++++--- pkg/configs/db/db.go | 14 +++++++------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/pkg/chunk/gcp/gcs_config_client.go b/pkg/chunk/gcp/gcs_config_client.go index 1da3f483b37..b6f25295eb5 100644 --- a/pkg/chunk/gcp/gcs_config_client.go +++ b/pkg/chunk/gcp/gcs_config_client.go @@ -35,7 +35,7 @@ type ConfigDBConfig struct { } // RegisterFlags registers flags. -func (cfg ConfigDBConfig) RegisterFlags(f *flag.FlagSet) { +func (cfg *ConfigDBConfig) RegisterFlags(f *flag.FlagSet) { f.StringVar(&cfg.BucketName, "gcs.configdb.bucketname", "", "Name of GCS bucket rule and alert configurations in") } @@ -52,8 +52,9 @@ func NewConfigClient(ctx context.Context, cfg ConfigDBConfig) (*ConfigClient, er } bucket := client.Bucket(cfg.BucketName) - if bucket == nil { - return nil, fmt.Errorf("bucket %v does not exist", cfg.BucketName) + _, err = bucket.Attrs(ctx) + if err != nil { + return nil, fmt.Errorf("unable to access bucket %v, %v", cfg.BucketName, err) } return &ConfigClient{ cfg: cfg, diff --git a/pkg/configs/db/db.go b/pkg/configs/db/db.go index b5a371ea295..1222d5dc6c6 100644 --- a/pkg/configs/db/db.go +++ b/pkg/configs/db/db.go @@ -13,9 +13,9 @@ import ( // Config configures the database. type Config struct { - Type string - PostgresConfig postgres.Config - GCSConfig gcp.ConfigDBConfig + Type string `yaml:"type"` + PostGres postgres.Config `yaml:"postgres,omitempty"` + GCS gcp.ConfigDBConfig `yaml:"gcs,omitempty"` // Allow injection of mock DBs for unit testing. Mock DB @@ -24,8 +24,8 @@ type Config struct { // RegisterFlags adds the flags required to configure this to the given FlagSet. func (cfg *Config) RegisterFlags(f *flag.FlagSet) { flag.StringVar(&cfg.Type, "configdb.type", "postgres", "Config database backend to utilize, (postgres, memory, gcp)") - cfg.PostgresConfig.RegisterFlags(f) - cfg.GCSConfig.RegisterFlags(f) + cfg.PostGres.RegisterFlags(f) + cfg.GCS.RegisterFlags(f) } // DB is the interface for the database. @@ -79,9 +79,9 @@ func New(cfg Config) (DB, error) { case "memory": d, err = memory.New() case "postgres": - d, err = postgres.New(cfg.PostgresConfig) + d, err = postgres.New(cfg.PostGres) case "gcs": - d, err = gcp.NewConfigClient(context.TODO(), cfg.GCSConfig) + d, err = gcp.NewConfigClient(context.TODO(), cfg.GCS) default: return nil, fmt.Errorf("Unknown database type: %s", cfg.Type) } From f8c9eaf750828d461be03d3e5a8b0eb5960d5e19 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 14 Jun 2019 01:00:26 -0400 Subject: [PATCH 03/60] don't return error if no config for user is currenty set Signed-off-by: Jacob Lisi --- pkg/chunk/gcp/gcs_config_client.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/chunk/gcp/gcs_config_client.go b/pkg/chunk/gcp/gcs_config_client.go index b6f25295eb5..49f94cdf8e7 100644 --- a/pkg/chunk/gcp/gcs_config_client.go +++ b/pkg/chunk/gcp/gcs_config_client.go @@ -70,6 +70,9 @@ func (g *ConfigClient) GetRulesConfig(ctx context.Context, userID string) (confi func (g *ConfigClient) getRulesConfig(ctx context.Context, ruleObj string) (configs.VersionedRulesConfig, error) { reader, err := g.bucket.Object(ruleObj).NewReader(ctx) + if err == storage.ErrObjectNotExist { + return configs.VersionedRulesConfig{}, nil + } if err != nil { return configs.VersionedRulesConfig{}, err } From 1ba663c7afed3dd944b1dc0fe3eae190b7e24b82 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 14 Jun 2019 01:14:33 -0400 Subject: [PATCH 04/60] fix adding configs when none exist Signed-off-by: Jacob Lisi --- pkg/chunk/gcp/gcs_config_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/chunk/gcp/gcs_config_client.go b/pkg/chunk/gcp/gcs_config_client.go index 49f94cdf8e7..5a43845cce0 100644 --- a/pkg/chunk/gcp/gcs_config_client.go +++ b/pkg/chunk/gcp/gcs_config_client.go @@ -102,7 +102,7 @@ func (g *ConfigClient) SetRulesConfig(ctx context.Context, userID string, oldCon // The supplied oldConfig must match the current config. If no config // exists, then oldConfig must be nil. Otherwise, it must exactly // equal the existing config. - if oldConfig.Equal(current.Config) { + if current.Config.Files != nil && !oldConfig.Equal(current.Config) { return false, errors.New("old config provided does not match what is currently stored") } From fb55660fa304dae9994867d04763597cd53cfd6b Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 14 Jun 2019 01:30:23 -0400 Subject: [PATCH 05/60] ensure setting configs in gcs performs cas Signed-off-by: Jacob Lisi --- pkg/chunk/gcp/gcs_config_client.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/chunk/gcp/gcs_config_client.go b/pkg/chunk/gcp/gcs_config_client.go index 5a43845cce0..f4184b60297 100644 --- a/pkg/chunk/gcp/gcs_config_client.go +++ b/pkg/chunk/gcp/gcs_config_client.go @@ -111,7 +111,15 @@ func (g *ConfigClient) SetRulesConfig(ctx context.Context, userID string, oldCon return false, err } - writer := g.bucket.Object(rulePrefix + userID).If(storage.Conditions{GenerationMatch: int64(current.ID)}).NewWriter(ctx) + objHandle := g.bucket.Object(rulePrefix + userID) + + if current.Config.Files != nil { + objHandle = objHandle.If(storage.Conditions{GenerationMatch: int64(current.ID)}) + } else { + objHandle = objHandle.If(storage.Conditions{DoesNotExist: true}) + } + + writer := objHandle.NewWriter(ctx) if _, err := writer.Write(cfgBytes); err != nil { return false, err } From 9656eb3d3af164fd47c8d047d21075079c67ac60 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 14 Jun 2019 01:50:35 -0400 Subject: [PATCH 06/60] add debug logging to gcs configdb Signed-off-by: Jacob Lisi --- pkg/chunk/gcp/gcs_config_client.go | 9 ++++++--- pkg/ruler/scheduler.go | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/chunk/gcp/gcs_config_client.go b/pkg/chunk/gcp/gcs_config_client.go index f4184b60297..3278f93d608 100644 --- a/pkg/chunk/gcp/gcs_config_client.go +++ b/pkg/chunk/gcp/gcs_config_client.go @@ -10,10 +10,11 @@ import ( "time" "cloud.google.com/go/storage" - "google.golang.org/api/iterator" - "github.com/cortexproject/cortex/pkg/configs" + "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log/level" "github.com/pkg/errors" + "google.golang.org/api/iterator" ) const ( @@ -71,6 +72,7 @@ func (g *ConfigClient) GetRulesConfig(ctx context.Context, userID string) (confi func (g *ConfigClient) getRulesConfig(ctx context.Context, ruleObj string) (configs.VersionedRulesConfig, error) { reader, err := g.bucket.Object(ruleObj).NewReader(ctx) if err == storage.ErrObjectNotExist { + level.Debug(util.Logger).Log("msg", "rule config does not exist", "name", ruleObj) return configs.VersionedRulesConfig{}, nil } if err != nil { @@ -169,16 +171,17 @@ func (g *ConfigClient) GetRules(ctx context.Context) (map[string]configs.Version ruleMap := map[string]configs.VersionedRulesConfig{} for { objAttrs, err := objs.Next() - if err == iterator.Done { break } + level.Debug(util.Logger).Log("msg", "checking gcs config", "config", objAttrs.Name) if err != nil { return nil, err } if objAttrs.Updated.After(g.lastPolled) { + level.Debug(util.Logger).Log("msg", "adding updated gcs config", "config", objAttrs.Name) rls, err := g.getRulesConfig(ctx, objAttrs.Name) if err != nil { return nil, err diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index 51dcd2bd117..dba13e6d3c1 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -166,7 +166,7 @@ func (s *scheduler) updateConfigs(now time.Time) error { // poll the configuration server. Not re-entrant. func (s *scheduler) poll() (map[string]configs.VersionedRulesConfig, error) { - cfgs, err := s.poller.GetRules(context.Background()) // Warning: this will produce an incorrect result if the configID ever overflows + cfgs, err := s.poller.GetRules(context.Background()) if err != nil { level.Warn(util.Logger).Log("msg", "scheduler: configs server poll failed", "err", err) return nil, err From 865008dbb2a6bd63a3b2ce9d0deeac9b6ab9c8e8 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 14 Jun 2019 02:01:51 -0400 Subject: [PATCH 07/60] fix unmarshalling rules from gcs Signed-off-by: Jacob Lisi --- pkg/chunk/gcp/gcs_config_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/chunk/gcp/gcs_config_client.go b/pkg/chunk/gcp/gcs_config_client.go index 3278f93d608..d835a5d40b7 100644 --- a/pkg/chunk/gcp/gcs_config_client.go +++ b/pkg/chunk/gcp/gcs_config_client.go @@ -86,7 +86,7 @@ func (g *ConfigClient) getRulesConfig(ctx context.Context, ruleObj string) (conf } config := configs.VersionedRulesConfig{} - err = json.Unmarshal(buf, &config) + err = json.Unmarshal(buf, &config.Config) if err != nil { return configs.VersionedRulesConfig{}, err } From dfcbc2f8136b187cbacf6d9d93c481ee5ef7bfd7 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Tue, 25 Jun 2019 18:23:11 -0400 Subject: [PATCH 08/60] initial refactor of config backend apis Signed-off-by: Jacob Lisi --- Makefile | 1 + go.sum | 1 + pkg/alertmanager/api.go | 129 ++ pkg/alertmanager/multitenant.go | 33 +- pkg/chunk/gcp/gcs_config_client.go | 227 ---- pkg/configs/api/api.go | 2 +- pkg/configs/api/api_test.go | 2 +- pkg/configs/api/helpers_test.go | 2 +- pkg/configs/client/client.go | 147 -- pkg/configs/client/config.go | 70 - pkg/configs/configs.go | 350 ++--- pkg/configs/db/db.go | 56 +- pkg/configs/db/dbtest/unit.go | 2 +- pkg/configs/db/memory/memory.go | 47 +- pkg/configs/db/postgres/postgres.go | 127 +- pkg/configs/db/timed.go | 25 +- pkg/configs/db/traced.go | 12 +- pkg/configs/legacy_configs/configs.go | 298 +++++ .../{ => legacy_configs}/configs_test.go | 0 pkg/configs/storage/clients/client/client.go | 202 +++ pkg/configs/storage/clients/client/config.go | 30 + .../clients}/client/configs_test.go | 2 +- pkg/configs/storage/clients/client_test.go | 1 + pkg/configs/storage/clients/gcp/gcs.go | 406 ++++++ pkg/configs/storage/factory.go | 46 + pkg/configs/storage/instrumented.go | 108 ++ pkg/cortex/cortex.go | 4 +- pkg/cortex/modules.go | 32 +- pkg/ruler/api.go | 213 ++- pkg/ruler/api_test.go | 447 ------- pkg/ruler/rulegroup/compat.go | 100 ++ pkg/ruler/rulegroup/rulegroup.go | 36 + pkg/ruler/rulegroup/rulegroup.pb.go | 1184 +++++++++++++++++ pkg/ruler/rulegroup/rulegroup.proto | 32 + pkg/ruler/ruler.go | 6 +- pkg/ruler/ruler_test.go | 89 +- pkg/ruler/scheduler.go | 223 ++-- pkg/ruler/scheduler_test.go | 50 +- 38 files changed, 3129 insertions(+), 1613 deletions(-) create mode 100644 pkg/alertmanager/api.go delete mode 100644 pkg/chunk/gcp/gcs_config_client.go delete mode 100644 pkg/configs/client/client.go delete mode 100644 pkg/configs/client/config.go create mode 100644 pkg/configs/legacy_configs/configs.go rename pkg/configs/{ => legacy_configs}/configs_test.go (100%) create mode 100644 pkg/configs/storage/clients/client/client.go create mode 100644 pkg/configs/storage/clients/client/config.go rename pkg/configs/{ => storage/clients}/client/configs_test.go (94%) create mode 100644 pkg/configs/storage/clients/client_test.go create mode 100644 pkg/configs/storage/clients/gcp/gcs.go create mode 100644 pkg/configs/storage/factory.go create mode 100644 pkg/configs/storage/instrumented.go delete mode 100644 pkg/ruler/api_test.go create mode 100644 pkg/ruler/rulegroup/compat.go create mode 100644 pkg/ruler/rulegroup/rulegroup.go create mode 100644 pkg/ruler/rulegroup/rulegroup.pb.go create mode 100644 pkg/ruler/rulegroup/rulegroup.proto diff --git a/Makefile b/Makefile index fe0aa5c8576..b0727189893 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,7 @@ pkg/ring/ring.pb.go: pkg/ring/ring.proto pkg/querier/frontend/frontend.pb.go: pkg/querier/frontend/frontend.proto pkg/chunk/storage/caching_index_client.pb.go: pkg/chunk/storage/caching_index_client.proto pkg/distributor/ha_tracker.pb.go: pkg/distributor/ha_tracker.proto +pkg/ruler/rulegroup/rulegroup.pb.go: pkg/ruler/rulegroup/rulegroup.proto all: $(UPTODATE_FILES) test: protos mod-check: protos diff --git a/go.sum b/go.sum index ff991c71dee..f949008782d 100644 --- a/go.sum +++ b/go.sum @@ -402,6 +402,7 @@ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/prometheus v0.0.0-20190417125241-3cc5f9d88062 h1:3ssjgrtXRvb5iJmsBgFNAUqQFeU/2MpraZjfw+bPOBs= github.com/prometheus/prometheus v0.0.0-20190417125241-3cc5f9d88062/go.mod h1:nqVG0x8dUO5cW+A9KWkaCaaMx7ERCDanTEjTm9i+g3o= +github.com/prometheus/prometheus v2.5.0+incompatible h1:7QPitgO2kOFG8ecuRn9O/4L9+10He72rVRJvMXrE9Hg= github.com/prometheus/tsdb v0.7.0/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.2-0.20190506134726-2ae028114c89 h1:r2TOLePIzxV3KQHQuFz5wPllXq+9Y3g0pjj989nizJo= github.com/prometheus/tsdb v0.7.2-0.20190506134726-2ae028114c89/go.mod h1:fSI0j+IUQrDd7+ZtR9WKIGtoYAYAJUKcKhYLG25tN4g= diff --git a/pkg/alertmanager/api.go b/pkg/alertmanager/api.go new file mode 100644 index 00000000000..bc0fabd6b2a --- /dev/null +++ b/pkg/alertmanager/api.go @@ -0,0 +1,129 @@ +package alertmanager + +import ( + "io/ioutil" + "net/http" + + "github.com/cortexproject/cortex/pkg/configs" + "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log/level" + "github.com/gorilla/mux" + "github.com/weaveworks/common/user" + "gopkg.in/yaml.v2" +) + +// API is used to provided endpoints to directly interact with the ruler +type API struct { + store configs.AlertStore +} + +// NewAPI returns a ruler API +func NewAPI(store configs.AlertStore) *API { + return &API{store} +} + +// RegisterRoutes registers the configs API HTTP routes with the provided Router. +func (a *API) RegisterRoutes(r *mux.Router) { + for _, route := range []struct { + name, method, path string + handler http.HandlerFunc + }{ + {"get_config", "GET", "/api/prom/alertmanager", a.getConfig}, + {"set_config", "POST", "/api/prom/alertmanager", a.setConfig}, + {"delete_config", "DELETE", "/api/prom/alertmanager", a.deleteConfig}, + } { + r.Handle(route.path, route.handler).Methods(route.method).Name(route.name) + } +} + +func (a *API) getConfig(w http.ResponseWriter, r *http.Request) { + userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) + if err != nil { + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + + logger := util.WithContext(r.Context(), util.Logger) + if err != nil { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if userID == "" { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + + cfg, err := a.store.GetAlertConfig(r.Context(), userID) + + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + d, err := yaml.Marshal(&cfg) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/yaml") + if err := yaml.NewEncoder(w).Encode(d); err != nil { + level.Error(logger).Log("msg", "error marshalling yaml alertmanager config", "err", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) +} + +func (a *API) setConfig(w http.ResponseWriter, r *http.Request) { + userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) + if err != nil { + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + + logger := util.WithContext(r.Context(), util.Logger) + if err != nil { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if userID == "" { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + + payload, err := ioutil.ReadAll(r.Body) + if err != nil { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + cfg := configs.AlertConfig{} + err = yaml.Unmarshal(payload, &cfg) + if err != nil { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + err = a.store.SetAlertConfig(r.Context(), userID, cfg) + if err != nil { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + w.WriteHeader(http.StatusOK) +} + +func (a *API) deleteConfig(w http.ResponseWriter, r *http.Request) { + +} diff --git a/pkg/alertmanager/multitenant.go b/pkg/alertmanager/multitenant.go index e49c7b9f4f3..0410c6ba092 100644 --- a/pkg/alertmanager/multitenant.go +++ b/pkg/alertmanager/multitenant.go @@ -23,8 +23,6 @@ import ( "github.com/weaveworks/mesh" "github.com/cortexproject/cortex/pkg/configs" - configs_client "github.com/cortexproject/cortex/pkg/configs/client" - "github.com/cortexproject/cortex/pkg/configs/db" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" ) @@ -215,7 +213,7 @@ func (cfg *MultitenantAlertmanagerConfig) RegisterFlags(f *flag.FlagSet) { type MultitenantAlertmanager struct { cfg *MultitenantAlertmanagerConfig - configsAPI db.Poller + store configs.AlertStore // The fallback config is stored as a string and parsed every time it's needed // because we mutate the parsed results and don't want those changes to take @@ -223,14 +221,11 @@ type MultitenantAlertmanager struct { fallbackConfig string // All the organization configurations that we have. Only used for instrumentation. - cfgs map[string]configs.Config + cfgs map[string]configs.AlertConfig alertmanagersMtx sync.Mutex alertmanagers map[string]*Alertmanager - latestConfig configs.ID - latestMutex sync.RWMutex - meshRouter *gossipFactory srvDiscovery *srvDiscovery @@ -239,17 +234,12 @@ type MultitenantAlertmanager struct { } // NewMultitenantAlertmanager creates a new MultitenantAlertmanager. -func NewMultitenantAlertmanager(cfg *MultitenantAlertmanagerConfig, cfgCfg configs_client.Config) (*MultitenantAlertmanager, error) { +func NewMultitenantAlertmanager(cfg *MultitenantAlertmanagerConfig, store configs.AlertStore) (*MultitenantAlertmanager, error) { err := os.MkdirAll(cfg.DataDir, 0777) if err != nil { return nil, fmt.Errorf("unable to create Alertmanager data directory %q: %s", cfg.DataDir, err) } - configsAPI, err := configs_client.New(cfgCfg) - if err != nil { - return nil, err - } - mrouter := initMesh(cfg.MeshListenAddr, cfg.MeshHWAddr, cfg.MeshNickname, cfg.MeshPassword) mrouter.Start() @@ -268,9 +258,8 @@ func NewMultitenantAlertmanager(cfg *MultitenantAlertmanagerConfig, cfgCfg confi gf := newGossipFactory(mrouter) am := &MultitenantAlertmanager{ cfg: cfg, - configsAPI: configsAPI, + store: store, fallbackConfig: string(fallbackConfig), - cfgs: map[string]configs.Config{}, alertmanagers: map[string]*Alertmanager{}, meshRouter: &gf, srvDiscovery: newSRVDiscovery(cfg.MeshPeerService, cfg.MeshPeerHost, cfg.MeshPeerRefreshInterval), @@ -327,7 +316,7 @@ func (am *MultitenantAlertmanager) Stop() { // Load the full set of configurations from the server, retrying with backoff // until we can get them. -func (am *MultitenantAlertmanager) loadAllConfigs() map[string]configs.View { +func (am *MultitenantAlertmanager) loadAllConfigs() map[string]configs.AlertConfig { backoff := util.NewBackoff(context.Background(), backoffConfig) for { cfgs, err := am.poll() @@ -350,8 +339,8 @@ func (am *MultitenantAlertmanager) updateConfigs(now time.Time) error { } // poll the configuration server. Not re-entrant. -func (am *MultitenantAlertmanager) poll() (map[string]configs.View, error) { - cfgs, err := am.configsAPI.GetAlerts(context.Background()) +func (am *MultitenantAlertmanager) poll() (map[string]configs.AlertConfig, error) { + cfgs, err := am.store.PollAlerts(context.Background()) if err != nil { level.Warn(util.Logger).Log("msg", "MultitenantAlertmanager: configs server poll failed", "err", err) return nil, err @@ -359,12 +348,12 @@ func (am *MultitenantAlertmanager) poll() (map[string]configs.View, error) { return cfgs, nil } -func (am *MultitenantAlertmanager) addNewConfigs(cfgs map[string]configs.View) { +func (am *MultitenantAlertmanager) addNewConfigs(cfgs map[string]configs.AlertConfig) { // TODO: instrument how many configs we have, both valid & invalid. level.Debug(util.Logger).Log("msg", "adding configurations", "num_configs", len(cfgs)) for userID, config := range cfgs { - err := am.setConfig(userID, config.Config) + err := am.setConfig(userID, config) if err != nil { level.Warn(util.Logger).Log("msg", "MultitenantAlertmanager: error applying config", "err", err) continue @@ -423,7 +412,7 @@ func (am *MultitenantAlertmanager) createTemplatesFile(userID, fn, content strin // setConfig applies the given configuration to the alertmanager for `userID`, // creating an alertmanager if it doesn't already exist. -func (am *MultitenantAlertmanager) setConfig(userID string, config configs.Config) error { +func (am *MultitenantAlertmanager) setConfig(userID string, config configs.AlertConfig) error { _, hasExisting := am.alertmanagers[userID] var amConfig *amconfig.Config var err error @@ -486,7 +475,7 @@ func (am *MultitenantAlertmanager) setConfig(userID string, config configs.Confi } // alertmanagerConfigFromConfig returns the Alertmanager config from the Cortex configuration. -func alertmanagerConfigFromConfig(c configs.Config) (*amconfig.Config, error) { +func alertmanagerConfigFromConfig(c configs.AlertConfig) (*amconfig.Config, error) { cfg, err := amconfig.Load(c.AlertmanagerConfig) if err != nil { return nil, fmt.Errorf("error parsing Alertmanager config: %s", err) diff --git a/pkg/chunk/gcp/gcs_config_client.go b/pkg/chunk/gcp/gcs_config_client.go deleted file mode 100644 index d835a5d40b7..00000000000 --- a/pkg/chunk/gcp/gcs_config_client.go +++ /dev/null @@ -1,227 +0,0 @@ -package gcp - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "strings" - "time" - - "cloud.google.com/go/storage" - "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/util" - "github.com/go-kit/kit/log/level" - "github.com/pkg/errors" - "google.golang.org/api/iterator" -) - -const ( - rulePrefix = "rules/" -) - -type ConfigClient struct { - cfg ConfigDBConfig - - client *storage.Client - bucket *storage.BucketHandle - - lastPolled time.Time -} - -// ConfigDBConfig is config for the GCS Chunk Client. -type ConfigDBConfig struct { - BucketName string `yaml:"bucket_name"` -} - -// RegisterFlags registers flags. -func (cfg *ConfigDBConfig) RegisterFlags(f *flag.FlagSet) { - f.StringVar(&cfg.BucketName, "gcs.configdb.bucketname", "", "Name of GCS bucket rule and alert configurations in") -} - -// NewConfigClient makes a new chunk.ObjectClient that writes chunks to GCS. -func NewConfigClient(ctx context.Context, cfg ConfigDBConfig) (*ConfigClient, error) { - option, err := gcsInstrumentation(ctx, "configs") - if err != nil { - return nil, err - } - - client, err := storage.NewClient(ctx, option) - if err != nil { - return nil, err - } - - bucket := client.Bucket(cfg.BucketName) - _, err = bucket.Attrs(ctx) - if err != nil { - return nil, fmt.Errorf("unable to access bucket %v, %v", cfg.BucketName, err) - } - return &ConfigClient{ - cfg: cfg, - client: client, - bucket: bucket, - lastPolled: time.Unix(0, 0), - }, nil -} - -func (g *ConfigClient) GetRulesConfig(ctx context.Context, userID string) (configs.VersionedRulesConfig, error) { - return g.getRulesConfig(ctx, rulePrefix+userID) -} - -func (g *ConfigClient) getRulesConfig(ctx context.Context, ruleObj string) (configs.VersionedRulesConfig, error) { - reader, err := g.bucket.Object(ruleObj).NewReader(ctx) - if err == storage.ErrObjectNotExist { - level.Debug(util.Logger).Log("msg", "rule config does not exist", "name", ruleObj) - return configs.VersionedRulesConfig{}, nil - } - if err != nil { - return configs.VersionedRulesConfig{}, err - } - defer reader.Close() - - buf, err := ioutil.ReadAll(reader) - if err != nil { - return configs.VersionedRulesConfig{}, err - } - - config := configs.VersionedRulesConfig{} - err = json.Unmarshal(buf, &config.Config) - if err != nil { - return configs.VersionedRulesConfig{}, err - } - - config.ID = configs.ID(reader.Attrs.Generation) - return config, nil -} - -func (g *ConfigClient) SetRulesConfig(ctx context.Context, userID string, oldConfig configs.RulesConfig, newConfig configs.RulesConfig) (bool, error) { - current, err := g.GetRulesConfig(ctx, userID) - if err != nil { - return false, err - } - - // The supplied oldConfig must match the current config. If no config - // exists, then oldConfig must be nil. Otherwise, it must exactly - // equal the existing config. - if current.Config.Files != nil && !oldConfig.Equal(current.Config) { - return false, errors.New("old config provided does not match what is currently stored") - } - - cfgBytes, err := json.Marshal(newConfig) - if err != nil { - return false, err - } - - objHandle := g.bucket.Object(rulePrefix + userID) - - if current.Config.Files != nil { - objHandle = objHandle.If(storage.Conditions{GenerationMatch: int64(current.ID)}) - } else { - objHandle = objHandle.If(storage.Conditions{DoesNotExist: true}) - } - - writer := objHandle.NewWriter(ctx) - if _, err := writer.Write(cfgBytes); err != nil { - return false, err - } - if err := writer.Close(); err != nil { - return true, err - } - - return true, nil -} - -func (g *ConfigClient) GetAllRulesConfigs(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { - objs := g.bucket.Objects(ctx, &storage.Query{ - Prefix: rulePrefix, - }) - - ruleMap := map[string]configs.VersionedRulesConfig{} - for { - objAttrs, err := objs.Next() - - if err == iterator.Done { - break - } - - if err != nil { - return nil, err - } - - rls, err := g.getRulesConfig(ctx, objAttrs.Name) - if err != nil { - return nil, err - } - ruleMap[strings.TrimPrefix(objAttrs.Name, rulePrefix)] = rls - } - - return ruleMap, nil -} - -func (g *ConfigClient) GetRulesConfigs(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { - panic("not implemented") -} - -func (g *ConfigClient) GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { - objs := g.bucket.Objects(ctx, &storage.Query{ - Prefix: rulePrefix, - }) - - ruleMap := map[string]configs.VersionedRulesConfig{} - for { - objAttrs, err := objs.Next() - if err == iterator.Done { - break - } - level.Debug(util.Logger).Log("msg", "checking gcs config", "config", objAttrs.Name) - - if err != nil { - return nil, err - } - - if objAttrs.Updated.After(g.lastPolled) { - level.Debug(util.Logger).Log("msg", "adding updated gcs config", "config", objAttrs.Name) - rls, err := g.getRulesConfig(ctx, objAttrs.Name) - if err != nil { - return nil, err - } - ruleMap[strings.TrimPrefix(objAttrs.Name, rulePrefix)] = rls - } - } - - g.lastPolled = time.Now() - return ruleMap, nil -} - -func (g *ConfigClient) GetConfig(ctx context.Context, userID string) (configs.View, error) { - panic("not implemented") -} - -func (g *ConfigClient) SetConfig(ctx context.Context, userID string, cfg configs.Config) error { - panic("not implemented") -} - -func (g *ConfigClient) GetAllConfigs(ctx context.Context) (map[string]configs.View, error) { - panic("not implemented") -} - -func (g *ConfigClient) GetConfigs(ctx context.Context, since configs.ID) (map[string]configs.View, error) { - panic("not implemented") -} - -func (g *ConfigClient) DeactivateConfig(ctx context.Context, userID string) error { - panic("not implemented") -} - -func (g *ConfigClient) RestoreConfig(ctx context.Context, userID string) error { - panic("not implemented") -} - -func (g *ConfigClient) Close() error { - panic("not implemented") -} - -func (g *ConfigClient) GetAlerts(ctx context.Context) (map[string]configs.View, error) { - panic("not implemented") -} diff --git a/pkg/configs/api/api.go b/pkg/configs/api/api.go index b72a5d6d47d..55998f394c6 100644 --- a/pkg/configs/api/api.go +++ b/pkg/configs/api/api.go @@ -13,8 +13,8 @@ import ( "github.com/gorilla/mux" amconfig "github.com/prometheus/alertmanager/config" - "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/configs/db" + configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" "github.com/cortexproject/cortex/pkg/util" "github.com/weaveworks/common/user" ) diff --git a/pkg/configs/api/api_test.go b/pkg/configs/api/api_test.go index 45abef4d3b6..1a3ba222b4f 100644 --- a/pkg/configs/api/api_test.go +++ b/pkg/configs/api/api_test.go @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/configs/api" + configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" ) const ( diff --git a/pkg/configs/api/helpers_test.go b/pkg/configs/api/helpers_test.go index b22359c96c4..96286d9ea7e 100644 --- a/pkg/configs/api/helpers_test.go +++ b/pkg/configs/api/helpers_test.go @@ -11,10 +11,10 @@ import ( "github.com/stretchr/testify/require" - "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/configs/api" "github.com/cortexproject/cortex/pkg/configs/db" "github.com/cortexproject/cortex/pkg/configs/db/dbtest" + configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" "github.com/weaveworks/common/user" ) diff --git a/pkg/configs/client/client.go b/pkg/configs/client/client.go deleted file mode 100644 index ddffc28cfe3..00000000000 --- a/pkg/configs/client/client.go +++ /dev/null @@ -1,147 +0,0 @@ -package client - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/url" - "sync" - "time" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/configs/db" - "github.com/cortexproject/cortex/pkg/util" - "github.com/go-kit/kit/log/level" -) - -// New creates a new ConfigClient. -func New(cfg Config) (db.Poller, error) { - // All of this falderal is to allow for a smooth transition away from - // using the configs server and toward directly connecting to the database. - // See https://github.com/cortexproject/cortex/issues/619 - if cfg.ConfigsAPIURL.URL != nil { - return instrumented{ - next: &configsClient{ - URL: cfg.ConfigsAPIURL.URL, - Timeout: cfg.ClientTimeout, - }, - }, nil - } - - poller, err := db.New(cfg.DBConfig) - if err != nil { - return nil, err - } - return poller, nil -} - -// configsClient allows retrieving recording and alerting rules from the configs server. -type configsClient struct { - URL *url.URL - Timeout time.Duration - - sync.RWMutex - latestRules configs.ID - latestAlerts configs.ID -} - -// GetRules implements ConfigClient. -func (c *configsClient) GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { - c.RLock() - since := c.latestRules - c.RUnlock() - - suffix := "" - if since != 0 { - suffix = fmt.Sprintf("?since=%d", since) - } - endpoint := fmt.Sprintf("%s/private/api/prom/configs/rules%s", c.URL.String(), suffix) - response, err := doRequest(endpoint, c.Timeout, since) - if err != nil { - return nil, err - } - configs := map[string]configs.VersionedRulesConfig{} - for id, view := range response.Configs { - cfg := view.GetVersionedRulesConfig() - if cfg != nil { - configs[id] = *cfg - } - } - - c.Lock() - c.latestRules = response.GetLatestConfigID() - c.Unlock() - - return configs, nil -} - -// GetAlerts implements ConfigClient. -func (c *configsClient) GetAlerts(ctx context.Context) (map[string]configs.View, error) { - c.RLock() - since := c.latestAlerts - c.RUnlock() - - suffix := "" - if since != 0 { - suffix = fmt.Sprintf("?since=%d", since) - } - endpoint := fmt.Sprintf("%s/private/api/prom/configs/alertmanager%s", c.URL.String(), suffix) - response, err := doRequest(endpoint, c.Timeout, since) - if err != nil { - return nil, err - } - - c.Lock() - c.latestAlerts = response.GetLatestConfigID() - c.Unlock() - - return response.Configs, nil -} - -func doRequest(endpoint string, timeout time.Duration, since configs.ID) (*ConfigsResponse, error) { - req, err := http.NewRequest("GET", endpoint, nil) - if err != nil { - return nil, err - } - - client := &http.Client{Timeout: timeout} - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("Invalid response from configs server: %v", resp.StatusCode) - } - - var config ConfigsResponse - if err := json.NewDecoder(resp.Body).Decode(&config); err != nil { - level.Error(util.Logger).Log("msg", "configs: couldn't decode JSON body", "err", err) - return nil, err - } - - config.since = since - return &config, nil -} - -// ConfigsResponse is a response from server for GetConfigs. -type ConfigsResponse struct { - // The version since which these configs were changed - since configs.ID - - // Configs maps user ID to their latest configs.View. - Configs map[string]configs.View `json:"configs"` -} - -// GetLatestConfigID returns the last config ID from a set of configs. -func (c ConfigsResponse) GetLatestConfigID() configs.ID { - latest := c.since - for _, config := range c.Configs { - if config.ID > latest { - latest = config.ID - } - } - return latest -} diff --git a/pkg/configs/client/config.go b/pkg/configs/client/config.go deleted file mode 100644 index 677472f2b3a..00000000000 --- a/pkg/configs/client/config.go +++ /dev/null @@ -1,70 +0,0 @@ -package client - -import ( - "context" - "flag" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/weaveworks/common/instrument" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/configs/db" - "github.com/cortexproject/cortex/pkg/util/flagext" -) - -// Config says where we can find the ruler configs. -type Config struct { - DBConfig db.Config - - // DEPRECATED - ConfigsAPIURL flagext.URLValue - - // DEPRECATED. HTTP timeout duration for requests made to the Weave Cloud - // configs service. - ClientTimeout time.Duration -} - -// RegisterFlags adds the flags required to config this to the given FlagSet -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - cfg.DBConfig.RegisterFlags(f) - f.Var(&cfg.ConfigsAPIURL, "ruler.configs.url", "DEPRECATED. URL of configs API server.") - f.DurationVar(&cfg.ClientTimeout, "ruler.client-timeout", 5*time.Second, "DEPRECATED. Timeout for requests to Weave Cloud configs service.") - flag.Var(&cfg.ConfigsAPIURL, "alertmanager.configs.url", "URL of configs API server.") - flag.DurationVar(&cfg.ClientTimeout, "alertmanager.configs.client-timeout", 5*time.Second, "Timeout for requests to Weave Cloud configs service.") -} - -var configsRequestDuration = instrument.NewHistogramCollector(prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "cortex", - Name: "configs_request_duration_seconds", - Help: "Time spent requesting configs.", - Buckets: prometheus.DefBuckets, -}, []string{"operation", "status_code"})) - -func init() { - configsRequestDuration.Register() -} - -type instrumented struct { - next db.Poller -} - -func (i instrumented) GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { - var cfgs map[string]configs.VersionedRulesConfig - err := instrument.CollectedRequest(context.Background(), "Configs.GetConfigs", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - cfgs, err = i.next.GetRules(ctx) // Warning: this will produce an incorrect result if the configID ever overflows - return err - }) - return cfgs, err -} - -func (i instrumented) GetAlerts(ctx context.Context) (map[string]configs.View, error) { - var cfgs map[string]configs.View - err := instrument.CollectedRequest(context.Background(), "Configs.GetConfigs", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - cfgs, err = i.next.GetAlerts(ctx) - return err - }) - return cfgs, err -} diff --git a/pkg/configs/configs.go b/pkg/configs/configs.go index 8e553407830..ed415b3c9f3 100644 --- a/pkg/configs/configs.go +++ b/pkg/configs/configs.go @@ -1,316 +1,120 @@ package configs import ( - "encoding/json" + "context" + "errors" "fmt" - "time" - "github.com/go-kit/kit/log" - "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/rulefmt" - "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/rules" - - legacy_promql "github.com/cortexproject/cortex/pkg/configs/legacy_promql" - "github.com/cortexproject/cortex/pkg/util" ) -// An ID is the ID of a single users's Cortex configuration. When a -// configuration changes, it gets a new ID. -type ID int - -// RuleFormatVersion indicates which Prometheus rule format (v1 vs. v2) to use in parsing. -type RuleFormatVersion int - -const ( - // RuleFormatV1 is the Prometheus 1.x rule format. - RuleFormatV1 RuleFormatVersion = iota - // RuleFormatV2 is the Prometheus 2.x rule format. - RuleFormatV2 RuleFormatVersion = iota +var ( + ErrGroupNotFound = errors.New("group does not exist") + ErrGroupNamespaceNotFound = errors.New("group namespace does not exist") + ErrUserNotFound = errors.New("no rule groups found for user") ) -// IsValid returns whether the rules format version is a valid (known) version. -func (v RuleFormatVersion) IsValid() bool { - switch v { - case RuleFormatV1, RuleFormatV2: - return true - default: - return false - } +// ConfigStore unifies the AlertStore and ConfigStore interface +type ConfigStore interface { + AlertStore + RuleStore } -// MarshalJSON implements json.Marshaler. -func (v RuleFormatVersion) MarshalJSON() ([]byte, error) { - switch v { - case RuleFormatV1: - return json.Marshal("1") - case RuleFormatV2: - return json.Marshal("2") - default: - return nil, fmt.Errorf("unknown rule format version %d", v) - } -} +// AlertStore stores config information and template files to configure alertmanager tenants +type AlertStore interface { + PollAlerts(ctx context.Context) (map[string]AlertConfig, error) -// UnmarshalJSON implements json.Unmarshaler. -func (v *RuleFormatVersion) UnmarshalJSON(data []byte) error { - var s string - if err := json.Unmarshal(data, &s); err != nil { - return err - } - switch s { - case "1": - *v = RuleFormatV1 - case "2": - *v = RuleFormatV2 - default: - return fmt.Errorf("unknown rule format version %q", string(data)) - } - return nil + GetAlertConfig(ctx context.Context, userID string) (AlertConfig, error) + SetAlertConfig(ctx context.Context, userID string, config AlertConfig) error + DeleteAlertConfig(ctx context.Context, userID string) error } -// A Config is a Cortex configuration for a single user. -type Config struct { - // RulesFiles maps from a rules filename to file contents. - RulesConfig RulesConfig - TemplateFiles map[string]string - AlertmanagerConfig string -} +// RuleStoreConditions are used to filter retrieived results from a rule store +type RuleStoreConditions struct { + // UserID specifies to only retrieve rules with this ID + UserID string -// configCompat is a compatibility struct to support old JSON config blobs -// saved in the config DB that didn't have a rule format version yet and -// just had a top-level field for the rule files. -type configCompat struct { - RulesFiles map[string]string `json:"rules_files"` - RuleFormatVersion RuleFormatVersion `json:"rule_format_version"` - TemplateFiles map[string]string `json:"template_files"` - AlertmanagerConfig string `json:"alertmanager_config"` + // Namespaces filters results only rule groups with the specified namespace + // are retreived + Namespace string } -// MarshalJSON implements json.Marshaler. -func (c Config) MarshalJSON() ([]byte, error) { - compat := &configCompat{ - RulesFiles: c.RulesConfig.Files, - RuleFormatVersion: c.RulesConfig.FormatVersion, - TemplateFiles: c.TemplateFiles, - AlertmanagerConfig: c.AlertmanagerConfig, - } +// RuleStore is used to store and retrieve rules +type RuleStore interface { + PollRules(ctx context.Context) (map[string][]RuleGroup, error) - return json.Marshal(compat) + ListRuleGroups(ctx context.Context, options RuleStoreConditions) ([]RuleNamespace, error) + GetRuleGroup(ctx context.Context, userID, namespace, group string) (rulefmt.RuleGroup, error) + SetRuleGroup(ctx context.Context, userID, namespace string, group rulefmt.RuleGroup) error + DeleteRuleGroup(ctx context.Context, userID, namespace string, group string) error } -// UnmarshalJSON implements json.Unmarshaler. -func (c *Config) UnmarshalJSON(data []byte) error { - compat := configCompat{} - if err := json.Unmarshal(data, &compat); err != nil { - return err - } - *c = Config{ - RulesConfig: RulesConfig{ - Files: compat.RulesFiles, - FormatVersion: compat.RuleFormatVersion, - }, - TemplateFiles: compat.TemplateFiles, - AlertmanagerConfig: compat.AlertmanagerConfig, - } - return nil +// AlertConfig is used to configure user alert managers +type AlertConfig struct { + TemplateFiles map[string]string `json:"template_files"` + AlertmanagerConfig string `json:"alertmanager_config"` } -// View is what's returned from the Weave Cloud configs service -// when we ask for all Cortex configurations. -// -// The configs service is essentially a JSON blob store that gives each -// _version_ of a configuration a unique ID and guarantees that later versions -// have greater IDs. -type View struct { - ID ID `json:"id"` - Config Config `json:"config"` - DeletedAt time.Time `json:"deleted_at"` +// RuleGroup is used to retrieve rules from the database to evaluate, +// an interface is used to allow for lazy evaluation implementations +type RuleGroup interface { + Rules(ctx context.Context) ([]rules.Rule, error) + Name() string } -// GetVersionedRulesConfig specializes the view to just the rules config. -func (v View) GetVersionedRulesConfig() *VersionedRulesConfig { - if v.Config.RulesConfig.Files == nil { - return nil - } - return &VersionedRulesConfig{ - ID: v.ID, - Config: v.Config.RulesConfig, - DeletedAt: v.DeletedAt, - } +// RuleNamespace is used to parse a slightly modified prometheus +// rule file format, if no namespace is set, the default namespace +// is used +type RuleNamespace struct { + Namespace string `yaml:"namespace"` + Groups []rulefmt.RuleGroup `yaml:"groups"` } -// RulesConfig is the rules configuration for a particular organization. -type RulesConfig struct { - FormatVersion RuleFormatVersion `json:"format_version"` - Files map[string]string `json:"files"` -} +// Validate each rule in the rule namespace is valid +func (r RuleNamespace) Validate() []error { + set := map[string]struct{}{} + var errs []error -// Equal compares two RulesConfigs for equality. -// -// instance Eq RulesConfig -func (c RulesConfig) Equal(o RulesConfig) bool { - if c.FormatVersion != o.FormatVersion { - return false - } - if len(o.Files) != len(c.Files) { - return false - } - for k, v1 := range c.Files { - v2, ok := o.Files[k] - if !ok || v1 != v2 { - return false + for _, g := range r.Groups { + if g.Name == "" { + errs = append(errs, fmt.Errorf("Groupname should not be empty")) } - } - return true -} -// Parse parses and validates the content of the rule files in a RulesConfig -// according to the passed rule format version. -func (c RulesConfig) Parse() (map[string][]rules.Rule, error) { - switch c.FormatVersion { - case RuleFormatV1: - return c.parseV1() - case RuleFormatV2: - return c.parseV2() - default: - return nil, fmt.Errorf("unknown rule format version %v", c.FormatVersion) - } -} - -// parseV2 parses and validates the content of the rule files in a RulesConfig -// according to the Prometheus 2.x rule format. -// -// NOTE: On one hand, we cannot return fully-fledged lists of rules.Group -// here yet, as creating a rules.Group requires already -// passing in rules.ManagerOptions options (which in turn require a -// notifier, appender, etc.), which we do not want to create simply -// for parsing. On the other hand, we should not return barebones -// rulefmt.RuleGroup sets here either, as only a fully-converted rules.Rule -// is able to track alert states over multiple rule evaluations. The caller -// would otherwise have to ensure to convert the rulefmt.RuleGroup only exactly -// once, not for every evaluation (or risk losing alert pending states). So -// it's probably better to just return a set of rules.Rule here. -func (c RulesConfig) parseV2() (map[string][]rules.Rule, error) { - groups := map[string][]rules.Rule{} - - for fn, content := range c.Files { - rgs, errs := rulefmt.Parse([]byte(content)) - if len(errs) > 0 { - return nil, fmt.Errorf("error parsing %s: %v", fn, errs[0]) + if _, ok := set[g.Name]; ok { + errs = append( + errs, + fmt.Errorf("groupname: \"%s\" is repeated in the same namespace", g.Name), + ) } - for _, rg := range rgs.Groups { - rls := make([]rules.Rule, 0, len(rg.Rules)) - for _, rl := range rg.Rules { - expr, err := promql.ParseExpr(rl.Expr) - if err != nil { - return nil, err - } + set[g.Name] = struct{}{} - if rl.Alert != "" { - rls = append(rls, rules.NewAlertingRule( - rl.Alert, - expr, - time.Duration(rl.For), - labels.FromMap(rl.Labels), - labels.FromMap(rl.Annotations), - true, - log.With(util.Logger, "alert", rl.Alert), - )) - continue - } - rls = append(rls, rules.NewRecordingRule( - rl.Record, - expr, - labels.FromMap(rl.Labels), - )) - } - - // Group names have to be unique in Prometheus, but only within one rules file. - groups[rg.Name+";"+fn] = rls - } + errs = append(errs, ValidateRuleGroup(g)...) } - return groups, nil + return errs } -// parseV1 parses and validates the content of the rule files in a RulesConfig -// according to the Prometheus 1.x rule format. -// -// The same comment about rule groups as on ParseV2() applies here. -func (c RulesConfig) parseV1() (map[string][]rules.Rule, error) { - result := map[string][]rules.Rule{} - for fn, content := range c.Files { - stmts, err := legacy_promql.ParseStmts(content) - if err != nil { - return nil, fmt.Errorf("error parsing %s: %s", fn, err) - } - ra := []rules.Rule{} - for _, stmt := range stmts { - var rule rules.Rule - - switch r := stmt.(type) { - case *legacy_promql.AlertStmt: - // legacy_promql.ParseStmts has parsed the whole rule for us. - // Ideally we'd just use r.Expr and pass that to rules.NewAlertingRule, - // but it is of the type legacy_proql.Expr and not promql.Expr. - // So we convert it back to a string, and then parse it again with the - // upstream parser to get it into the right type. - expr, err := promql.ParseExpr(r.Expr.String()) - if err != nil { - return nil, err - } - - rule = rules.NewAlertingRule( - r.Name, expr, r.Duration, r.Labels, r.Annotations, true, - log.With(util.Logger, "alert", r.Name), - ) - - case *legacy_promql.RecordStmt: - expr, err := promql.ParseExpr(r.Expr.String()) - if err != nil { - return nil, err - } - - rule = rules.NewRecordingRule(r.Name, expr, r.Labels) - - default: - return nil, fmt.Errorf("ruler.GetRules: unknown statement type") +// ValidateRuleGroup validates a rulegroup +func ValidateRuleGroup(g rulefmt.RuleGroup) []error { + var errs []error + for i, r := range g.Rules { + for _, err := range r.Validate() { + var ruleName string + if r.Alert != "" { + ruleName = r.Alert + } else { + ruleName = r.Record } - ra = append(ra, rule) + errs = append(errs, &rulefmt.Error{ + Group: g.Name, + Rule: i, + RuleName: ruleName, + Err: err, + }) } - result[fn] = ra } - return result, nil -} - -// VersionedRulesConfig is a RulesConfig together with a version. -// `data Versioned a = Versioned { id :: ID , config :: a }` -type VersionedRulesConfig struct { - ID ID `json:"id"` - Config RulesConfig `json:"config"` - DeletedAt time.Time `json:"deleted_at"` -} -// IsDeleted tells you if the config is deleted. -func (vr VersionedRulesConfig) IsDeleted() bool { - return !vr.DeletedAt.IsZero() -} - -func GetLatestConfigID(cfgs map[string]View, latest ID) ID { - for _, config := range cfgs { - if config.ID > latest { - latest = config.ID - } - } - return latest -} - -func GetLatestRulesID(cfgs map[string]VersionedRulesConfig, latest ID) ID { - for _, config := range cfgs { - if config.ID > latest { - latest = config.ID - } - } - return latest + return errs } diff --git a/pkg/configs/db/db.go b/pkg/configs/db/db.go index 1222d5dc6c6..15452d44949 100644 --- a/pkg/configs/db/db.go +++ b/pkg/configs/db/db.go @@ -4,18 +4,19 @@ import ( "context" "flag" "fmt" + "io/ioutil" + "net/url" - "github.com/cortexproject/cortex/pkg/chunk/gcp" - "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/configs/db/memory" "github.com/cortexproject/cortex/pkg/configs/db/postgres" + configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" ) // Config configures the database. type Config struct { - Type string `yaml:"type"` - PostGres postgres.Config `yaml:"postgres,omitempty"` - GCS gcp.ConfigDBConfig `yaml:"gcs,omitempty"` + URI string + MigrationsDir string + PasswordFile string // Allow injection of mock DBs for unit testing. Mock DB @@ -23,15 +24,13 @@ type Config struct { // RegisterFlags adds the flags required to configure this to the given FlagSet. func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - flag.StringVar(&cfg.Type, "configdb.type", "postgres", "Config database backend to utilize, (postgres, memory, gcp)") - cfg.PostGres.RegisterFlags(f) - cfg.GCS.RegisterFlags(f) + flag.StringVar(&cfg.URI, "database.uri", "postgres://postgres@configs-db.weave.local/configs?sslmode=disable", "URI where the database can be found (for dev you can use memory://)") + flag.StringVar(&cfg.MigrationsDir, "database.migrations", "", "Path where the database migration files can be found") + flag.StringVar(&cfg.PasswordFile, "database.password-file", "", "File containing password (username goes in URI)") } // DB is the interface for the database. type DB interface { - Poller - // GetRulesConfig gets the user's ruler config GetRulesConfig(ctx context.Context, userID string) (configs.VersionedRulesConfig, error) @@ -59,31 +58,36 @@ type DB interface { Close() error } -// Poller is the interface for getting recently updated rules from the database -type Poller interface { - GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) - GetAlerts(ctx context.Context) (map[string]configs.View, error) -} - // New creates a new database. func New(cfg Config) (DB, error) { if cfg.Mock != nil { return cfg.Mock, nil } - var ( - d DB - err error - ) - switch cfg.Type { + u, err := url.Parse(cfg.URI) + if err != nil { + return nil, err + } + + if len(cfg.PasswordFile) != 0 { + if u.User == nil { + return nil, fmt.Errorf("--database.password-file requires username in --database.uri") + } + passwordBytes, err := ioutil.ReadFile(cfg.PasswordFile) + if err != nil { + return nil, fmt.Errorf("Could not read database password file: %v", err) + } + u.User = url.UserPassword(u.User.Username(), string(passwordBytes)) + } + + var d DB + switch u.Scheme { case "memory": - d, err = memory.New() + d, err = memory.New(u.String(), cfg.MigrationsDir) case "postgres": - d, err = postgres.New(cfg.PostGres) - case "gcs": - d, err = gcp.NewConfigClient(context.TODO(), cfg.GCS) + d, err = postgres.New(u.String(), cfg.MigrationsDir) default: - return nil, fmt.Errorf("Unknown database type: %s", cfg.Type) + return nil, fmt.Errorf("Unknown database type: %s", u.Scheme) } if err != nil { return nil, err diff --git a/pkg/configs/db/dbtest/unit.go b/pkg/configs/db/dbtest/unit.go index 265565a217b..ec03bb0b064 100644 --- a/pkg/configs/db/dbtest/unit.go +++ b/pkg/configs/db/dbtest/unit.go @@ -15,7 +15,7 @@ import ( func Setup(t *testing.T) db.DB { require.NoError(t, logging.Setup("debug")) database, err := db.New(db.Config{ - Type: "memory", + URI: "memory://", }) require.NoError(t, err) return database diff --git a/pkg/configs/db/memory/memory.go b/pkg/configs/db/memory/memory.go index b8ce83b0f1f..5d3498441d5 100644 --- a/pkg/configs/db/memory/memory.go +++ b/pkg/configs/db/memory/memory.go @@ -4,27 +4,22 @@ import ( "context" "database/sql" "fmt" - "sync" "time" - "github.com/cortexproject/cortex/pkg/configs" + configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" ) // DB is an in-memory database for testing, and local development type DB struct { cfgs map[string]configs.View id uint - - sync.RWMutex - latestPoll configs.ID } // New creates a new in-memory database -func New() (*DB, error) { +func New(_, _ string) (*DB, error) { return &DB{ - cfgs: map[string]configs.View{}, - id: 0, - latestPoll: -1, + cfgs: map[string]configs.View{}, + id: 0, }, nil } @@ -148,37 +143,3 @@ func (d *DB) GetRulesConfigs(ctx context.Context, since configs.ID) (map[string] } return cfgs, nil } - -func (d *DB) GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { - d.RLock() - since := d.latestPoll - d.RUnlock() - - rls, err := d.GetRulesConfigs(ctx, since) - if err != nil { - return nil, err - } - - d.Lock() - d.latestPoll = configs.GetLatestRulesID(rls, since) - d.Unlock() - - return rls, nil -} - -func (d *DB) GetAlerts(ctx context.Context) (map[string]configs.View, error) { - d.RLock() - since := d.latestPoll - d.RUnlock() - - cfgs, err := d.GetConfigs(ctx, since) - if err != nil { - return nil, err - } - - d.Lock() - d.latestPoll = configs.GetLatestConfigID(cfgs, since) - d.Unlock() - - return cfgs, nil -} diff --git a/pkg/configs/db/postgres/postgres.go b/pkg/configs/db/postgres/postgres.go index 27abd4815a2..52dd57b2c60 100644 --- a/pkg/configs/db/postgres/postgres.go +++ b/pkg/configs/db/postgres/postgres.go @@ -4,15 +4,11 @@ import ( "context" "database/sql" "encoding/json" - "flag" "fmt" - "io/ioutil" - "net/url" - "sync" "time" "github.com/Masterminds/squirrel" - "github.com/cortexproject/cortex/pkg/configs" + configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "github.com/lib/pq" @@ -38,27 +34,10 @@ var ( } ) -type Config struct { - URI string - MigrationsDir string - PasswordFile string -} - -// RegisterFlags adds the flags required to configure this to the given FlagSet. -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - flag.StringVar(&cfg.URI, "database.uri", "postgres://postgres@configs-db.weave.local/configs?sslmode=disable", "URI where the database can be found") - flag.StringVar(&cfg.MigrationsDir, "database.migrations", "", "Path where the database migration files can be found") - flag.StringVar(&cfg.PasswordFile, "database.password-file", "", "File containing password (username goes in URI)") -} - // DB is a postgres db, for dev and production type DB struct { dbProxy squirrel.StatementBuilderType - - sync.RWMutex - latestAlertPoll configs.ID - latestRulePoll configs.ID } type dbProxy interface { @@ -84,53 +63,35 @@ func dbWait(db *sql.DB) error { } // New creates a new postgres DB -func New(cfg Config) (*DB, error) { - uri, err := url.Parse(cfg.URI) - if err != nil { - return nil, err - } - - if len(cfg.PasswordFile) != 0 { - if uri.User == nil { - return nil, fmt.Errorf("--database.password-file requires username in --database.uri") - } - passwordBytes, err := ioutil.ReadFile(cfg.PasswordFile) - if err != nil { - return nil, fmt.Errorf("Could not read database password file: %v", err) - } - uri.User = url.UserPassword(uri.User.Username(), string(passwordBytes)) - } - - db, err := sql.Open("postgres", uri.String()) +func New(uri, migrationsDir string) (DB, error) { + db, err := sql.Open("postgres", uri) if err != nil { - return nil, errors.Wrap(err, "cannot open postgres db") + return DB{}, errors.Wrap(err, "cannot open postgres db") } if err := dbWait(db); err != nil { - return nil, errors.Wrap(err, "cannot establish db connection") + return DB{}, errors.Wrap(err, "cannot establish db connection") } - if cfg.MigrationsDir != "" { + if migrationsDir != "" { level.Info(util.Logger).Log("msg", "running database migrations...") - if errs, ok := migrate.UpSync(uri.String(), cfg.MigrationsDir); !ok { + if errs, ok := migrate.UpSync(uri, migrationsDir); !ok { for _, err := range errs { level.Error(util.Logger).Log("err", err) } - return nil, errors.New("database migrations failed") + return DB{}, errors.New("database migrations failed") } } - return &DB{ + return DB{ dbProxy: db, StatementBuilderType: statementBuilder(db), - latestAlertPoll: -1, - latestRulePoll: -1, }, err } var statementBuilder = squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar).RunWith -func (d *DB) findConfigs(filter squirrel.Sqlizer) (map[string]configs.View, error) { +func (d DB) findConfigs(filter squirrel.Sqlizer) (map[string]configs.View, error) { rows, err := d.Select("id", "owner_id", "config", "deleted_at"). Options("DISTINCT ON (owner_id)"). From("configs"). @@ -162,7 +123,7 @@ func (d *DB) findConfigs(filter squirrel.Sqlizer) (map[string]configs.View, erro } // GetConfig gets a configuration. -func (d *DB) GetConfig(ctx context.Context, userID string) (configs.View, error) { +func (d DB) GetConfig(ctx context.Context, userID string) (configs.View, error) { var cfgView configs.View var cfgBytes []byte var deletedAt pq.NullTime @@ -181,7 +142,7 @@ func (d *DB) GetConfig(ctx context.Context, userID string) (configs.View, error) } // SetConfig sets a configuration. -func (d *DB) SetConfig(ctx context.Context, userID string, cfg configs.Config) error { +func (d DB) SetConfig(ctx context.Context, userID string, cfg configs.Config) error { if !cfg.RulesConfig.FormatVersion.IsValid() { return fmt.Errorf("invalid rule format version %v", cfg.RulesConfig.FormatVersion) } @@ -198,12 +159,12 @@ func (d *DB) SetConfig(ctx context.Context, userID string, cfg configs.Config) e } // GetAllConfigs gets all of the configs. -func (d *DB) GetAllConfigs(ctx context.Context) (map[string]configs.View, error) { +func (d DB) GetAllConfigs(ctx context.Context) (map[string]configs.View, error) { return d.findConfigs(allConfigs) } // GetConfigs gets all of the configs that have changed recently. -func (d *DB) GetConfigs(ctx context.Context, since configs.ID) (map[string]configs.View, error) { +func (d DB) GetConfigs(ctx context.Context, since configs.ID) (map[string]configs.View, error) { return d.findConfigs(squirrel.And{ allConfigs, squirrel.Gt{"id": since}, @@ -211,7 +172,7 @@ func (d *DB) GetConfigs(ctx context.Context, since configs.ID) (map[string]confi } // GetRulesConfig gets the latest alertmanager config for a user. -func (d *DB) GetRulesConfig(ctx context.Context, userID string) (configs.VersionedRulesConfig, error) { +func (d DB) GetRulesConfig(ctx context.Context, userID string) (configs.VersionedRulesConfig, error) { current, err := d.GetConfig(ctx, userID) if err != nil { return configs.VersionedRulesConfig{}, err @@ -224,9 +185,9 @@ func (d *DB) GetRulesConfig(ctx context.Context, userID string) (configs.Version } // SetRulesConfig sets the current alertmanager config for a user. -func (d *DB) SetRulesConfig(ctx context.Context, userID string, oldConfig, newConfig configs.RulesConfig) (bool, error) { +func (d DB) SetRulesConfig(ctx context.Context, userID string, oldConfig, newConfig configs.RulesConfig) (bool, error) { updated := false - err := d.Transaction(func(tx *DB) error { + err := d.Transaction(func(tx DB) error { current, err := d.GetConfig(ctx, userID) if err != nil && err != sql.ErrNoRows { return err @@ -249,7 +210,7 @@ func (d *DB) SetRulesConfig(ctx context.Context, userID string, oldConfig, newCo // findRulesConfigs helps GetAllRulesConfigs and GetRulesConfigs retrieve the // set of all active rules configurations across all our users. -func (d *DB) findRulesConfigs(filter squirrel.Sqlizer) (map[string]configs.VersionedRulesConfig, error) { +func (d DB) findRulesConfigs(filter squirrel.Sqlizer) (map[string]configs.VersionedRulesConfig, error) { rows, err := d.Select("id", "owner_id", "config ->> 'rules_files'", "config ->> 'rule_format_version'", "deleted_at"). Options("DISTINCT ON (owner_id)"). From("configs"). @@ -300,12 +261,12 @@ func (d *DB) findRulesConfigs(filter squirrel.Sqlizer) (map[string]configs.Versi } // GetAllRulesConfigs gets all alertmanager configs for all users. -func (d *DB) GetAllRulesConfigs(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { +func (d DB) GetAllRulesConfigs(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { return d.findRulesConfigs(allConfigs) } // GetRulesConfigs gets all the alertmanager configs that have changed since a given config. -func (d *DB) GetRulesConfigs(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { +func (d DB) GetRulesConfigs(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { return d.findRulesConfigs(squirrel.And{ allConfigs, squirrel.Gt{"id": since}, @@ -315,7 +276,7 @@ func (d *DB) GetRulesConfigs(ctx context.Context, since configs.ID) (map[string] // SetDeletedAtConfig sets a deletedAt for configuration // by adding a single new row with deleted_at set // the same as SetConfig is actually insert -func (d *DB) SetDeletedAtConfig(ctx context.Context, userID string, deletedAt pq.NullTime, cfg configs.Config) error { +func (d DB) SetDeletedAtConfig(ctx context.Context, userID string, deletedAt pq.NullTime, cfg configs.Config) error { cfgBytes, err := json.Marshal(cfg) if err != nil { return err @@ -328,7 +289,7 @@ func (d *DB) SetDeletedAtConfig(ctx context.Context, userID string, deletedAt pq } // DeactivateConfig deactivates a configuration. -func (d *DB) DeactivateConfig(ctx context.Context, userID string) error { +func (d DB) DeactivateConfig(ctx context.Context, userID string) error { cfg, err := d.GetConfig(ctx, userID) if err != nil { return err @@ -337,7 +298,7 @@ func (d *DB) DeactivateConfig(ctx context.Context, userID string) error { } // RestoreConfig restores configuration. -func (d *DB) RestoreConfig(ctx context.Context, userID string) error { +func (d DB) RestoreConfig(ctx context.Context, userID string) error { cfg, err := d.GetConfig(ctx, userID) if err != nil { return err @@ -347,7 +308,7 @@ func (d *DB) RestoreConfig(ctx context.Context, userID string) error { // Transaction runs the given function in a postgres transaction. If fn returns // an error the txn will be rolled back. -func (d *DB) Transaction(f func(*DB) error) error { +func (d DB) Transaction(f func(DB) error) error { if _, ok := d.dbProxy.(*sql.Tx); ok { // Already in a nested transaction return f(d) @@ -357,7 +318,7 @@ func (d *DB) Transaction(f func(*DB) error) error { if err != nil { return err } - err = f(&DB{ + err = f(DB{ dbProxy: tx, StatementBuilderType: statementBuilder(tx), }) @@ -372,7 +333,7 @@ func (d *DB) Transaction(f func(*DB) error) error { } // Close finishes using the db -func (d *DB) Close() error { +func (d DB) Close() error { if db, ok := d.dbProxy.(interface { Close() error }); ok { @@ -380,39 +341,3 @@ func (d *DB) Close() error { } return nil } - -// GetRules returns all the rule groups added since the previous poll -func (d *DB) GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { - d.RLock() - since := d.latestRulePoll - d.RUnlock() - - rls, err := d.GetRulesConfigs(ctx, since) - if err != nil { - return nil, err - } - - d.Lock() - d.latestRulePoll = configs.GetLatestRulesID(rls, since) - d.Unlock() - - return rls, nil -} - -// GetAlerts returns all the alerts configs added since the previous poll -func (d *DB) GetAlerts(ctx context.Context) (map[string]configs.View, error) { - d.RLock() - since := d.latestAlertPoll - d.RUnlock() - - cfgs, err := d.GetConfigs(ctx, since) - if err != nil { - return nil, err - } - - d.Lock() - d.latestAlertPoll = configs.GetLatestConfigID(cfgs, since) - d.Unlock() - - return cfgs, nil -} diff --git a/pkg/configs/db/timed.go b/pkg/configs/db/timed.go index efbeeaa1dd8..afc8784e6dc 100644 --- a/pkg/configs/db/timed.go +++ b/pkg/configs/db/timed.go @@ -3,7 +3,7 @@ package db import ( "context" - "github.com/cortexproject/cortex/pkg/configs" + configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" "github.com/prometheus/client_golang/prometheus" "github.com/weaveworks/common/instrument" ) @@ -138,26 +138,3 @@ func (t timed) GetRulesConfigs(ctx context.Context, since configs.ID) (map[strin return cfgs, err } - -func (t timed) GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { - var cfgs map[string]configs.VersionedRulesConfig - err := instrument.CollectedRequest(ctx, "DB.GetRules", databaseRequestDuration, instrument.ErrorCode, func(ctx context.Context) error { - var err error - cfgs, err = t.d.GetRules(ctx) - return err - }) - - return cfgs, err -} - -func (t timed) GetAlerts(ctx context.Context) (map[string]configs.View, error) { - var ( - cfgs map[string]configs.View - ) - err := instrument.CollectedRequest(ctx, "DB.GetAlerts", databaseRequestDuration, instrument.ErrorCode, func(ctx context.Context) error { - var err error - cfgs, err = t.d.GetAlerts(ctx) - return err - }) - return cfgs, err -} diff --git a/pkg/configs/db/traced.go b/pkg/configs/db/traced.go index 6d249f3a32a..5a54a605eb7 100644 --- a/pkg/configs/db/traced.go +++ b/pkg/configs/db/traced.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/cortexproject/cortex/pkg/configs" + configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" ) @@ -72,13 +72,3 @@ func (t traced) GetRulesConfigs(ctx context.Context, since configs.ID) (cfgs map defer func() { t.trace("GetConfigs", since, cfgs, err) }() return t.d.GetRulesConfigs(ctx, since) } - -func (t traced) GetRules(ctx context.Context) (cfgs map[string]configs.VersionedRulesConfig, err error) { - defer func() { t.trace("GetRules", cfgs, err) }() - return t.d.GetRules(ctx) -} - -func (t traced) GetAlerts(ctx context.Context) (cfgs map[string]configs.View, err error) { - defer func() { t.trace("GetAlerts", cfgs, err) }() - return t.d.GetAlerts(ctx) -} diff --git a/pkg/configs/legacy_configs/configs.go b/pkg/configs/legacy_configs/configs.go new file mode 100644 index 00000000000..fcf9c593125 --- /dev/null +++ b/pkg/configs/legacy_configs/configs.go @@ -0,0 +1,298 @@ +package configs + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/go-kit/kit/log" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/rulefmt" + "github.com/prometheus/prometheus/promql" + "github.com/prometheus/prometheus/rules" + + legacy_promql "github.com/cortexproject/cortex/pkg/configs/legacy_promql" + "github.com/cortexproject/cortex/pkg/util" +) + +// An ID is the ID of a single users's Cortex configuration. When a +// configuration changes, it gets a new ID. +type ID int + +// RuleFormatVersion indicates which Prometheus rule format (v1 vs. v2) to use in parsing. +type RuleFormatVersion int + +const ( + // RuleFormatV1 is the Prometheus 1.x rule format. + RuleFormatV1 RuleFormatVersion = iota + // RuleFormatV2 is the Prometheus 2.x rule format. + RuleFormatV2 RuleFormatVersion = iota +) + +// IsValid returns whether the rules format version is a valid (known) version. +func (v RuleFormatVersion) IsValid() bool { + switch v { + case RuleFormatV1, RuleFormatV2: + return true + default: + return false + } +} + +// MarshalJSON implements json.Marshaler. +func (v RuleFormatVersion) MarshalJSON() ([]byte, error) { + switch v { + case RuleFormatV1: + return json.Marshal("1") + case RuleFormatV2: + return json.Marshal("2") + default: + return nil, fmt.Errorf("unknown rule format version %d", v) + } +} + +// UnmarshalJSON implements json.Unmarshaler. +func (v *RuleFormatVersion) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + switch s { + case "1": + *v = RuleFormatV1 + case "2": + *v = RuleFormatV2 + default: + return fmt.Errorf("unknown rule format version %q", string(data)) + } + return nil +} + +// A Config is a Cortex configuration for a single user. +type Config struct { + // RulesFiles maps from a rules filename to file contents. + RulesConfig RulesConfig + TemplateFiles map[string]string + AlertmanagerConfig string +} + +// configCompat is a compatibility struct to support old JSON config blobs +// saved in the config DB that didn't have a rule format version yet and +// just had a top-level field for the rule files. +type configCompat struct { + RulesFiles map[string]string `json:"rules_files"` + RuleFormatVersion RuleFormatVersion `json:"rule_format_version"` + TemplateFiles map[string]string `json:"template_files"` + AlertmanagerConfig string `json:"alertmanager_config"` +} + +// MarshalJSON implements json.Marshaler. +func (c Config) MarshalJSON() ([]byte, error) { + compat := &configCompat{ + RulesFiles: c.RulesConfig.Files, + RuleFormatVersion: c.RulesConfig.FormatVersion, + TemplateFiles: c.TemplateFiles, + AlertmanagerConfig: c.AlertmanagerConfig, + } + + return json.Marshal(compat) +} + +// UnmarshalJSON implements json.Unmarshaler. +func (c *Config) UnmarshalJSON(data []byte) error { + compat := configCompat{} + if err := json.Unmarshal(data, &compat); err != nil { + return err + } + *c = Config{ + RulesConfig: RulesConfig{ + Files: compat.RulesFiles, + FormatVersion: compat.RuleFormatVersion, + }, + TemplateFiles: compat.TemplateFiles, + AlertmanagerConfig: compat.AlertmanagerConfig, + } + return nil +} + +// View is what's returned from the Weave Cloud configs service +// when we ask for all Cortex configurations. +// +// The configs service is essentially a JSON blob store that gives each +// _version_ of a configuration a unique ID and guarantees that later versions +// have greater IDs. +type View struct { + ID ID `json:"id"` + Config Config `json:"config"` + DeletedAt time.Time `json:"deleted_at"` +} + +// GetVersionedRulesConfig specializes the view to just the rules config. +func (v View) GetVersionedRulesConfig() *VersionedRulesConfig { + if v.Config.RulesConfig.Files == nil { + return nil + } + return &VersionedRulesConfig{ + ID: v.ID, + Config: v.Config.RulesConfig, + DeletedAt: v.DeletedAt, + } +} + +// RulesConfig is the rules configuration for a particular organization. +type RulesConfig struct { + FormatVersion RuleFormatVersion `json:"format_version"` + Files map[string]string `json:"files"` +} + +// Equal compares two RulesConfigs for equality. +// +// instance Eq RulesConfig +func (c RulesConfig) Equal(o RulesConfig) bool { + if c.FormatVersion != o.FormatVersion { + return false + } + if len(o.Files) != len(c.Files) { + return false + } + for k, v1 := range c.Files { + v2, ok := o.Files[k] + if !ok || v1 != v2 { + return false + } + } + return true +} + +// Parse parses and validates the content of the rule files in a RulesConfig +// according to the passed rule format version. +func (c RulesConfig) Parse() (map[string][]rules.Rule, error) { + switch c.FormatVersion { + case RuleFormatV1: + return c.parseV1() + case RuleFormatV2: + return c.parseV2() + default: + return nil, fmt.Errorf("unknown rule format version %v", c.FormatVersion) + } +} + +// parseV2 parses and validates the content of the rule files in a RulesConfig +// according to the Prometheus 2.x rule format. +// +// NOTE: On one hand, we cannot return fully-fledged lists of rules.Group +// here yet, as creating a rules.Group requires already +// passing in rules.ManagerOptions options (which in turn require a +// notifier, appender, etc.), which we do not want to create simply +// for parsing. On the other hand, we should not return barebones +// rulefmt.RuleGroup sets here either, as only a fully-converted rules.Rule +// is able to track alert states over multiple rule evaluations. The caller +// would otherwise have to ensure to convert the rulefmt.RuleGroup only exactly +// once, not for every evaluation (or risk losing alert pending states). So +// it's probably better to just return a set of rules.Rule here. +func (c RulesConfig) parseV2() (map[string][]rules.Rule, error) { + groups := map[string][]rules.Rule{} + + for fn, content := range c.Files { + rgs, errs := rulefmt.Parse([]byte(content)) + if len(errs) > 0 { + return nil, fmt.Errorf("error parsing %s: %v", fn, errs[0]) + } + + for _, rg := range rgs.Groups { + rls := make([]rules.Rule, 0, len(rg.Rules)) + for _, rl := range rg.Rules { + expr, err := promql.ParseExpr(rl.Expr) + if err != nil { + return nil, err + } + + if rl.Alert != "" { + rls = append(rls, rules.NewAlertingRule( + rl.Alert, + expr, + time.Duration(rl.For), + labels.FromMap(rl.Labels), + labels.FromMap(rl.Annotations), + true, + log.With(util.Logger, "alert", rl.Alert), + )) + continue + } + rls = append(rls, rules.NewRecordingRule( + rl.Record, + expr, + labels.FromMap(rl.Labels), + )) + } + + // Group names have to be unique in Prometheus, but only within one rules file. + groups[rg.Name+";"+fn] = rls + } + } + + return groups, nil +} + +// parseV1 parses and validates the content of the rule files in a RulesConfig +// according to the Prometheus 1.x rule format. +// +// The same comment about rule groups as on ParseV2() applies here. +func (c RulesConfig) parseV1() (map[string][]rules.Rule, error) { + result := map[string][]rules.Rule{} + for fn, content := range c.Files { + stmts, err := legacy_promql.ParseStmts(content) + if err != nil { + return nil, fmt.Errorf("error parsing %s: %s", fn, err) + } + ra := []rules.Rule{} + for _, stmt := range stmts { + var rule rules.Rule + + switch r := stmt.(type) { + case *legacy_promql.AlertStmt: + // legacy_promql.ParseStmts has parsed the whole rule for us. + // Ideally we'd just use r.Expr and pass that to rules.NewAlertingRule, + // but it is of the type legacy_proql.Expr and not promql.Expr. + // So we convert it back to a string, and then parse it again with the + // upstream parser to get it into the right type. + expr, err := promql.ParseExpr(r.Expr.String()) + if err != nil { + return nil, err + } + + rule = rules.NewAlertingRule( + r.Name, expr, r.Duration, r.Labels, r.Annotations, true, + log.With(util.Logger, "alert", r.Name), + ) + + case *legacy_promql.RecordStmt: + expr, err := promql.ParseExpr(r.Expr.String()) + if err != nil { + return nil, err + } + + rule = rules.NewRecordingRule(r.Name, expr, r.Labels) + + default: + return nil, fmt.Errorf("ruler.GetRules: unknown statement type") + } + ra = append(ra, rule) + } + result[fn] = ra + } + return result, nil +} + +// VersionedRulesConfig is a RulesConfig together with a version. +// `data Versioned a = Versioned { id :: ID , config :: a }` +type VersionedRulesConfig struct { + ID ID `json:"id"` + Config RulesConfig `json:"config"` + DeletedAt time.Time `json:"deleted_at"` +} + +// IsDeleted tells you if the config is deleted. +func (vr VersionedRulesConfig) IsDeleted() bool { + return !vr.DeletedAt.IsZero() +} diff --git a/pkg/configs/configs_test.go b/pkg/configs/legacy_configs/configs_test.go similarity index 100% rename from pkg/configs/configs_test.go rename to pkg/configs/legacy_configs/configs_test.go diff --git a/pkg/configs/storage/clients/client/client.go b/pkg/configs/storage/clients/client/client.go new file mode 100644 index 00000000000..e15f66af866 --- /dev/null +++ b/pkg/configs/storage/clients/client/client.go @@ -0,0 +1,202 @@ +package client + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/cortexproject/cortex/pkg/ruler/rulegroup" + + "github.com/cortexproject/cortex/pkg/configs" + legacy_configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" + "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log/level" + "github.com/prometheus/prometheus/pkg/rulefmt" +) + +// New creates a new ConfigClient. +func New(cfg Config) (configs.ConfigStore, error) { + return &configsClient{ + URL: cfg.ConfigsAPIURL.URL, + Timeout: cfg.ClientTimeout, + + lastPoll: 0, + }, nil +} + +// configsClient allows retrieving recording and alerting rules from the configs server. +type configsClient struct { + URL *url.URL + Timeout time.Duration + + lastPoll legacy_configs.ID +} + +// GetRules implements ConfigClient. +func (c *configsClient) GetRules(ctx context.Context, since legacy_configs.ID) (map[string]legacy_configs.VersionedRulesConfig, error) { + suffix := "" + if since != 0 { + suffix = fmt.Sprintf("?since=%d", since) + } + endpoint := fmt.Sprintf("%s/private/api/prom/configs/rules%s", c.URL.String(), suffix) + response, err := doRequest(endpoint, c.Timeout, since) + if err != nil { + return nil, err + } + configs := map[string]legacy_configs.VersionedRulesConfig{} + for id, view := range response.Configs { + cfg := view.GetVersionedRulesConfig() + if cfg != nil { + configs[id] = *cfg + } + } + return configs, nil +} + +// GetAlerts implements ConfigClient. +func (c *configsClient) GetAlerts(ctx context.Context, since legacy_configs.ID) (*ConfigsResponse, error) { + suffix := "" + if since != 0 { + suffix = fmt.Sprintf("?since=%d", since) + } + endpoint := fmt.Sprintf("%s/private/api/prom/configs/alertmanager%s", c.URL.String(), suffix) + return doRequest(endpoint, c.Timeout, since) +} + +func doRequest(endpoint string, timeout time.Duration, since legacy_configs.ID) (*ConfigsResponse, error) { + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return nil, err + } + + client := &http.Client{Timeout: timeout} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Invalid response from configs server: %v", resp.StatusCode) + } + + var config ConfigsResponse + if err := json.NewDecoder(resp.Body).Decode(&config); err != nil { + level.Error(util.Logger).Log("msg", "configs: couldn't decode JSON body", "err", err) + return nil, err + } + + config.since = since + return &config, nil +} + +// ConfigsResponse is a response from server for GetConfigs. +type ConfigsResponse struct { + // The version since which these configs were changed + since legacy_configs.ID + + // Configs maps user ID to their latest configs.View. + Configs map[string]legacy_configs.View `json:"configs"` +} + +// GetLatestConfigID returns the last config ID from a set of configs. +func (c ConfigsResponse) GetLatestConfigID() legacy_configs.ID { + latest := c.since + for _, config := range c.Configs { + if config.ID > latest { + latest = config.ID + } + } + return latest +} + +func (c *configsClient) PollAlerts(ctx context.Context) (map[string]configs.AlertConfig, error) { + resp, err := c.GetAlerts(ctx, c.lastPoll) + if err != nil { + return nil, err + } + + newConfigs := map[string]configs.AlertConfig{} + for user, c := range resp.Configs { + newConfigs[user] = configs.AlertConfig{ + TemplateFiles: c.Config.TemplateFiles, + AlertmanagerConfig: c.Config.AlertmanagerConfig, + } + } + + c.lastPoll = resp.GetLatestConfigID() + + return newConfigs, nil +} + +// PollRules polls the configdb server and returns the updated rule groups +func (c *configsClient) PollRules(ctx context.Context) (map[string][]configs.RuleGroup, error) { + resp, err := c.GetAlerts(ctx, c.lastPoll) + if err != nil { + return nil, err + } + + newRules := map[string][]configs.RuleGroup{} + + for user, cfg := range resp.Configs { + userRules := []configs.RuleGroup{} + rls := cfg.GetVersionedRulesConfig() + rMap, err := rls.Config.Parse() + if err != nil { + return nil, err + } + for groupSlug, r := range rMap { + name, file := decomposeGroupSlug(groupSlug) + userRules = append(userRules, rulegroup.NewRuleGroup(name, file, user, r)) + } + newRules[user] = userRules + } + + if err != nil { + return nil, err + } + + c.lastPoll = resp.GetLatestConfigID() + + return newRules, nil +} + +// decomposeGroupSlug breaks the group slug from Parse +// into it's group name and file name +func decomposeGroupSlug(slug string) (string, string) { + components := strings.Split(slug, ";") + return components[0], components[1] +} + +func (c *configsClient) GetAlertConfig(ctx context.Context, userID string) (configs.AlertConfig, error) { + return configs.AlertConfig{}, errors.New("remote configdb client does not implement GetAlertConfig") +} + +func (c *configsClient) SetAlertConfig(ctx context.Context, userID string, config configs.AlertConfig) error { + return errors.New("remote configdb client does not implement SetAlertConfig") +} + +func (c *configsClient) DeleteAlertConfig(ctx context.Context, userID string) error { + return errors.New("remote configdb client does not implement DeleteAlertConfig") +} + +func (c *configsClient) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) ([]configs.RuleNamespace, error) { + return nil, errors.New("remote configdb client does not implement ListRule") +} + +func (c *configsClient) GetRuleGroup(ctx context.Context, userID, namespace, group string) (rulefmt.RuleGroup, error) { + return rulefmt.RuleGroup{}, errors.New("remote configdb client does not implement GetRuleGroup") +} + +func (c *configsClient) SetRuleGroup(ctx context.Context, userID, namespace string, group rulefmt.RuleGroup) error { + return errors.New("remote configdb client does not implement SetRuleGroup") +} + +func (c *configsClient) DeleteRuleGroup(ctx context.Context, userID, namespace string, group string) error { + return errors.New("remote configdb client does not implement DeleteRuleGroup") +} diff --git a/pkg/configs/storage/clients/client/config.go b/pkg/configs/storage/clients/client/config.go new file mode 100644 index 00000000000..210abc9124f --- /dev/null +++ b/pkg/configs/storage/clients/client/config.go @@ -0,0 +1,30 @@ +package client + +import ( + "flag" + "time" + + "github.com/cortexproject/cortex/pkg/configs/db" + "github.com/cortexproject/cortex/pkg/util/flagext" +) + +// Config says where we can find the ruler configs. +type Config struct { + DBConfig db.Config + + // DEPRECATED + ConfigsAPIURL flagext.URLValue + + // DEPRECATED. HTTP timeout duration for requests made to the Weave Cloud + // configs service. + ClientTimeout time.Duration +} + +// RegisterFlags adds the flags required to config this to the given FlagSet +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + cfg.DBConfig.RegisterFlags(f) + f.Var(&cfg.ConfigsAPIURL, "ruler.configs.url", "DEPRECATED. URL of configs API server.") + f.DurationVar(&cfg.ClientTimeout, "ruler.client-timeout", 5*time.Second, "DEPRECATED. Timeout for requests to Weave Cloud configs service.") + flag.Var(&cfg.ConfigsAPIURL, "alertmanager.configs.url", "URL of configs API server.") + flag.DurationVar(&cfg.ClientTimeout, "alertmanager.configs.client-timeout", 5*time.Second, "Timeout for requests to Weave Cloud configs service.") +} diff --git a/pkg/configs/client/configs_test.go b/pkg/configs/storage/clients/client/configs_test.go similarity index 94% rename from pkg/configs/client/configs_test.go rename to pkg/configs/storage/clients/client/configs_test.go index af7acf25b45..20ae24b2b73 100644 --- a/pkg/configs/client/configs_test.go +++ b/pkg/configs/storage/clients/client/configs_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/cortexproject/cortex/pkg/configs" + configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" "github.com/stretchr/testify/assert" ) diff --git a/pkg/configs/storage/clients/client_test.go b/pkg/configs/storage/clients/client_test.go new file mode 100644 index 00000000000..2ea39020bba --- /dev/null +++ b/pkg/configs/storage/clients/client_test.go @@ -0,0 +1 @@ +package clients diff --git a/pkg/configs/storage/clients/gcp/gcs.go b/pkg/configs/storage/clients/gcp/gcs.go new file mode 100644 index 00000000000..7917c4668b9 --- /dev/null +++ b/pkg/configs/storage/clients/gcp/gcs.go @@ -0,0 +1,406 @@ +package gcp + +import ( + "context" + "encoding/json" + "flag" + "io/ioutil" + "strings" + "time" + + "github.com/cortexproject/cortex/pkg/ruler/rulegroup" + + "cloud.google.com/go/storage" + "github.com/cortexproject/cortex/pkg/configs" + "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log/level" + "github.com/golang/protobuf/proto" + "github.com/prometheus/prometheus/pkg/rulefmt" + "google.golang.org/api/iterator" +) + +const ( + alertPrefix = "alerts/" + rulePrefix = "rules/" +) + +// GCSConfig is config for the GCS Chunk Client. +type GCSConfig struct { + BucketName string `yaml:"bucket_name"` +} + +// RegisterFlagsWithPrefix registers flags. +func (cfg *GCSConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.StringVar(&cfg.BucketName, prefix+"gcs.bucketname", "", "Name of GCS bucket to put chunks in.") +} + +type gcsConfigClient struct { + client *storage.Client + bucket *storage.BucketHandle + + lastPolled time.Time +} + +// NewGCSConfigClient makes a new chunk.ObjectClient that writes chunks to GCS. +func NewGCSConfigClient(ctx context.Context, cfg GCSConfig) (configs.ConfigStore, error) { + client, err := storage.NewClient(ctx) + if err != nil { + return nil, err + } + bucket := client.Bucket(cfg.BucketName) + return &gcsConfigClient{ + client: client, + bucket: bucket, + + lastPolled: time.Unix(0, 0), + }, nil +} + +func (g *gcsConfigClient) getAlertConfig(ctx context.Context, obj string) (configs.AlertConfig, error) { + reader, err := g.bucket.Object(obj).NewReader(ctx) + if err == storage.ErrObjectNotExist { + level.Debug(util.Logger).Log("msg", "object does not exist", "name", obj) + return configs.AlertConfig{}, nil + } + if err != nil { + return configs.AlertConfig{}, err + } + defer reader.Close() + + buf, err := ioutil.ReadAll(reader) + if err != nil { + return configs.AlertConfig{}, err + } + + config := configs.AlertConfig{} + err = json.Unmarshal(buf, &config) + if err != nil { + return configs.AlertConfig{}, err + } + + return config, nil +} + +func (g *gcsConfigClient) PollAlerts(ctx context.Context) (map[string]configs.AlertConfig, error) { + objs := g.bucket.Objects(ctx, &storage.Query{ + Prefix: rulePrefix, + }) + + alertMap := map[string]configs.AlertConfig{} + for { + objAttrs, err := objs.Next() + if err == iterator.Done { + break + } + level.Debug(util.Logger).Log("msg", "checking gcs config", "config", objAttrs.Name) + + if err != nil { + return nil, err + } + + if objAttrs.Updated.After(g.lastPolled) { + level.Debug(util.Logger).Log("msg", "adding updated gcs config", "config", objAttrs.Name) + rls, err := g.getAlertConfig(ctx, objAttrs.Name) + if err != nil { + return nil, err + } + alertMap[strings.TrimPrefix(objAttrs.Name, alertPrefix)] = rls + } + } + + g.lastPolled = time.Now() + return alertMap, nil +} + +func (g *gcsConfigClient) GetAlertConfig(ctx context.Context, userID string) (configs.AlertConfig, error) { + return g.getAlertConfig(ctx, alertPrefix+userID) +} + +func (g *gcsConfigClient) SetAlertConfig(ctx context.Context, userID string, cfg configs.AlertConfig) error { + cfgBytes, err := json.Marshal(cfg) + if err != nil { + return err + } + + objHandle := g.bucket.Object(alertPrefix + userID) + + writer := objHandle.NewWriter(ctx) + if _, err := writer.Write(cfgBytes); err != nil { + return err + } + + if err := writer.Close(); err != nil { + return err + } + + return nil +} + +func (g *gcsConfigClient) DeleteAlertConfig(ctx context.Context, userID string) error { + return nil +} + +func (g *gcsConfigClient) PollRules(ctx context.Context) (map[string][]configs.RuleGroup, error) { + objs := g.bucket.Objects(ctx, &storage.Query{ + Delimiter: "/", + Prefix: rulePrefix, + }) + + updatedUsers := []string{} + for { + user, err := objs.Next() + if err == iterator.Done { + break + } + level.Debug(util.Logger).Log("msg", "checking gcs for updated rules", "user", user.Name) + + if err != nil { + return nil, err + } + + updated, err := g.checkUser(ctx, user.Name) + if err != nil { + return nil, err + } + + if updated { + updatedUsers = append(updatedUsers, user.Name) + } + } + + ruleMap := map[string][]configs.RuleGroup{} + + for _, user := range updatedUsers { + rgs, err := g.getAllRuleGroups(ctx, user) + if err != nil { + return nil, err + } + + ruleMap[user] = rgs + } + + g.lastPolled = time.Now() + return ruleMap, nil +} + +func (g *gcsConfigClient) checkUser(ctx context.Context, userID string) (bool, error) { + objs := g.bucket.Objects(ctx, &storage.Query{ + Prefix: generateRuleHandle(userID, "", ""), + }) + + for { + rg, err := objs.Next() + if err == iterator.Done { + break + } + level.Debug(util.Logger).Log("msg", "checking gcs config", "config", rg.Name) + + if err != nil { + return false, err + } + + if rg.Updated.After(g.lastPolled) { + level.Debug(util.Logger).Log("msg", "updated rulegroups found", "user", userID) + return true, nil + } + } + + return false, nil +} + +func (g *gcsConfigClient) getAllRuleGroups(ctx context.Context, userID string) ([]configs.RuleGroup, error) { + it := g.bucket.Objects(ctx, &storage.Query{ + Prefix: generateRuleHandle(userID, "", ""), + }) + + rgs := []configs.RuleGroup{} + + for { + obj, err := it.Next() + if err == iterator.Done { + break + } + + if err != nil { + return []configs.RuleGroup{}, err + } + + rgProto, err := g.getRuleGroup(ctx, obj.Name) + if err != nil { + return []configs.RuleGroup{}, err + } + + rg, err := rulegroup.GenerateRuleGroup(userID, rgProto) + if err != nil { + return []configs.RuleGroup{}, err + } + + rgs = append(rgs, rg) + } + + return rgs, nil +} + +func (g *gcsConfigClient) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) ([]configs.RuleNamespace, error) { + it := g.bucket.Objects(ctx, &storage.Query{ + Delimiter: "/", + Prefix: generateRuleHandle(options.UserID, "", ""), + }) + + nss := []configs.RuleNamespace{} + + for { + obj, err := it.Next() + if err == iterator.Done { + break + } + + if err != nil { + return []configs.RuleNamespace{}, err + } + + if obj.Name == options.Namespace && options.Namespace != "" { + + ns, err := g.getRuleNamespace(ctx, options.UserID, obj.Name) + if err != nil { + return []configs.RuleNamespace{}, err + } + + nss = append(nss, ns) + } + } + + return nss, nil +} + +func (g *gcsConfigClient) getRuleNamespace(ctx context.Context, userID string, namespace string) (configs.RuleNamespace, error) { + it := g.bucket.Objects(ctx, &storage.Query{ + Prefix: generateRuleHandle(userID, namespace, ""), + }) + + ns := configs.RuleNamespace{ + Namespace: namespace, + Groups: []rulefmt.RuleGroup{}, + } + + for { + obj, err := it.Next() + if err == iterator.Done { + break + } + + if err != nil { + return configs.RuleNamespace{}, err + } + + rg, err := g.getRuleGroup(ctx, obj.Name) + if err != nil { + return configs.RuleNamespace{}, err + } + + ns.Groups = append(ns.Groups, rulegroup.FromProto(rg)) + } + + return ns, nil +} + +func (g *gcsConfigClient) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (rulefmt.RuleGroup, error) { + handle := generateRuleHandle(userID, namespace, group) + rg, err := g.getRuleGroup(ctx, handle) + if err != nil { + return rulefmt.RuleGroup{}, err + } + + if rg == nil { + return rulefmt.RuleGroup{}, configs.ErrGroupNotFound + } + return rulegroup.FromProto(rg), nil +} + +func (g *gcsConfigClient) getRuleGroup(ctx context.Context, handle string) (*rulegroup.RuleGroup, error) { + reader, err := g.bucket.Object(handle).NewReader(ctx) + if err == storage.ErrObjectNotExist { + level.Debug(util.Logger).Log("msg", "rule group does not exist", "name", handle) + return nil, nil + } + if err != nil { + return nil, err + } + defer reader.Close() + + buf, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + + rg := &rulegroup.RuleGroup{} + + err = proto.Unmarshal(buf, rg) + if err != nil { + return nil, err + } + + return rg, nil +} + +func (g *gcsConfigClient) SetRuleGroup(ctx context.Context, userID string, namespace string, group rulefmt.RuleGroup) error { + rg := rulegroup.ToProto(namespace, group) + rgBytes, err := proto.Marshal(&rg) + if err != nil { + return err + } + + handle := generateRuleHandle(userID, namespace, group.Name) + objHandle := g.bucket.Object(handle) + + writer := objHandle.NewWriter(ctx) + if _, err := writer.Write(rgBytes); err != nil { + return err + } + + if err := writer.Close(); err != nil { + return err + } + + return nil +} + +func (g *gcsConfigClient) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { + handle := generateRuleHandle(userID, namespace, group) + rg, err := g.getRuleGroup(ctx, handle) + if err != nil { + return nil + } + + if rg == nil { + return configs.ErrGroupNotFound + } + + rg.Deleted = true + + rgBytes, err := proto.Marshal(rg) + if err != nil { + return err + } + + objHandle := g.bucket.Object(handle) + + writer := objHandle.NewWriter(ctx) + if _, err := writer.Write(rgBytes); err != nil { + return err + } + + if err := writer.Close(); err != nil { + return err + } + + return nil +} + +func generateRuleHandle(id, namespace, name string) string { + prefix := rulePrefix + "/" + id + "/" + if namespace == "" { + return prefix + } + return prefix + namespace + "/" + name +} diff --git a/pkg/configs/storage/factory.go b/pkg/configs/storage/factory.go new file mode 100644 index 00000000000..847917fe8fc --- /dev/null +++ b/pkg/configs/storage/factory.go @@ -0,0 +1,46 @@ +package storage + +import ( + "context" + "flag" + "fmt" + + "github.com/cortexproject/cortex/pkg/configs" + "github.com/cortexproject/cortex/pkg/configs/storage/clients/client" + "github.com/cortexproject/cortex/pkg/configs/storage/clients/gcp" +) + +// Config is used to config an alertstore +type Config struct { + BackendType string + + ClientConfig client.Config + GCSConfig gcp.GCSConfig +} + +// RegisterFlags registers flags. +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + f.StringVar(&cfg.BackendType, "configdb.backend", "client", "backend to use for storing and retrieving alerts") + cfg.ClientConfig.RegisterFlags(f) + cfg.GCSConfig.RegisterFlagsWithPrefix("configs.", f) +} + +// New returns a configstore +func New(cfg Config) (configs.ConfigStore, error) { + var ( + store configs.ConfigStore + err error + ) + switch cfg.BackendType { + case "client": + store, err = client.New(cfg.ClientConfig) + case "gcp": + store, err = gcp.NewGCSConfigClient(context.Background(), cfg.GCSConfig) + default: + return nil, fmt.Errorf("Unrecognized config storage client %v, choose one of: client, gcp", cfg.BackendType) + } + + return &instrumented{ + next: store, + }, err +} diff --git a/pkg/configs/storage/instrumented.go b/pkg/configs/storage/instrumented.go new file mode 100644 index 00000000000..0aabf394c02 --- /dev/null +++ b/pkg/configs/storage/instrumented.go @@ -0,0 +1,108 @@ +package storage + +import ( + "context" + + "github.com/cortexproject/cortex/pkg/configs" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pkg/rulefmt" + "github.com/weaveworks/common/instrument" +) + +var configsRequestDuration = instrument.NewHistogramCollector(prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "configs_request_duration_seconds", + Help: "Time spent requesting configs.", + Buckets: prometheus.DefBuckets, +}, []string{"operation", "status_code"})) + +func init() { + configsRequestDuration.Register() +} + +type instrumented struct { + next configs.ConfigStore +} + +func (i instrumented) PollAlerts(ctx context.Context) (map[string]configs.AlertConfig, error) { + var cfgs map[string]configs.AlertConfig + err := instrument.CollectedRequest(context.Background(), "Configs.PollAlerts", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { + var err error + cfgs, err = i.next.PollAlerts(ctx) + return err + }) + return cfgs, err +} + +func (i instrumented) GetAlertConfig(ctx context.Context, userID string) (configs.AlertConfig, error) { + var cfg configs.AlertConfig + err := instrument.CollectedRequest(context.Background(), "Configs.GetAlertConfig", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { + var err error + cfg, err = i.next.GetAlertConfig(ctx, userID) + return err + }) + return cfg, err +} + +func (i instrumented) SetAlertConfig(ctx context.Context, userID string, config configs.AlertConfig) error { + return instrument.CollectedRequest(context.Background(), "Configs.SetAlertConfig", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { + var err error + err = i.next.SetAlertConfig(ctx, userID, config) + return err + }) +} + +func (i instrumented) DeleteAlertConfig(ctx context.Context, userID string) error { + return instrument.CollectedRequest(context.Background(), "Configs.DeleteAlertConfig", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { + var err error + err = i.next.DeleteAlertConfig(ctx, userID) + return err + }) +} + +func (i instrumented) PollRules(ctx context.Context) (map[string][]configs.RuleGroup, error) { + var cfgs map[string][]configs.RuleGroup + err := instrument.CollectedRequest(context.Background(), "Configs.PollRules", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { + var err error + cfgs, err = i.next.PollRules(ctx) + return err + }) + + return cfgs, err +} + +func (i instrumented) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) ([]configs.RuleNamespace, error) { + var cfgs []configs.RuleNamespace + err := instrument.CollectedRequest(context.Background(), "Configs.ListRuleGroups", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { + var err error + cfgs, err = i.next.ListRuleGroups(ctx, options) + return err + }) + return cfgs, err +} + +func (i instrumented) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (rulefmt.RuleGroup, error) { + var cfg rulefmt.RuleGroup + err := instrument.CollectedRequest(context.Background(), "Configs.GetRuleGroup", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { + var err error + cfg, err = i.next.GetRuleGroup(ctx, userID, namespace, group) + return err + }) + return cfg, err +} + +func (i instrumented) SetRuleGroup(ctx context.Context, userID string, namespace string, group rulefmt.RuleGroup) error { + return instrument.CollectedRequest(context.Background(), "Configs.SetRuleGroup", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { + var err error + err = i.next.SetRuleGroup(ctx, userID, namespace, group) + return err + }) +} + +func (i instrumented) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { + return instrument.CollectedRequest(context.Background(), "Configs.DeleteRuleGroup", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { + var err error + err = i.next.DeleteRuleGroup(ctx, userID, namespace, group) + return err + }) +} diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index 19604dd3ba0..b1144143bff 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -18,8 +18,8 @@ import ( "github.com/cortexproject/cortex/pkg/chunk/storage" chunk_util "github.com/cortexproject/cortex/pkg/chunk/util" "github.com/cortexproject/cortex/pkg/configs/api" - config_client "github.com/cortexproject/cortex/pkg/configs/client" "github.com/cortexproject/cortex/pkg/configs/db" + config_storage "github.com/cortexproject/cortex/pkg/configs/storage" "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/ingester" "github.com/cortexproject/cortex/pkg/ingester/client" @@ -70,7 +70,7 @@ type Config struct { Encoding encoding.Config `yaml:"-"` // No yaml for this, it only works with flags. Ruler ruler.Config `yaml:"ruler,omitempty"` - ConfigStore config_client.Config `yaml:"config_store,omitempty"` + ConfigStore config_storage.Config `yaml:"config_store,omitempty"` Alertmanager alertmanager.MultitenantAlertmanagerConfig `yaml:"alertmanager,omitempty"` } diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 10c5a55c7c5..ec293631b98 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -21,8 +21,8 @@ import ( "github.com/cortexproject/cortex/pkg/chunk" "github.com/cortexproject/cortex/pkg/chunk/storage" "github.com/cortexproject/cortex/pkg/configs/api" - config_client "github.com/cortexproject/cortex/pkg/configs/client" "github.com/cortexproject/cortex/pkg/configs/db" + config_storage "github.com/cortexproject/cortex/pkg/configs/storage" "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/ingester" "github.com/cortexproject/cortex/pkg/ingester/client" @@ -320,12 +320,12 @@ func (t *Cortex) initRuler(cfg *Config) (err error) { cfg.Ruler.LifecyclerConfig.ListenPort = &cfg.Server.GRPCListenPort queryable, engine := querier.New(cfg.Querier, t.distributor, t.store) - poller, err := config_client.New(cfg.ConfigStore) + store, err := config_storage.New(cfg.ConfigStore) if err != nil { return err } - t.ruler, err = ruler.NewRuler(cfg.Ruler, engine, queryable, t.distributor, poller) + t.ruler, err = ruler.NewRuler(cfg.Ruler, engine, queryable, t.distributor, store) if err != nil { return } @@ -333,11 +333,8 @@ func (t *Cortex) initRuler(cfg *Config) (err error) { // Only serve the API for setting & getting rules configs if we're not // serving configs from the configs API. Allows for smoother // migration. See https://github.com/cortexproject/cortex/issues/619 - if cfg.ConfigStore.ConfigsAPIURL.URL == nil { - a, err := ruler.NewAPIFromConfig(cfg.ConfigStore.DBConfig) - if err != nil { - return err - } + if cfg.ConfigStore.BackendType != "client" { + a := ruler.NewAPI(store) a.RegisterRoutes(t.server.HTTP) } @@ -351,7 +348,7 @@ func (t *Cortex) stopRuler() error { } func (t *Cortex) initConfigs(cfg *Config) (err error) { - t.configDB, err = db.New(cfg.ConfigStore.DBConfig) + t.configDB, err = db.New(cfg.ConfigStore.ClientConfig.DBConfig) if err != nil { return } @@ -367,14 +364,27 @@ func (t *Cortex) stopConfigs() error { } func (t *Cortex) initAlertmanager(cfg *Config) (err error) { - t.alertmanager, err = alertmanager.NewMultitenantAlertmanager(&cfg.Alertmanager, cfg.ConfigStore) + store, err := config_storage.New(cfg.ConfigStore) + if err != nil { + return err + } + + t.alertmanager, err = alertmanager.NewMultitenantAlertmanager(&cfg.Alertmanager, store) if err != nil { return } + + // Only serve the API for setting & getting alert configs if we're not + // serving configs from the configs API. Allows for smoother + // migration. See https://github.com/cortexproject/cortex/issues/619 + if cfg.ConfigStore.BackendType != "client" { + a := alertmanager.NewAPI(store) + a.RegisterRoutes(t.server.HTTP) + } + go t.alertmanager.Run() t.server.HTTP.PathPrefix("/status").Handler(t.alertmanager.GetStatusHandler()) - // TODO this clashed with the queirer and the distributor, so we cannot // run them in the same process. t.server.HTTP.PathPrefix("/api/prom").Handler(middleware.AuthenticateUser.Wrap(t.alertmanager)) diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index aeee8fe5ba2..ea358d3e0d2 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -1,42 +1,33 @@ package ruler import ( - "database/sql" - "encoding/json" - "fmt" + "errors" + "io/ioutil" "net/http" - "github.com/go-kit/kit/log/level" - "github.com/gorilla/mux" + "github.com/prometheus/prometheus/pkg/rulefmt" "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/configs/db" "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log/level" + "github.com/gorilla/mux" "github.com/weaveworks/common/user" + "gopkg.in/yaml.v2" ) -// API implements the configs api. -type API struct { - db db.DB - http.Handler -} +var ( + ErrNoNamespace = errors.New("a namespace must be provided in the url") + ErrNoGroupName = errors.New("a matching group name must be provided in the url") +) -// NewAPIFromConfig makes a new API from our database config. -func NewAPIFromConfig(cfg db.Config) (*API, error) { - db, err := db.New(cfg) - if err != nil { - return nil, err - } - return NewAPI(db), nil +// API is used to provided endpoints to directly interact with the ruler +type API struct { + store configs.RuleStore } -// NewAPI creates a new API. -func NewAPI(db db.DB) *API { - a := &API{db: db} - r := mux.NewRouter() - a.RegisterRoutes(r) - a.Handler = r - return a +// NewAPI returns a ruler API +func NewAPI(store configs.RuleStore) *API { + return &API{store} } // RegisterRoutes registers the configs API HTTP routes with the provided Router. @@ -45,74 +36,190 @@ func (a *API) RegisterRoutes(r *mux.Router) { name, method, path string handler http.HandlerFunc }{ - {"get_rules", "GET", "/api/prom/rules", a.getConfig}, - {"cas_rules", "POST", "/api/prom/rules", a.casConfig}, + {"list_rules", "GET", "/api/prom/rules", a.listRules}, + {"list_rules_namespace", "GET", "/api/prom/rules/{namespace}/", a.listRules}, + {"get_rulegroup", "GET", "/api/prom/rules/{namespace}/{groupName}", a.getRuleGroup}, + {"set_namespace", "POST", "/api/prom/rules/{namespace}/", a.setRuleNamespace}, + {"set_rulegroup", "POST", "/api/prom/rules/{namespace}/", a.setRuleNamespace}, + {"delete_rulegroup", "DELETE", "/api/prom/rules/{namespace}/{groupName}", a.deleteRuleGroup}, } { r.Handle(route.path, route.handler).Methods(route.method).Name(route.name) } } -// getConfig returns the request configuration. -func (a *API) getConfig(w http.ResponseWriter, r *http.Request) { +func (a *API) listRules(w http.ResponseWriter, r *http.Request) { userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) if err != nil { http.Error(w, err.Error(), http.StatusUnauthorized) return } + logger := util.WithContext(r.Context(), util.Logger) + if err != nil { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } - cfg, err := a.db.GetRulesConfig(r.Context(), userID) - if err == sql.ErrNoRows { - http.Error(w, "No configuration", http.StatusNotFound) + if userID == "" { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + + vars := mux.Vars(r) + + rgs, err := a.store.ListRuleGroups(r.Context(), configs.RuleStoreConditions{ + UserID: userID, + Namespace: vars["namespace"], + }) + + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) return - } else if err != nil { - level.Error(logger).Log("msg", "error getting config", "err", err) + } + + d, err := yaml.Marshal(&rgs) + if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(cfg); err != nil { - level.Error(logger).Log("msg", "error encoding config", "err", err) + w.Header().Set("Content-Type", "application/yaml") + if err := yaml.NewEncoder(w).Encode(d); err != nil { + level.Error(logger).Log("msg", "error marshalling yaml rule groups", "err", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } + + w.WriteHeader(http.StatusOK) } -type configUpdateRequest struct { - OldConfig configs.RulesConfig `json:"old_config"` - NewConfig configs.RulesConfig `json:"new_config"` +func (a *API) getRuleGroup(w http.ResponseWriter, r *http.Request) { + +} + +func (a *API) setRuleNamespace(w http.ResponseWriter, r *http.Request) { + userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) + if err != nil { + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + + logger := util.WithContext(r.Context(), util.Logger) + if err != nil { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if userID == "" { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + + vars := mux.Vars(r) + + namespace, set := vars["namespace"] + if !set { + level.Error(logger).Log("err", err.Error()) + http.Error(w, ErrNoNamespace.Error(), http.StatusBadRequest) + return + } + + payload, err := ioutil.ReadAll(r.Body) + if err != nil { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + rgs := []rulefmt.RuleGroup{} + err = yaml.Unmarshal(payload, &rgs) + if err != nil { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + for _, rg := range rgs { + a.store.SetRuleGroup(r.Context(), userID, namespace, rg) + } + w.WriteHeader(http.StatusOK) } -func (a *API) casConfig(w http.ResponseWriter, r *http.Request) { +func (a *API) setRuleGroup(w http.ResponseWriter, r *http.Request) { userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) if err != nil { http.Error(w, err.Error(), http.StatusUnauthorized) return } + logger := util.WithContext(r.Context(), util.Logger) + if err != nil { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } - var updateReq configUpdateRequest - if err := json.NewDecoder(r.Body).Decode(&updateReq); err != nil { - level.Error(logger).Log("msg", "error decoding json body", "err", err) + if userID == "" { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + + vars := mux.Vars(r) + + namespace, set := vars["namespace"] + if !set { + level.Error(logger).Log("err", err.Error()) + http.Error(w, ErrNoNamespace.Error(), http.StatusBadRequest) + return + } + + groupName, set := vars["groupName"] + if !set { + level.Error(logger).Log("err", err.Error()) + http.Error(w, ErrNoGroupName.Error(), http.StatusBadRequest) + return + } + + payload, err := ioutil.ReadAll(r.Body) + if err != nil { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + rg := rulefmt.RuleGroup{} + err = yaml.Unmarshal(payload, &rg) + if err != nil { + level.Error(logger).Log("err", err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) return } - if _, err = updateReq.NewConfig.Parse(); err != nil { - level.Error(logger).Log("msg", "invalid rules", "err", err) - http.Error(w, fmt.Sprintf("Invalid rules: %v", err), http.StatusBadRequest) + // Set the group name to match the name provided in + // the url + rg.Name = groupName + errs := configs.ValidateRuleGroup(rg) + if len(errs) > 0 { + level.Error(logger).Log("err", err.Error()) + http.Error(w, errs[0].Error(), http.StatusBadRequest) return } - updated, err := a.db.SetRulesConfig(r.Context(), userID, updateReq.OldConfig, updateReq.NewConfig) + err = a.store.SetRuleGroup(r.Context(), userID, namespace, rg) if err != nil { - level.Error(logger).Log("msg", "error storing config", "err", err) + level.Error(logger).Log("err", err.Error()) http.Error(w, err.Error(), http.StatusInternalServerError) return } - if !updated { - http.Error(w, "Supplied configuration doesn't match current configuration", http.StatusConflict) - } - w.WriteHeader(http.StatusNoContent) + + w.WriteHeader(http.StatusOK) +} + +func (a *API) deleteRuleGroup(w http.ResponseWriter, r *http.Request) { + } diff --git a/pkg/ruler/api_test.go b/pkg/ruler/api_test.go deleted file mode 100644 index 8db25989993..00000000000 --- a/pkg/ruler/api_test.go +++ /dev/null @@ -1,447 +0,0 @@ -package ruler - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/configs/api" - "github.com/cortexproject/cortex/pkg/configs/db" - "github.com/cortexproject/cortex/pkg/configs/db/dbtest" - "github.com/weaveworks/common/user" -) - -const ( - endpoint = "/api/prom/rules" -) - -var ( - app *API - database db.DB - counter int - poller db.Poller -) - -// setup sets up the environment for the tests. -func setup(t *testing.T) { - database = dbtest.Setup(t) - app = NewAPI(database) - counter = 0 - var err error - poller = database - require.NoError(t, err) -} - -// cleanup cleans up the environment after a test. -func cleanup(t *testing.T) { - dbtest.Cleanup(t, database) -} - -// request makes a request to the configs API. -func request(t *testing.T, handler http.Handler, method, urlStr string, body io.Reader) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - r, err := http.NewRequest(method, urlStr, body) - require.NoError(t, err) - handler.ServeHTTP(w, r) - return w -} - -// requestAsUser makes a request to the configs API as the given user. -func requestAsUser(t *testing.T, handler http.Handler, userID string, method, urlStr string, body io.Reader) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - r, err := http.NewRequest(method, urlStr, body) - require.NoError(t, err) - r = r.WithContext(user.InjectOrgID(r.Context(), userID)) - user.InjectOrgIDIntoHTTPRequest(r.Context(), r) - handler.ServeHTTP(w, r) - return w -} - -// makeString makes a string, guaranteed to be unique within a test. -func makeString(pattern string) string { - counter++ - return fmt.Sprintf(pattern, counter) -} - -// makeUserID makes an arbitrary user ID. Guaranteed to be unique within a test. -func makeUserID() string { - return makeString("user%d") -} - -// makeRulerConfig makes an arbitrary ruler config -func makeRulerConfig(rfv configs.RuleFormatVersion) configs.RulesConfig { - switch rfv { - case configs.RuleFormatV1: - return configs.RulesConfig{ - Files: map[string]string{ - "filename.rules": makeString(` -# Config no. %d. -ALERT ScrapeFailed - IF up != 1 - FOR 10m - LABELS { severity="warning" } - ANNOTATIONS { - summary = "Scrape of {{$labels.job}} (pod: {{$labels.instance}}) failed.", - description = "Prometheus cannot reach the /metrics page on the {{$labels.instance}} pod.", - impact = "We have no monitoring data for {{$labels.job}} - {{$labels.instance}}. At worst, it's completely down. At best, we cannot reliably respond to operational issues.", - dashboardURL = "$${base_url}/admin/prometheus/targets", - } - `), - }, - FormatVersion: configs.RuleFormatV1, - } - case configs.RuleFormatV2: - return configs.RulesConfig{ - Files: map[string]string{ - "filename.rules": makeString(` -# Config no. %d. -groups: -- name: example - rules: - - alert: ScrapeFailed - expr: 'up != 1' - for: 10m - labels: - severity: warning - annotations: - summary: "Scrape of {{$labels.job}} (pod: {{$labels.instance}}) failed." - description: "Prometheus cannot reach the /metrics page on the {{$labels.instance}} pod." - impact: "We have no monitoring data for {{$labels.job}} - {{$labels.instance}}. At worst, it's completely down. At best, we cannot reliably respond to operational issues." - dashboardURL: "$${base_url}/admin/prometheus/targets" - `), - }, - FormatVersion: configs.RuleFormatV2, - } - default: - panic("unknown rule format") - } -} - -// parseVersionedRulesConfig parses a configs.VersionedRulesConfig from JSON. -func parseVersionedRulesConfig(t *testing.T, b []byte) configs.VersionedRulesConfig { - var x configs.VersionedRulesConfig - err := json.Unmarshal(b, &x) - require.NoError(t, err, "Could not unmarshal JSON: %v", string(b)) - return x -} - -// post a config -func post(t *testing.T, userID string, oldConfig configs.RulesConfig, newConfig configs.RulesConfig) configs.VersionedRulesConfig { - updateRequest := configUpdateRequest{ - OldConfig: oldConfig, - NewConfig: newConfig, - } - b, err := json.Marshal(updateRequest) - require.NoError(t, err) - reader := bytes.NewReader(b) - w := requestAsUser(t, app, userID, "POST", endpoint, reader) - require.Equal(t, http.StatusNoContent, w.Code) - return get(t, userID) -} - -// get a config -func get(t *testing.T, userID string) configs.VersionedRulesConfig { - w := requestAsUser(t, app, userID, "GET", endpoint, nil) - return parseVersionedRulesConfig(t, w.Body.Bytes()) -} - -// configs returns 404 if no config has been created yet. -func Test_GetConfig_NotFound(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - w := requestAsUser(t, app, userID, "GET", endpoint, nil) - assert.Equal(t, http.StatusNotFound, w.Code) -} - -// configs returns 401 to requests without authentication. -func Test_PostConfig_Anonymous(t *testing.T) { - setup(t) - defer cleanup(t) - - w := request(t, app, "POST", endpoint, nil) - assert.Equal(t, http.StatusUnauthorized, w.Code) -} - -// Posting to a configuration sets it so that you can get it again. -func Test_PostConfig_CreatesConfig(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - config := makeRulerConfig(configs.RuleFormatV2) - result := post(t, userID, configs.RulesConfig{}, config) - assert.Equal(t, config, result.Config) -} - -// Posting an invalid config when there's none set returns an error and leaves the config unset. -func Test_PostConfig_InvalidNewConfig(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - invalidConfig := configs.RulesConfig{ - Files: map[string]string{ - "some.rules": "invalid config", - }, - FormatVersion: configs.RuleFormatV2, - } - updateRequest := configUpdateRequest{ - OldConfig: configs.RulesConfig{}, - NewConfig: invalidConfig, - } - b, err := json.Marshal(updateRequest) - require.NoError(t, err) - reader := bytes.NewReader(b) - { - w := requestAsUser(t, app, userID, "POST", endpoint, reader) - require.Equal(t, http.StatusBadRequest, w.Code) - } - { - w := requestAsUser(t, app, userID, "GET", endpoint, nil) - require.Equal(t, http.StatusNotFound, w.Code) - } -} - -// Posting a v1 rule format configuration sets it so that you can get it again. -func Test_PostConfig_UpdatesConfig_V1RuleFormat(t *testing.T) { - setup(t) - app = NewAPI(database) - defer cleanup(t) - - userID := makeUserID() - config1 := makeRulerConfig(configs.RuleFormatV1) - view1 := post(t, userID, configs.RulesConfig{}, config1) - config2 := makeRulerConfig(configs.RuleFormatV1) - view2 := post(t, userID, config1, config2) - assert.True(t, view2.ID > view1.ID, "%v > %v", view2.ID, view1.ID) - assert.Equal(t, config2, view2.Config) -} - -// Posting an invalid v1 rule format config when there's one already set returns an error and leaves the config as is. -func Test_PostConfig_InvalidChangedConfig_V1RuleFormat(t *testing.T) { - setup(t) - app = NewAPI(database) - defer cleanup(t) - - userID := makeUserID() - config := makeRulerConfig(configs.RuleFormatV1) - post(t, userID, configs.RulesConfig{}, config) - invalidConfig := configs.RulesConfig{ - Files: map[string]string{ - "some.rules": "invalid config", - }, - FormatVersion: configs.RuleFormatV1, - } - updateRequest := configUpdateRequest{ - OldConfig: configs.RulesConfig{}, - NewConfig: invalidConfig, - } - b, err := json.Marshal(updateRequest) - require.NoError(t, err) - reader := bytes.NewReader(b) - { - w := requestAsUser(t, app, userID, "POST", endpoint, reader) - require.Equal(t, http.StatusBadRequest, w.Code) - } - result := get(t, userID) - assert.Equal(t, config, result.Config) -} - -// Posting a v2 rule format configuration sets it so that you can get it again. -func Test_PostConfig_UpdatesConfig_V2RuleFormat(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - config1 := makeRulerConfig(configs.RuleFormatV2) - view1 := post(t, userID, configs.RulesConfig{}, config1) - config2 := makeRulerConfig(configs.RuleFormatV2) - view2 := post(t, userID, config1, config2) - assert.True(t, view2.ID > view1.ID, "%v > %v", view2.ID, view1.ID) - assert.Equal(t, config2, view2.Config) -} - -// Posting an invalid v2 rule format config when there's one already set returns an error and leaves the config as is. -func Test_PostConfig_InvalidChangedConfig_V2RuleFormat(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - config := makeRulerConfig(configs.RuleFormatV2) - post(t, userID, configs.RulesConfig{}, config) - invalidConfig := configs.RulesConfig{ - Files: map[string]string{ - "some.rules": "invalid config", - }, - } - updateRequest := configUpdateRequest{ - OldConfig: configs.RulesConfig{}, - NewConfig: invalidConfig, - } - b, err := json.Marshal(updateRequest) - require.NoError(t, err) - reader := bytes.NewReader(b) - { - w := requestAsUser(t, app, userID, "POST", endpoint, reader) - require.Equal(t, http.StatusBadRequest, w.Code) - } - result := get(t, userID) - assert.Equal(t, config, result.Config) -} - -// Posting a config with an invalid rule format version returns an error and leaves the config as is. -func Test_PostConfig_InvalidChangedConfig_InvalidRuleFormat(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - config := makeRulerConfig(configs.RuleFormatV2) - post(t, userID, configs.RulesConfig{}, config) - - // We have to provide the marshaled JSON manually here because json.Marshal() would error - // on a bad rule format version. - reader := strings.NewReader(`{"old_config":{"format_version":"1","files":null},"new_config":{"format_version":"","files":{"filename.rules":"# Empty."}}}`) - { - w := requestAsUser(t, app, userID, "POST", endpoint, reader) - require.Equal(t, http.StatusBadRequest, w.Code) - } - result := get(t, userID) - assert.Equal(t, config, result.Config) -} - -// Different users can have different configurations. -func Test_PostConfig_MultipleUsers(t *testing.T) { - setup(t) - defer cleanup(t) - - userID1 := makeUserID() - userID2 := makeUserID() - config1 := post(t, userID1, configs.RulesConfig{}, makeRulerConfig(configs.RuleFormatV2)) - config2 := post(t, userID2, configs.RulesConfig{}, makeRulerConfig(configs.RuleFormatV2)) - foundConfig1 := get(t, userID1) - assert.Equal(t, config1, foundConfig1) - foundConfig2 := get(t, userID2) - assert.Equal(t, config2, foundConfig2) - assert.True(t, config2.ID > config1.ID, "%v > %v", config2.ID, config1.ID) -} - -// GetAllConfigs returns an empty list of configs if there aren't any. -func Test_GetAllConfigs_Empty(t *testing.T) { - setup(t) - defer cleanup(t) - - configs, err := poller.GetRules(context.Background()) - assert.NoError(t, err, "error getting configs") - assert.Equal(t, 0, len(configs)) -} - -// GetAllConfigs returns all created configs. -func Test_GetAllConfigs(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - config := makeRulerConfig(configs.RuleFormatV2) - view := post(t, userID, configs.RulesConfig{}, config) - - found, err := poller.GetRules(context.Background()) - assert.NoError(t, err, "error getting configs") - assert.Equal(t, map[string]configs.VersionedRulesConfig{ - userID: view, - }, found) -} - -// GetAllConfigs returns the *newest* versions of all created configs. -func Test_GetAllConfigs_Newest(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - - config1 := post(t, userID, configs.RulesConfig{}, makeRulerConfig(configs.RuleFormatV2)) - config2 := post(t, userID, config1.Config, makeRulerConfig(configs.RuleFormatV2)) - lastCreated := post(t, userID, config2.Config, makeRulerConfig(configs.RuleFormatV2)) - - found, err := poller.GetRules(context.Background()) - assert.NoError(t, err, "error getting configs") - assert.Equal(t, map[string]configs.VersionedRulesConfig{ - userID: lastCreated, - }, found) -} - -// postAlertmanagerConfig posts an alertmanager config to the alertmanager configs API. -func postAlertmanagerConfig(t *testing.T, userID, configFile string) { - config := configs.Config{ - AlertmanagerConfig: configFile, - RulesConfig: configs.RulesConfig{}, - } - b, err := json.Marshal(config) - require.NoError(t, err) - reader := bytes.NewReader(b) - configsAPI := api.New(database) - w := requestAsUser(t, configsAPI, userID, "POST", "/api/prom/configs/alertmanager", reader) - require.Equal(t, http.StatusNoContent, w.Code) -} - -// getAlertmanagerConfig posts an alertmanager config to the alertmanager configs API. -func getAlertmanagerConfig(t *testing.T, userID string) string { - w := requestAsUser(t, api.New(database), userID, "GET", "/api/prom/configs/alertmanager", nil) - var x configs.View - b := w.Body.Bytes() - err := json.Unmarshal(b, &x) - require.NoError(t, err, "Could not unmarshal JSON: %v", string(b)) - return x.Config.AlertmanagerConfig -} - -// If a user has only got alertmanager config set, then we learn nothing about them via GetConfigs. -func Test_AlertmanagerConfig_NotInAllConfigs(t *testing.T) { - setup(t) - defer cleanup(t) - - config := makeString(` - # Config no. %d. - route: - receiver: noop - - receivers: - - name: noop`) - postAlertmanagerConfig(t, makeUserID(), config) - - found, err := poller.GetRules(context.Background()) - assert.NoError(t, err, "error getting configs") - assert.Equal(t, map[string]configs.VersionedRulesConfig{}, found) -} - -// Setting a ruler config doesn't change alertmanager config. -func Test_AlertmanagerConfig_RulerConfigDoesntChangeIt(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - alertmanagerConfig := makeString(` - # Config no. %d. - route: - receiver: noop - - receivers: - - name: noop`) - postAlertmanagerConfig(t, userID, alertmanagerConfig) - - rulerConfig := makeRulerConfig(configs.RuleFormatV2) - post(t, userID, configs.RulesConfig{}, rulerConfig) - - newAlertmanagerConfig := getAlertmanagerConfig(t, userID) - assert.Equal(t, alertmanagerConfig, newAlertmanagerConfig) -} diff --git a/pkg/ruler/rulegroup/compat.go b/pkg/ruler/rulegroup/compat.go new file mode 100644 index 00000000000..6821b08ab29 --- /dev/null +++ b/pkg/ruler/rulegroup/compat.go @@ -0,0 +1,100 @@ +package rulegroup + +import ( + time "time" + + "github.com/cortexproject/cortex/pkg/configs" + "github.com/cortexproject/cortex/pkg/ingester/client" + "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/rulefmt" + "github.com/prometheus/prometheus/promql" + "github.com/prometheus/prometheus/rules" +) + +// ToProto transforms a formatted prometheus rulegroup to a rule group protobuf +func ToProto(namespace string, rl rulefmt.RuleGroup) RuleGroup { + dur := time.Duration(rl.Interval) + rg := RuleGroup{ + Name: rl.Name, + Namespace: namespace, + Interval: &dur, + } + + rules := make([]*Rule, len(rl.Rules)) + for i := range rl.Rules { + f := time.Duration(rl.Rules[i].For) + + rules[i] = &Rule{ + Expr: rl.Rules[i].Expr, + Record: rl.Rules[i].Record, + Alert: rl.Rules[i].Alert, + + For: &f, + Labels: client.FromLabelsToLabelAdapaters(labels.FromMap(rl.Rules[i].Labels)), + Annotations: client.FromLabelsToLabelAdapaters(labels.FromMap(rl.Rules[i].Labels)), + } + } + + return rg +} + +// FromProto generates a rulefmt RuleGroup +func FromProto(rg *RuleGroup) rulefmt.RuleGroup { + formattedRuleGroup := rulefmt.RuleGroup{ + Name: rg.GetName(), + Interval: model.Duration(*rg.Interval), + Rules: make([]rulefmt.Rule, len(rg.GetRules())), + } + + for i, rl := range rg.GetRules() { + formattedRuleGroup.Rules[i] = rulefmt.Rule{ + Record: rl.GetRecord(), + Alert: rl.GetAlert(), + Expr: rl.GetExpr(), + For: model.Duration(*rl.GetFor()), + Labels: client.FromLabelAdaptersToLabels(rl.Labels).Map(), + Annotations: client.FromLabelAdaptersToLabels(rl.Annotations).Map(), + } + } + + return formattedRuleGroup +} + +// GenerateRuleGroup returns a functional rulegroup from a proto +func GenerateRuleGroup(userID string, rg *RuleGroup) (configs.RuleGroup, error) { + rls := make([]rules.Rule, 0, len(rg.Rules)) + for _, rl := range rg.Rules { + expr, err := promql.ParseExpr(rl.GetExpr()) + if err != nil { + return nil, err + } + + if rl.Alert != "" { + rls = append(rls, rules.NewAlertingRule( + rl.Alert, + expr, + *rl.GetFor(), + client.FromLabelAdaptersToLabels(rl.Labels), + client.FromLabelAdaptersToLabels(rl.Annotations), + true, + log.With(util.Logger, "alert", rl.Alert), + )) + continue + } + rls = append(rls, rules.NewRecordingRule( + rl.Record, + expr, + client.FromLabelAdaptersToLabels(rl.Labels), + )) + } + + return NewRuleGroup( + rg.GetName(), + rg.GetNamespace(), + userID, + rls, + ), nil +} diff --git a/pkg/ruler/rulegroup/rulegroup.go b/pkg/ruler/rulegroup/rulegroup.go new file mode 100644 index 00000000000..dba13b8c1a5 --- /dev/null +++ b/pkg/ruler/rulegroup/rulegroup.go @@ -0,0 +1,36 @@ +package rulegroup + +import ( + "context" + + "github.com/cortexproject/cortex/pkg/configs" + "github.com/prometheus/prometheus/rules" +) + +// TODO: Add a lazy rule group that only loads rules when they are needed + +type ruleGroup struct { + name string + namespace string + user string + + rules []rules.Rule +} + +func (rg *ruleGroup) Rules(ctx context.Context) ([]rules.Rule, error) { + return rg.rules, nil +} + +func (rg *ruleGroup) Name() string { + return rg.user + "/" + rg.namespace + "/" + rg.name +} + +// NewRuleGroup returns a rulegroup +func NewRuleGroup(name, namespace, user string, rules []rules.Rule) configs.RuleGroup { + return &ruleGroup{ + name: name, + namespace: namespace, + user: user, + rules: rules, + } +} diff --git a/pkg/ruler/rulegroup/rulegroup.pb.go b/pkg/ruler/rulegroup/rulegroup.pb.go new file mode 100644 index 00000000000..3d23ac5871f --- /dev/null +++ b/pkg/ruler/rulegroup/rulegroup.pb.go @@ -0,0 +1,1184 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: rulegroup.proto + +package rulegroup + +import ( + fmt "fmt" + _ "github.com/cortexproject/cortex/pkg/ingester/client" + github_com_cortexproject_cortex_pkg_ingester_client "github.com/cortexproject/cortex/pkg/ingester/client" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + github_com_gogo_protobuf_types "github.com/gogo/protobuf/types" + _ "github.com/golang/protobuf/ptypes/duration" + io "io" + math "math" + reflect "reflect" + strings "strings" + time "time" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf +var _ = time.Kitchen + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +type RuleGroup struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Namespace string `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` + Interval *time.Duration `protobuf:"bytes,3,opt,name=interval,proto3,stdduration" json:"interval,omitempty"` + Rules []*Rule `protobuf:"bytes,4,rep,name=rules,proto3" json:"rules,omitempty"` + Deleted bool `protobuf:"varint,5,opt,name=deleted,proto3" json:"deleted,omitempty"` +} + +func (m *RuleGroup) Reset() { *m = RuleGroup{} } +func (*RuleGroup) ProtoMessage() {} +func (*RuleGroup) Descriptor() ([]byte, []int) { + return fileDescriptor_3324c775b6f77a4a, []int{0} +} +func (m *RuleGroup) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RuleGroup) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_RuleGroup.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *RuleGroup) XXX_Merge(src proto.Message) { + xxx_messageInfo_RuleGroup.Merge(m, src) +} +func (m *RuleGroup) XXX_Size() int { + return m.Size() +} +func (m *RuleGroup) XXX_DiscardUnknown() { + xxx_messageInfo_RuleGroup.DiscardUnknown(m) +} + +var xxx_messageInfo_RuleGroup proto.InternalMessageInfo + +func (m *RuleGroup) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *RuleGroup) GetNamespace() string { + if m != nil { + return m.Namespace + } + return "" +} + +func (m *RuleGroup) GetInterval() *time.Duration { + if m != nil { + return m.Interval + } + return nil +} + +func (m *RuleGroup) GetRules() []*Rule { + if m != nil { + return m.Rules + } + return nil +} + +func (m *RuleGroup) GetDeleted() bool { + if m != nil { + return m.Deleted + } + return false +} + +type Rule struct { + Expr string `protobuf:"bytes,1,opt,name=expr,proto3" json:"expr,omitempty"` + Record string `protobuf:"bytes,2,opt,name=record,proto3" json:"record,omitempty"` + Alert string `protobuf:"bytes,3,opt,name=alert,proto3" json:"alert,omitempty"` + For *time.Duration `protobuf:"bytes,4,opt,name=for,proto3,stdduration" json:"for,omitempty"` + Labels []github_com_cortexproject_cortex_pkg_ingester_client.LabelAdapter `protobuf:"bytes,5,rep,name=labels,proto3,customtype=github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter" json:"labels"` + Annotations []github_com_cortexproject_cortex_pkg_ingester_client.LabelAdapter `protobuf:"bytes,6,rep,name=annotations,proto3,customtype=github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter" json:"annotations"` +} + +func (m *Rule) Reset() { *m = Rule{} } +func (*Rule) ProtoMessage() {} +func (*Rule) Descriptor() ([]byte, []int) { + return fileDescriptor_3324c775b6f77a4a, []int{1} +} +func (m *Rule) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Rule) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Rule.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Rule) XXX_Merge(src proto.Message) { + xxx_messageInfo_Rule.Merge(m, src) +} +func (m *Rule) XXX_Size() int { + return m.Size() +} +func (m *Rule) XXX_DiscardUnknown() { + xxx_messageInfo_Rule.DiscardUnknown(m) +} + +var xxx_messageInfo_Rule proto.InternalMessageInfo + +func (m *Rule) GetExpr() string { + if m != nil { + return m.Expr + } + return "" +} + +func (m *Rule) GetRecord() string { + if m != nil { + return m.Record + } + return "" +} + +func (m *Rule) GetAlert() string { + if m != nil { + return m.Alert + } + return "" +} + +func (m *Rule) GetFor() *time.Duration { + if m != nil { + return m.For + } + return nil +} + +func init() { + proto.RegisterType((*RuleGroup)(nil), "rulegroup.RuleGroup") + proto.RegisterType((*Rule)(nil), "rulegroup.Rule") +} + +func init() { proto.RegisterFile("rulegroup.proto", fileDescriptor_3324c775b6f77a4a) } + +var fileDescriptor_3324c775b6f77a4a = []byte{ + // 439 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x93, 0xb1, 0x8e, 0xd4, 0x30, + 0x10, 0x86, 0xe3, 0xdb, 0xec, 0x72, 0xf1, 0x16, 0x27, 0x2c, 0x84, 0xcc, 0x09, 0x79, 0xa3, 0x93, + 0x90, 0xd2, 0x90, 0x88, 0xa3, 0xa4, 0x00, 0x56, 0x48, 0x50, 0x50, 0xa0, 0x94, 0x74, 0x4e, 0x32, + 0x67, 0x02, 0xbe, 0x38, 0x72, 0x1c, 0x44, 0x83, 0xc4, 0x23, 0x50, 0xf2, 0x08, 0xbc, 0x03, 0x2f, + 0x70, 0xe5, 0x96, 0x27, 0x8a, 0x83, 0xcd, 0x36, 0x74, 0xdc, 0x23, 0x20, 0x3b, 0xc9, 0xe5, 0x4a, + 0x84, 0x44, 0x95, 0xf9, 0x67, 0xc6, 0x33, 0x9f, 0xff, 0xc8, 0xf8, 0x40, 0xb7, 0x12, 0x84, 0x56, + 0x6d, 0x1d, 0xd7, 0x5a, 0x19, 0x45, 0x82, 0xab, 0xc4, 0xe1, 0x7d, 0x51, 0x9a, 0x37, 0x6d, 0x16, + 0xe7, 0xea, 0x34, 0x11, 0x4a, 0xa8, 0xc4, 0x75, 0x64, 0xed, 0x89, 0x53, 0x4e, 0xb8, 0xa8, 0x3f, + 0x79, 0xc8, 0x84, 0x52, 0x42, 0xc2, 0xd4, 0x55, 0xb4, 0x9a, 0x9b, 0x52, 0x55, 0x43, 0xfd, 0xc9, + 0xb5, 0x71, 0xb9, 0xd2, 0x06, 0x3e, 0xd4, 0x5a, 0xbd, 0x85, 0xdc, 0x0c, 0x2a, 0xa9, 0xdf, 0x89, + 0xa4, 0xac, 0x04, 0x34, 0x06, 0x74, 0x92, 0xcb, 0x12, 0xaa, 0xb1, 0xd4, 0x4f, 0x38, 0xfa, 0x86, + 0x70, 0x90, 0xb6, 0x12, 0x9e, 0x5b, 0x3c, 0x42, 0xb0, 0x5f, 0xf1, 0x53, 0xa0, 0x28, 0x44, 0x51, + 0x90, 0xba, 0x98, 0xdc, 0xc5, 0x81, 0xfd, 0x36, 0x35, 0xcf, 0x81, 0xee, 0xb9, 0xc2, 0x94, 0x20, + 0x8f, 0xf0, 0x7e, 0x59, 0x19, 0xd0, 0xef, 0xb9, 0xa4, 0xb3, 0x10, 0x45, 0xcb, 0xe3, 0x3b, 0x71, + 0x0f, 0x1d, 0x8f, 0xd0, 0xf1, 0xb3, 0x01, 0x7a, 0xed, 0x7f, 0xf9, 0xb1, 0x42, 0xe9, 0xd5, 0x01, + 0x72, 0x0f, 0xcf, 0xad, 0x35, 0x0d, 0xf5, 0xc3, 0x59, 0xb4, 0x3c, 0x3e, 0x88, 0x27, 0xe7, 0x2c, + 0x53, 0xda, 0x57, 0x09, 0xc5, 0x37, 0x0a, 0x90, 0x60, 0xa0, 0xa0, 0xf3, 0x10, 0x45, 0xfb, 0xe9, + 0x28, 0x8f, 0x7e, 0xef, 0x61, 0xdf, 0x76, 0x5a, 0x70, 0x7b, 0xf7, 0x11, 0xdc, 0xc6, 0xe4, 0x36, + 0x5e, 0x68, 0xc8, 0x95, 0x2e, 0x06, 0xea, 0x41, 0x91, 0x5b, 0x78, 0xce, 0x25, 0x68, 0xe3, 0x78, + 0x83, 0xb4, 0x17, 0xe4, 0x01, 0x9e, 0x9d, 0x28, 0x4d, 0xfd, 0xbf, 0xbb, 0x83, 0xed, 0x25, 0x0d, + 0x5e, 0x48, 0x9e, 0x81, 0x6c, 0xe8, 0xdc, 0xf1, 0xdf, 0x8c, 0x07, 0x6b, 0x5f, 0xda, 0xec, 0x2b, + 0x5e, 0xea, 0xf5, 0x8b, 0xb3, 0x8b, 0x95, 0xf7, 0xfd, 0x62, 0xf5, 0x2f, 0x3f, 0xaa, 0x1f, 0xf3, + 0xb4, 0xe0, 0xb5, 0x01, 0x9d, 0x0e, 0xab, 0xc8, 0x47, 0xbc, 0xe4, 0x55, 0xa5, 0x8c, 0xa3, 0x69, + 0xe8, 0xe2, 0xff, 0x6f, 0xbe, 0xbe, 0x6f, 0xfd, 0x78, 0xb3, 0x65, 0xde, 0xf9, 0x96, 0x79, 0x97, + 0x5b, 0x86, 0x3e, 0x75, 0x0c, 0x7d, 0xed, 0x18, 0x3a, 0xeb, 0x18, 0xda, 0x74, 0x0c, 0xfd, 0xec, + 0x18, 0xfa, 0xd5, 0x31, 0xef, 0xb2, 0x63, 0xe8, 0xf3, 0x8e, 0x79, 0x9b, 0x1d, 0xf3, 0xce, 0x77, + 0xcc, 0x7b, 0x3d, 0xbd, 0x80, 0x6c, 0xe1, 0x2c, 0x7d, 0xf8, 0x27, 0x00, 0x00, 0xff, 0xff, 0x04, + 0xbb, 0x4d, 0x8a, 0x26, 0x03, 0x00, 0x00, +} + +func (this *RuleGroup) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*RuleGroup) + if !ok { + that2, ok := that.(RuleGroup) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Name != that1.Name { + return false + } + if this.Namespace != that1.Namespace { + return false + } + if this.Interval != nil && that1.Interval != nil { + if *this.Interval != *that1.Interval { + return false + } + } else if this.Interval != nil { + return false + } else if that1.Interval != nil { + return false + } + if len(this.Rules) != len(that1.Rules) { + return false + } + for i := range this.Rules { + if !this.Rules[i].Equal(that1.Rules[i]) { + return false + } + } + if this.Deleted != that1.Deleted { + return false + } + return true +} +func (this *Rule) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*Rule) + if !ok { + that2, ok := that.(Rule) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Expr != that1.Expr { + return false + } + if this.Record != that1.Record { + return false + } + if this.Alert != that1.Alert { + return false + } + if this.For != nil && that1.For != nil { + if *this.For != *that1.For { + return false + } + } else if this.For != nil { + return false + } else if that1.For != nil { + return false + } + if len(this.Labels) != len(that1.Labels) { + return false + } + for i := range this.Labels { + if !this.Labels[i].Equal(that1.Labels[i]) { + return false + } + } + if len(this.Annotations) != len(that1.Annotations) { + return false + } + for i := range this.Annotations { + if !this.Annotations[i].Equal(that1.Annotations[i]) { + return false + } + } + return true +} +func (this *RuleGroup) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 9) + s = append(s, "&rulegroup.RuleGroup{") + s = append(s, "Name: "+fmt.Sprintf("%#v", this.Name)+",\n") + s = append(s, "Namespace: "+fmt.Sprintf("%#v", this.Namespace)+",\n") + s = append(s, "Interval: "+fmt.Sprintf("%#v", this.Interval)+",\n") + if this.Rules != nil { + s = append(s, "Rules: "+fmt.Sprintf("%#v", this.Rules)+",\n") + } + s = append(s, "Deleted: "+fmt.Sprintf("%#v", this.Deleted)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func (this *Rule) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 10) + s = append(s, "&rulegroup.Rule{") + s = append(s, "Expr: "+fmt.Sprintf("%#v", this.Expr)+",\n") + s = append(s, "Record: "+fmt.Sprintf("%#v", this.Record)+",\n") + s = append(s, "Alert: "+fmt.Sprintf("%#v", this.Alert)+",\n") + s = append(s, "For: "+fmt.Sprintf("%#v", this.For)+",\n") + s = append(s, "Labels: "+fmt.Sprintf("%#v", this.Labels)+",\n") + s = append(s, "Annotations: "+fmt.Sprintf("%#v", this.Annotations)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func valueToGoStringRulegroup(v interface{}, typ string) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) +} +func (m *RuleGroup) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RuleGroup) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintRulegroup(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if len(m.Namespace) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintRulegroup(dAtA, i, uint64(len(m.Namespace))) + i += copy(dAtA[i:], m.Namespace) + } + if m.Interval != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintRulegroup(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdDuration(*m.Interval))) + n1, err := github_com_gogo_protobuf_types.StdDurationMarshalTo(*m.Interval, dAtA[i:]) + if err != nil { + return 0, err + } + i += n1 + } + if len(m.Rules) > 0 { + for _, msg := range m.Rules { + dAtA[i] = 0x22 + i++ + i = encodeVarintRulegroup(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.Deleted { + dAtA[i] = 0x28 + i++ + if m.Deleted { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + return i, nil +} + +func (m *Rule) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Rule) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Expr) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintRulegroup(dAtA, i, uint64(len(m.Expr))) + i += copy(dAtA[i:], m.Expr) + } + if len(m.Record) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintRulegroup(dAtA, i, uint64(len(m.Record))) + i += copy(dAtA[i:], m.Record) + } + if len(m.Alert) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintRulegroup(dAtA, i, uint64(len(m.Alert))) + i += copy(dAtA[i:], m.Alert) + } + if m.For != nil { + dAtA[i] = 0x22 + i++ + i = encodeVarintRulegroup(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdDuration(*m.For))) + n2, err := github_com_gogo_protobuf_types.StdDurationMarshalTo(*m.For, dAtA[i:]) + if err != nil { + return 0, err + } + i += n2 + } + if len(m.Labels) > 0 { + for _, msg := range m.Labels { + dAtA[i] = 0x2a + i++ + i = encodeVarintRulegroup(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if len(m.Annotations) > 0 { + for _, msg := range m.Annotations { + dAtA[i] = 0x32 + i++ + i = encodeVarintRulegroup(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func encodeVarintRulegroup(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *RuleGroup) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovRulegroup(uint64(l)) + } + l = len(m.Namespace) + if l > 0 { + n += 1 + l + sovRulegroup(uint64(l)) + } + if m.Interval != nil { + l = github_com_gogo_protobuf_types.SizeOfStdDuration(*m.Interval) + n += 1 + l + sovRulegroup(uint64(l)) + } + if len(m.Rules) > 0 { + for _, e := range m.Rules { + l = e.Size() + n += 1 + l + sovRulegroup(uint64(l)) + } + } + if m.Deleted { + n += 2 + } + return n +} + +func (m *Rule) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Expr) + if l > 0 { + n += 1 + l + sovRulegroup(uint64(l)) + } + l = len(m.Record) + if l > 0 { + n += 1 + l + sovRulegroup(uint64(l)) + } + l = len(m.Alert) + if l > 0 { + n += 1 + l + sovRulegroup(uint64(l)) + } + if m.For != nil { + l = github_com_gogo_protobuf_types.SizeOfStdDuration(*m.For) + n += 1 + l + sovRulegroup(uint64(l)) + } + if len(m.Labels) > 0 { + for _, e := range m.Labels { + l = e.Size() + n += 1 + l + sovRulegroup(uint64(l)) + } + } + if len(m.Annotations) > 0 { + for _, e := range m.Annotations { + l = e.Size() + n += 1 + l + sovRulegroup(uint64(l)) + } + } + return n +} + +func sovRulegroup(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozRulegroup(x uint64) (n int) { + return sovRulegroup(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *RuleGroup) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&RuleGroup{`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `Namespace:` + fmt.Sprintf("%v", this.Namespace) + `,`, + `Interval:` + strings.Replace(fmt.Sprintf("%v", this.Interval), "Duration", "duration.Duration", 1) + `,`, + `Rules:` + strings.Replace(fmt.Sprintf("%v", this.Rules), "Rule", "Rule", 1) + `,`, + `Deleted:` + fmt.Sprintf("%v", this.Deleted) + `,`, + `}`, + }, "") + return s +} +func (this *Rule) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&Rule{`, + `Expr:` + fmt.Sprintf("%v", this.Expr) + `,`, + `Record:` + fmt.Sprintf("%v", this.Record) + `,`, + `Alert:` + fmt.Sprintf("%v", this.Alert) + `,`, + `For:` + strings.Replace(fmt.Sprintf("%v", this.For), "Duration", "duration.Duration", 1) + `,`, + `Labels:` + fmt.Sprintf("%v", this.Labels) + `,`, + `Annotations:` + fmt.Sprintf("%v", this.Annotations) + `,`, + `}`, + }, "") + return s +} +func valueToStringRulegroup(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *RuleGroup) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRulegroup + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RuleGroup: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RuleGroup: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRulegroup + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRulegroup + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRulegroup + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Namespace", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRulegroup + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRulegroup + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRulegroup + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Namespace = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Interval", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRulegroup + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRulegroup + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRulegroup + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Interval == nil { + m.Interval = new(time.Duration) + } + if err := github_com_gogo_protobuf_types.StdDurationUnmarshal(m.Interval, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Rules", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRulegroup + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRulegroup + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRulegroup + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Rules = append(m.Rules, &Rule{}) + if err := m.Rules[len(m.Rules)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRulegroup + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Deleted = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipRulegroup(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRulegroup + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthRulegroup + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Rule) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRulegroup + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Rule: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Rule: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Expr", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRulegroup + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRulegroup + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRulegroup + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Expr = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Record", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRulegroup + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRulegroup + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRulegroup + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Record = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Alert", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRulegroup + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRulegroup + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRulegroup + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Alert = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field For", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRulegroup + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRulegroup + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRulegroup + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.For == nil { + m.For = new(time.Duration) + } + if err := github_com_gogo_protobuf_types.StdDurationUnmarshal(m.For, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRulegroup + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRulegroup + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRulegroup + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Labels = append(m.Labels, github_com_cortexproject_cortex_pkg_ingester_client.LabelAdapter{}) + if err := m.Labels[len(m.Labels)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Annotations", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRulegroup + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRulegroup + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRulegroup + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Annotations = append(m.Annotations, github_com_cortexproject_cortex_pkg_ingester_client.LabelAdapter{}) + if err := m.Annotations[len(m.Annotations)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRulegroup(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRulegroup + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthRulegroup + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipRulegroup(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRulegroup + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRulegroup + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRulegroup + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthRulegroup + } + iNdEx += length + if iNdEx < 0 { + return 0, ErrInvalidLengthRulegroup + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRulegroup + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipRulegroup(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + if iNdEx < 0 { + return 0, ErrInvalidLengthRulegroup + } + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthRulegroup = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowRulegroup = fmt.Errorf("proto: integer overflow") +) diff --git a/pkg/ruler/rulegroup/rulegroup.proto b/pkg/ruler/rulegroup/rulegroup.proto new file mode 100644 index 00000000000..9d7973b100b --- /dev/null +++ b/pkg/ruler/rulegroup/rulegroup.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package rulegroup; + +option go_package = "rulegroup"; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; +import "google/protobuf/duration.proto"; +import "github.com/cortexproject/cortex/pkg/ingester/client/cortex.proto"; + +option (gogoproto.marshaler_all) = true; +option (gogoproto.unmarshaler_all) = true; + + +message RuleGroup { + string name = 1; + string namespace = 2; + google.protobuf.Duration interval = 3 [(gogoproto.stdduration) = true]; + + repeated Rule rules = 4; + + bool deleted = 5; +} + +message Rule { + string expr = 1; + string record = 2; + string alert = 3; + google.protobuf.Duration for = 4 [(gogoproto.stdduration) = true]; + repeated cortex.LabelPair labels = 5 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter"];; + repeated cortex.LabelPair annotations = 6 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter"];; +} \ No newline at end of file diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 6cc1bdc6950..123903c74f7 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -22,7 +22,7 @@ import ( "golang.org/x/net/context" "golang.org/x/net/context/ctxhttp" - "github.com/cortexproject/cortex/pkg/configs/db" + "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/util" @@ -130,7 +130,7 @@ type Ruler struct { } // NewRuler creates a new ruler from a distributor and chunk store. -func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d *distributor.Distributor, poller db.Poller) (*Ruler, error) { +func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d *distributor.Distributor, store configs.RuleStore) (*Ruler, error) { if cfg.NumWorkers <= 0 { return nil, fmt.Errorf("must have at least 1 worker, got %d", cfg.NumWorkers) } @@ -151,7 +151,7 @@ func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d workerWG: &sync.WaitGroup{}, } - ruler.scheduler = newScheduler(poller, cfg.EvaluationInterval, cfg.EvaluationInterval, ruler.newGroup) + ruler.scheduler = newScheduler(store, cfg.EvaluationInterval, cfg.EvaluationInterval, ruler.newGroup) // If sharding is enabled, create/join a ring to distribute tokens to // the ruler diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index ee039439dd5..95518a60fa4 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -4,11 +4,13 @@ import ( "context" "net/http" "net/http/httptest" + "strings" "sync" "testing" "time" "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/rulefmt" "github.com/prometheus/prometheus/promql" "github.com/cortexproject/cortex/pkg/configs" @@ -22,14 +24,91 @@ import ( "github.com/weaveworks/common/user" ) -type mockRuleStore struct{} +type mockRuleStore struct { + rules map[string]rulefmt.RuleGroup -func (m *mockRuleStore) GetRules(ctx context.Context) (map[string]configs.VersionedRulesConfig, error) { - return map[string]configs.VersionedRulesConfig{}, nil + pollPayload map[string][]configs.RuleGroup } -func (m *mockRuleStore) GetAlerts(ctx context.Context) (map[string]configs.View, error) { - return map[string]configs.View{}, nil +func (m *mockRuleStore) PollRules(ctx context.Context) (map[string][]configs.RuleGroup, error) { + pollPayload := m.pollPayload + m.pollPayload = map[string][]configs.RuleGroup{} + return pollPayload, nil +} + +func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) ([]configs.RuleNamespace, error) { + groupPrefix := userID + ":" + + namespaces := []string{} + nss := []configs.RuleNamespace{} + for n := range m.rules { + if strings.HasPrefix(n, groupPrefix) { + components := strings.Split(n, ":") + if len(components) != 3 { + continue + } + namespaces = append(namespaces, components[1]) + } + } + + if len(namespaces) == 0 { + return nss, configs.ErrUserNotFound + } + + for _, n := range namespaces { + ns, err := m.getRuleNamespace(ctx, userID, n) + if err != nil { + continue + } + + nss = append(nss, ns) + } + + return nss, nil +} + +func (m *mockRuleStore) getRuleNamespace(ctx context.Context, userID string, namespace string) (configs.RuleNamespace, error) { + groupPrefix := userID + ":" + namespace + ":" + + ns := configs.RuleNamespace{ + Namespace: namespace, + Groups: []rulefmt.RuleGroup{}, + } + for n, g := range m.rules { + if strings.HasPrefix(n, groupPrefix) { + ns.Groups = append(ns.Groups, g) + } + } + + if len(ns.Groups) == 0 { + return ns, configs.ErrGroupNamespaceNotFound + } + + return ns, nil +} + +func (m *mockRuleStore) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (rulefmt.RuleGroup, error) { + groupID := userID + ":" + namespace + ":" + group + g, ok := m.rules[groupID] + + if !ok { + return rulefmt.RuleGroup{}, configs.ErrGroupNotFound + } + + return g, nil + +} + +func (m *mockRuleStore) SetRuleGroup(ctx context.Context, userID string, namespace string, group rulefmt.RuleGroup) error { + groupID := userID + ":" + namespace + ":" + group.Name + m.rules[groupID] = group + return nil +} + +func (m *mockRuleStore) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { + groupID := userID + ":" + namespace + ":" + group + delete(m.rules, groupID) + return nil } func defaultRulerConfig() Config { diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index dba13e6d3c1..8c0dfa2544b 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -9,16 +9,13 @@ import ( "sync" "time" + "github.com/cortexproject/cortex/pkg/configs" + "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "github.com/jonboulle/clockwork" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/prometheus/rules" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/configs/db" - "github.com/cortexproject/cortex/pkg/util" ) var backoffConfig = util.BackoffConfig{ @@ -50,12 +47,13 @@ var ( ) type workItem struct { - userID string - groupName string - hash uint32 - group *group - scheduled time.Time - generation configs.ID // a monotonically increasing number used to spot out of date work items + userID string + groupName string + hash uint32 + group *group + scheduled time.Time + + done chan struct{} } // Key implements ScheduledItem @@ -70,7 +68,7 @@ func (w workItem) Scheduled() time.Time { // Defer returns a work item with updated rules, rescheduled to a later time. func (w workItem) Defer(interval time.Duration) workItem { - return workItem{w.userID, w.groupName, w.hash, w.group, w.scheduled.Add(interval), w.generation} + return workItem{w.userID, w.groupName, w.hash, w.group, w.scheduled.Add(interval), w.done} } func (w workItem) String() string { @@ -78,30 +76,30 @@ func (w workItem) String() string { } type userConfig struct { - rules map[string][]rules.Rule - generation configs.ID // a monotonically increasing number used to spot out of date work items + done chan struct{} + id string + rules []configs.RuleGroup } type groupFactory func(userID string, groupName string, rls []rules.Rule) (*group, error) type scheduler struct { - poller db.Poller + store configs.RuleStore evaluationInterval time.Duration // how often we re-evaluate each rule set q *SchedulingQueue pollInterval time.Duration // how often we check for new config - cfgs map[string]userConfig // all rules for all users - latestConfig configs.ID // # of last update received from config - groupFn groupFactory // function to create a new group + cfgs map[string]userConfig // all rules for all users + groupFn groupFactory // function to create a new group sync.RWMutex done chan struct{} } // newScheduler makes a new scheduler. -func newScheduler(poller db.Poller, evaluationInterval, pollInterval time.Duration, groupFn groupFactory) *scheduler { +func newScheduler(store configs.RuleStore, evaluationInterval, pollInterval time.Duration, groupFn groupFactory) *scheduler { return &scheduler{ - poller: poller, + store: store, evaluationInterval: evaluationInterval, pollInterval: pollInterval, q: NewSchedulingQueue(clockwork.NewRealClock()), @@ -116,13 +114,16 @@ func newScheduler(poller db.Poller, evaluationInterval, pollInterval time.Durati func (s *scheduler) Run() { level.Debug(util.Logger).Log("msg", "scheduler started") - // Load initial set of all configurations before polling for new ones. - s.addNewConfigs(time.Now(), s.loadAllConfigs()) + err := s.updateConfigs(context.TODO()) + if err != nil { + level.Error(util.Logger).Log("msg", "scheduler: error updating rule groups", "err", err) + } + ticker := time.NewTicker(s.pollInterval) for { select { - case now := <-ticker.C: - err := s.updateConfigs(now) + case <-ticker.C: + err := s.updateConfigs(context.TODO()) if err != nil { level.Warn(util.Logger).Log("msg", "scheduler: error updating configs", "err", err) } @@ -140,117 +141,92 @@ func (s *scheduler) Stop() { level.Debug(util.Logger).Log("msg", "scheduler stopped") } -// Load the full set of configurations from the server, retrying with backoff -// until we can get them. -func (s *scheduler) loadAllConfigs() map[string]configs.VersionedRulesConfig { - backoff := util.NewBackoff(context.Background(), backoffConfig) - for { - cfgs, err := s.poll() - if err == nil { - level.Debug(util.Logger).Log("msg", "scheduler: initial configuration load", "num_configs", len(cfgs)) - return cfgs - } - level.Warn(util.Logger).Log("msg", "scheduler: error fetching all configurations, backing off", "err", err) - backoff.Wait() - } -} - -func (s *scheduler) updateConfigs(now time.Time) error { - cfgs, err := s.poll() +func (s *scheduler) updateConfigs(ctx context.Context) error { + cfgs, err := s.store.PollRules(ctx) if err != nil { return err } - s.addNewConfigs(now, cfgs) - return nil -} -// poll the configuration server. Not re-entrant. -func (s *scheduler) poll() (map[string]configs.VersionedRulesConfig, error) { - cfgs, err := s.poller.GetRules(context.Background()) - if err != nil { - level.Warn(util.Logger).Log("msg", "scheduler: configs server poll failed", "err", err) - return nil, err + for user, cfg := range cfgs { + s.addUserConfig(ctx, user, cfg) } - return cfgs, nil -} - -// computeNextEvalTime Computes when a user's rules should be next evaluated, based on how far we are through an evaluation cycle -func (s *scheduler) computeNextEvalTime(hasher hash.Hash64, now time.Time, userID string) time.Time { - intervalNanos := float64(s.evaluationInterval.Nanoseconds()) - // Compute how far we are into the current evaluation cycle - currentEvalCyclePoint := math.Mod(float64(now.UnixNano()), intervalNanos) - hasher.Reset() - hasher.Write([]byte(userID)) - offset := math.Mod( - // We subtract our current point in the cycle to cause the entries - // before 'now' to wrap around to the end. - // We don't want this to come out negative, so we add the interval to it - float64(hasher.Sum64())+intervalNanos-currentEvalCyclePoint, - intervalNanos) - return now.Add(time.Duration(int64(offset))) + return nil } -func (s *scheduler) addNewConfigs(now time.Time, cfgs map[string]configs.VersionedRulesConfig) { - // TODO: instrument how many configs we have, both valid & invalid. - level.Debug(util.Logger).Log("msg", "adding configurations", "num_configs", len(cfgs)) +func (s *scheduler) addUserConfig(ctx context.Context, userID string, rgs []configs.RuleGroup) { + now := time.Now() hasher := fnv.New64a() - for userID, config := range cfgs { - s.addUserConfig(now, hasher, userID, config) - } - - configUpdates.Add(float64(len(cfgs))) - s.Lock() - lenCfgs := len(s.cfgs) - s.Unlock() - totalConfigs.Set(float64(lenCfgs)) -} - -func (s *scheduler) addUserConfig(now time.Time, hasher hash.Hash64, userID string, config configs.VersionedRulesConfig) { - rulesByGroup, err := config.Config.Parse() - if err != nil { - // XXX: This means that if a user has a working configuration and - // they submit a broken one, we'll keep processing the last known - // working configuration, and they'll never know. - // TODO: Provide a way of deleting / cancelling recording rules. - level.Warn(util.Logger).Log("msg", "scheduler: invalid Cortex configuration", "user_id", userID, "err", err) - return - } + level.Info(util.Logger).Log("msg", "scheduler: updating rules for user", "user_id", userID, "num_groups", len(rgs)) - level.Info(util.Logger).Log("msg", "scheduler: updating rules for user", "user_id", userID, "num_groups", len(rulesByGroup), "is_deleted", config.IsDeleted()) - s.Lock() - // if deleted remove from map, otherwise - update map - if config.IsDeleted() { - delete(s.cfgs, userID) - s.Unlock() - return - } - s.cfgs[userID] = userConfig{rules: rulesByGroup, generation: config.ID} - s.Unlock() + // create a new userchan for rulegroups of this user + userChan := make(chan struct{}) ringHasher := fnv.New32a() - evalTime := s.computeNextEvalTime(hasher, now, userID) + evalTime := computeNextEvalTime(hasher, now, float64(s.evaluationInterval.Nanoseconds()), userID) workItems := []workItem{} - for group, rules := range rulesByGroup { - level.Debug(util.Logger).Log("msg", "scheduler: updating rules for user and group", "user_id", userID, "group", group, "num_rules", len(rules)) - g, err := s.groupFn(userID, group, rules) + for _, rg := range rgs { + rules, err := rg.Rules(ctx) if err != nil { - // XXX: similarly to above if a user has a working configuration and - // for some reason we cannot create a group for the new one we'll use - // the last known working configuration - level.Warn(util.Logger).Log("msg", "scheduler: failed to create group for user", "user_id", userID, "group", group, "err", err) + level.Warn(util.Logger).Log("msg", "scheduler: failed to create group for user", "user_id", userID, "group", rg.Name(), "err", err) return } + + level.Debug(util.Logger).Log("msg", "scheduler: updating rules for user and group", "user_id", userID, "group", rg.Name(), "num_rules", len(rules)) + g, err := s.groupFn(userID, rg.Name(), rules) + if err != nil { + level.Warn(util.Logger).Log("msg", "scheduler: failed to create group for user", "user_id", userID, "group", rg.Name(), "err", err) + return + } + ringHasher.Reset() - ringHasher.Write([]byte(userID + ":" + group)) + ringHasher.Write([]byte(rg.Name())) hash := ringHasher.Sum32() - workItems = append(workItems, workItem{userID, group, hash, g, evalTime, config.ID}) + workItems = append(workItems, workItem{userID, rg.Name(), hash, g, evalTime, userChan}) } + + s.updateUserConfig(ctx, userConfig{ + id: userID, + rules: rgs, + done: userChan, + }) + for _, i := range workItems { - totalRuleGroups.Inc() s.addWorkItem(i) } + + totalRuleGroups.Add(float64(len(workItems))) +} + +func (s *scheduler) updateUserConfig(ctx context.Context, cfg userConfig) { + // Retreive any previous configuration and update to the new configuration + s.Lock() + curr, exists := s.cfgs[cfg.id] + s.cfgs[cfg.id] = cfg + s.Unlock() + + if exists { + close(curr.done) // If a previous configuration exists, ensure it is closed + } + + return +} + +// computeNextEvalTime Computes when a user's rules should be next evaluated, based on how far we are through an evaluation cycle +func computeNextEvalTime(hasher hash.Hash64, now time.Time, intervalNanos float64, userID string) time.Time { + // Compute how far we are into the current evaluation cycle + currentEvalCyclePoint := math.Mod(float64(now.UnixNano()), intervalNanos) + + hasher.Reset() + hasher.Write([]byte(userID)) + offset := math.Mod( + // We subtract our current point in the cycle to cause the entries + // before 'now' to wrap around to the end. + // We don't want this to come out negative, so we add the interval to it + float64(hasher.Sum64())+intervalNanos-currentEvalCyclePoint, + intervalNanos) + return now.Add(time.Duration(int64(offset))) } func (s *scheduler) addWorkItem(i workItem) { @@ -285,21 +261,14 @@ func (s *scheduler) nextWorkItem() *workItem { // workItemDone marks the given item as being ready to be rescheduled. func (s *scheduler) workItemDone(i workItem) { - s.Lock() - config, found := s.cfgs[i.userID] - var currentRules []rules.Rule - if found { - currentRules = config.rules[i.groupName] - } - s.Unlock() - if !found || len(currentRules) == 0 || i.generation < config.generation { - // Warning: this test will produce an incorrect result if the generation ever overflows - level.Debug(util.Logger).Log("msg", "scheduler: stopping item", "user_id", i.userID, "group", i.groupName, "found", found, "len", len(currentRules)) - totalRuleGroups.Dec() + select { + case <-i.done: + // Unschedule the work item + level.Debug(util.Logger).Log("msg", "scheduler: work item dropped", "item", i) return + default: + next := i.Defer(s.evaluationInterval) + level.Debug(util.Logger).Log("msg", "scheduler: work item rescheduled", "item", i, "time", next.scheduled.Format(timeLogFormat)) + s.addWorkItem(next) } - - next := i.Defer(s.evaluationInterval) - level.Debug(util.Logger).Log("msg", "scheduler: work item rescheduled", "item", i, "time", next.scheduled.Format(timeLogFormat)) - s.addWorkItem(next) } diff --git a/pkg/ruler/scheduler_test.go b/pkg/ruler/scheduler_test.go index 1a2a48fa0d4..82c0cec8755 100644 --- a/pkg/ruler/scheduler_test.go +++ b/pkg/ruler/scheduler_test.go @@ -1,13 +1,15 @@ package ruler import ( + "context" "strconv" "testing" "time" - "github.com/stretchr/testify/assert" - + "github.com/cortexproject/cortex/pkg/configs" + "github.com/cortexproject/cortex/pkg/ruler/rulegroup" "github.com/prometheus/prometheus/rules" + "github.com/stretchr/testify/assert" ) type fakeHasher struct { @@ -44,7 +46,7 @@ func TestSchedulerComputeNextEvalTime(t *testing.T) { // We use the fake hasher to give us control over the hash output // so that we can test the wrap-around behaviour of the modulo fakeUserID := strconv.FormatInt(hashResult, 10) - return s.computeNextEvalTime(&h, time.Unix(0, now), fakeUserID).UnixNano() + return computeNextEvalTime(&h, time.Unix(0, now), 15, fakeUserID).UnixNano() } { cycleStartTime := int64(30) @@ -71,27 +73,43 @@ func TestSchedulerComputeNextEvalTime(t *testing.T) { func TestSchedulerRulesOverlap(t *testing.T) { s := newScheduler(nil, 15, 15, nil) userID := "bob" - groupName := "test" + groupOne := "test1" + groupTwo := "test2" next := time.Now() - ruleSet := []rules.Rule{ - nil, + ruleSetsOne := []configs.RuleGroup{ + rulegroup.NewRuleGroup(groupOne, "default", userID, []rules.Rule{nil}), } - ruleSets := map[string][]rules.Rule{} - ruleSets[groupName] = ruleSet - cfg := userConfig{generation: 1, rules: ruleSets} - s.cfgs[userID] = cfg - w1 := workItem{userID: userID, groupName: groupName, scheduled: next, generation: cfg.generation} - s.workItemDone(w1) + ruleSetsTwo := []configs.RuleGroup{ + rulegroup.NewRuleGroup(groupTwo, "default", userID, []rules.Rule{nil}), + } + userChanOne := make(chan struct{}) + userChanTwo := make(chan struct{}) + + cfgOne := userConfig{rules: ruleSetsOne, done: userChanOne} + cfgTwo := userConfig{rules: ruleSetsTwo, done: userChanTwo} + + s.updateUserConfig(context.Background(), cfgOne) + w0 := workItem{userID: userID, groupName: groupOne, scheduled: next, done: userChanOne} + s.workItemDone(w0) item := s.q.Dequeue().(workItem) - assert.Equal(t, w1.generation, item.generation) + assert.Equal(t, item.groupName, groupOne) - w0 := workItem{userID: userID, groupName: groupName, scheduled: next, generation: cfg.generation - 1} - s.workItemDone(w1) + // create a new workitem for the updated ruleset + w1 := workItem{userID: userID, groupName: groupTwo, scheduled: next, done: userChanTwo} + + // Apply the new config, scheduling the previous config to be dropped + s.updateUserConfig(context.Background(), cfgTwo) + + // Reschedule the old config first, then the new config s.workItemDone(w0) + s.workItemDone(w1) + + // Ensure the old config was dropped due to the done channel being closed + // when the new user config was updated item = s.q.Dequeue().(workItem) - assert.Equal(t, w1.generation, item.generation) + assert.Equal(t, item.groupName, groupTwo) s.q.Close() assert.Equal(t, nil, s.q.Dequeue()) From 2bd16dcc6d28208bbca750e2a76c5a67cf504827 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Tue, 25 Jun 2019 18:52:18 -0400 Subject: [PATCH 09/60] add debug logging and fix yaml response for rules Signed-off-by: Jacob Lisi --- pkg/alertmanager/api.go | 4 +--- pkg/configs/storage/clients/gcp/gcs.go | 3 +-- pkg/ruler/api.go | 13 ++++++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/alertmanager/api.go b/pkg/alertmanager/api.go index bc0fabd6b2a..02153dffc6d 100644 --- a/pkg/alertmanager/api.go +++ b/pkg/alertmanager/api.go @@ -70,13 +70,11 @@ func (a *API) getConfig(w http.ResponseWriter, r *http.Request) { } w.Header().Set("Content-Type", "application/yaml") - if err := yaml.NewEncoder(w).Encode(d); err != nil { + if _, err := w.Write(d); err != nil { level.Error(logger).Log("msg", "error marshalling yaml alertmanager config", "err", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } - - w.WriteHeader(http.StatusOK) } func (a *API) setConfig(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/configs/storage/clients/gcp/gcs.go b/pkg/configs/storage/clients/gcp/gcs.go index 7917c4668b9..3c1c5b97737 100644 --- a/pkg/configs/storage/clients/gcp/gcs.go +++ b/pkg/configs/storage/clients/gcp/gcs.go @@ -260,12 +260,11 @@ func (g *gcsConfigClient) ListRuleGroups(ctx context.Context, options configs.Ru } if obj.Name == options.Namespace && options.Namespace != "" { - + level.Debug(util.Logger).Log("msg", "listing rule groups", "namespace", obj.Name) ns, err := g.getRuleNamespace(ctx, options.UserID, obj.Name) if err != nil { return []configs.RuleNamespace{}, err } - nss = append(nss, ns) } } diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index ea358d3e0d2..8ee7dfd08a6 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -36,7 +36,7 @@ func (a *API) RegisterRoutes(r *mux.Router) { name, method, path string handler http.HandlerFunc }{ - {"list_rules", "GET", "/api/prom/rules", a.listRules}, + {"list_rules", "GET", "/api/prom/rules/", a.listRules}, {"list_rules_namespace", "GET", "/api/prom/rules/{namespace}/", a.listRules}, {"get_rulegroup", "GET", "/api/prom/rules/{namespace}/{groupName}", a.getRuleGroup}, {"set_namespace", "POST", "/api/prom/rules/{namespace}/", a.setRuleNamespace}, @@ -69,6 +69,7 @@ func (a *API) listRules(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) + level.Debug(logger).Log("msg", "retrieving rule groups from rule store", "userID", userID) rgs, err := a.store.ListRuleGroups(r.Context(), configs.RuleStoreConditions{ UserID: userID, Namespace: vars["namespace"], @@ -79,20 +80,22 @@ func (a *API) listRules(w http.ResponseWriter, r *http.Request) { return } + level.Debug(logger).Log("msg", "retrieved rule groups from rule store", "userID", userID, "num_namespaces", len(rgs)) + d, err := yaml.Marshal(&rgs) if err != nil { + level.Error(logger).Log("msg", "error marshalling yaml rule groups", "err", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/yaml") - if err := yaml.NewEncoder(w).Encode(d); err != nil { - level.Error(logger).Log("msg", "error marshalling yaml rule groups", "err", err) + w.Write(d) + if _, err := w.Write(d); err != nil { + level.Error(logger).Log("msg", "error writing yaml response", "err", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } - - w.WriteHeader(http.StatusOK) } func (a *API) getRuleGroup(w http.ResponseWriter, r *http.Request) { From fcfcc9b42e1b54e3cee97825241321bd35e279fb Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Wed, 26 Jun 2019 00:08:53 -0400 Subject: [PATCH 10/60] return 404 when no rule groups are found Signed-off-by: Jacob Lisi --- pkg/ruler/api.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 8ee7dfd08a6..b1f3504a14f 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -16,8 +16,9 @@ import ( ) var ( - ErrNoNamespace = errors.New("a namespace must be provided in the url") - ErrNoGroupName = errors.New("a matching group name must be provided in the url") + ErrNoNamespace = errors.New("a namespace must be provided in the url") + ErrNoGroupName = errors.New("a matching group name must be provided in the url") + ErrNoRuleGroups = errors.New("no rule groups found") ) // API is used to provided endpoints to directly interact with the ruler @@ -82,6 +83,12 @@ func (a *API) listRules(w http.ResponseWriter, r *http.Request) { level.Debug(logger).Log("msg", "retrieved rule groups from rule store", "userID", userID, "num_namespaces", len(rgs)) + if len(rgs) == 0 { + level.Info(logger).Log("msg", "no rule groups found", "userID", userID) + http.Error(w, ErrNoRuleGroups.Error(), http.StatusNotFound) + return + } + d, err := yaml.Marshal(&rgs) if err != nil { level.Error(logger).Log("msg", "error marshalling yaml rule groups", "err", err) From eebd4003969e0e84335a58af9012f941fc8e6c63 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Wed, 26 Jun 2019 23:03:16 -0400 Subject: [PATCH 11/60] add set rule group api and fix rule gcs backend Signed-off-by: Jacob Lisi --- pkg/configs/storage/clients/gcp/gcs.go | 2 +- pkg/ruler/api.go | 49 +++++++++++--------------- 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/pkg/configs/storage/clients/gcp/gcs.go b/pkg/configs/storage/clients/gcp/gcs.go index 3c1c5b97737..7759711cd82 100644 --- a/pkg/configs/storage/clients/gcp/gcs.go +++ b/pkg/configs/storage/clients/gcp/gcs.go @@ -397,7 +397,7 @@ func (g *gcsConfigClient) DeleteRuleGroup(ctx context.Context, userID string, na } func generateRuleHandle(id, namespace, name string) string { - prefix := rulePrefix + "/" + id + "/" + prefix := rulePrefix + id + "/" if namespace == "" { return prefix } diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index b1f3504a14f..6e9a2dca567 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -40,8 +40,7 @@ func (a *API) RegisterRoutes(r *mux.Router) { {"list_rules", "GET", "/api/prom/rules/", a.listRules}, {"list_rules_namespace", "GET", "/api/prom/rules/{namespace}/", a.listRules}, {"get_rulegroup", "GET", "/api/prom/rules/{namespace}/{groupName}", a.getRuleGroup}, - {"set_namespace", "POST", "/api/prom/rules/{namespace}/", a.setRuleNamespace}, - {"set_rulegroup", "POST", "/api/prom/rules/{namespace}/", a.setRuleNamespace}, + {"set_rulegroup", "POST", "/api/prom/rules/{namespace}/", a.setRuleGroup}, {"delete_rulegroup", "DELETE", "/api/prom/rules/{namespace}/{groupName}", a.deleteRuleGroup}, } { r.Handle(route.path, route.handler).Methods(route.method).Name(route.name) @@ -106,10 +105,6 @@ func (a *API) listRules(w http.ResponseWriter, r *http.Request) { } func (a *API) getRuleGroup(w http.ResponseWriter, r *http.Request) { - -} - -func (a *API) setRuleNamespace(w http.ResponseWriter, r *http.Request) { userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) if err != nil { http.Error(w, err.Error(), http.StatusUnauthorized) @@ -130,33 +125,39 @@ func (a *API) setRuleNamespace(w http.ResponseWriter, r *http.Request) { } vars := mux.Vars(r) + ns, exists := vars["namespace"] + if !exists { + http.Error(w, ErrNoNamespace.Error(), http.StatusUnauthorized) + return + } - namespace, set := vars["namespace"] - if !set { - level.Error(logger).Log("err", err.Error()) - http.Error(w, ErrNoNamespace.Error(), http.StatusBadRequest) + gn, exists := vars["groupName"] + if !exists { + http.Error(w, ErrNoGroupName.Error(), http.StatusUnauthorized) return } - payload, err := ioutil.ReadAll(r.Body) + rg, err := a.store.GetRuleGroup(r.Context(), userID, ns, gn) + if err != nil { - level.Error(logger).Log("err", err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) return } - rgs := []rulefmt.RuleGroup{} - err = yaml.Unmarshal(payload, &rgs) + d, err := yaml.Marshal(&rg) if err != nil { - level.Error(logger).Log("err", err.Error()) - http.Error(w, err.Error(), http.StatusBadRequest) + level.Error(logger).Log("msg", "error marshalling yaml rule groups", "err", err) + http.Error(w, err.Error(), http.StatusInternalServerError) return } - for _, rg := range rgs { - a.store.SetRuleGroup(r.Context(), userID, namespace, rg) + w.Header().Set("Content-Type", "application/yaml") + w.Write(d) + if _, err := w.Write(d); err != nil { + level.Error(logger).Log("msg", "error writing yaml response", "err", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return } - w.WriteHeader(http.StatusOK) } func (a *API) setRuleGroup(w http.ResponseWriter, r *http.Request) { @@ -188,13 +189,6 @@ func (a *API) setRuleGroup(w http.ResponseWriter, r *http.Request) { return } - groupName, set := vars["groupName"] - if !set { - level.Error(logger).Log("err", err.Error()) - http.Error(w, ErrNoGroupName.Error(), http.StatusBadRequest) - return - } - payload, err := ioutil.ReadAll(r.Body) if err != nil { level.Error(logger).Log("err", err.Error()) @@ -210,9 +204,6 @@ func (a *API) setRuleGroup(w http.ResponseWriter, r *http.Request) { return } - // Set the group name to match the name provided in - // the url - rg.Name = groupName errs := configs.ValidateRuleGroup(rg) if len(errs) > 0 { level.Error(logger).Log("err", err.Error()) From f4a78aa5645b9a6ab01b7d1df56e733e0ec3aa34 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Wed, 26 Jun 2019 23:25:49 -0400 Subject: [PATCH 12/60] add logging statements to read logging api Signed-off-by: Jacob Lisi --- pkg/ruler/api.go | 17 +++++++++++++---- pkg/ruler/rulegroup/compat.go | 2 ++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 6e9a2dca567..804b0c27a65 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -67,13 +67,20 @@ func (a *API) listRules(w http.ResponseWriter, r *http.Request) { return } + options := configs.RuleStoreConditions{ + UserID: userID, + } + vars := mux.Vars(r) + namespace, set := vars["namespace"] + if set { + level.Debug(logger).Log("msg", "retrieving rule groups with namespace", "userID", userID, "namespace", namespace) + options.Namespace = namespace + } + level.Debug(logger).Log("msg", "retrieving rule groups from rule store", "userID", userID) - rgs, err := a.store.ListRuleGroups(r.Context(), configs.RuleStoreConditions{ - UserID: userID, - Namespace: vars["namespace"], - }) + rgs, err := a.store.ListRuleGroups(r.Context(), options) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -196,6 +203,8 @@ func (a *API) setRuleGroup(w http.ResponseWriter, r *http.Request) { return } + level.Debug(logger).Log("msg", "attempting to unmarshal rulegroup", "userID", userID, "group", string(payload)) + rg := rulefmt.RuleGroup{} err = yaml.Unmarshal(payload, &rg) if err != nil { diff --git a/pkg/ruler/rulegroup/compat.go b/pkg/ruler/rulegroup/compat.go index 6821b08ab29..3298181a7d0 100644 --- a/pkg/ruler/rulegroup/compat.go +++ b/pkg/ruler/rulegroup/compat.go @@ -38,6 +38,8 @@ func ToProto(namespace string, rl rulefmt.RuleGroup) RuleGroup { } } + rg.Rules = rules + return rg } From c2a70f37801fe3a8dbcf97eddb5561df72c42f0e Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Thu, 27 Jun 2019 00:25:34 -0400 Subject: [PATCH 13/60] fix list rules for GCS rule store Signed-off-by: Jacob Lisi --- pkg/configs/storage/clients/gcp/gcs.go | 45 ++++++++++++++++++++------ pkg/ruler/api.go | 6 ++-- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/pkg/configs/storage/clients/gcp/gcs.go b/pkg/configs/storage/clients/gcp/gcs.go index 7759711cd82..022858b2258 100644 --- a/pkg/configs/storage/clients/gcp/gcs.go +++ b/pkg/configs/storage/clients/gcp/gcs.go @@ -3,6 +3,7 @@ package gcp import ( "context" "encoding/json" + "errors" "flag" "io/ioutil" "strings" @@ -24,6 +25,10 @@ const ( rulePrefix = "rules/" ) +var ( + ErrBadRuleGroup = errors.New("unable to decompose handle for rule object") +) + // GCSConfig is config for the GCS Chunk Client. type GCSConfig struct { BucketName string `yaml:"bucket_name"` @@ -243,12 +248,10 @@ func (g *gcsConfigClient) getAllRuleGroups(ctx context.Context, userID string) ( func (g *gcsConfigClient) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) ([]configs.RuleNamespace, error) { it := g.bucket.Objects(ctx, &storage.Query{ - Delimiter: "/", - Prefix: generateRuleHandle(options.UserID, "", ""), + Prefix: generateRuleHandle(options.UserID, options.Namespace, ""), }) - nss := []configs.RuleNamespace{} - + namespaces := map[string]bool{} for { obj, err := it.Next() if err == iterator.Done { @@ -259,16 +262,27 @@ func (g *gcsConfigClient) ListRuleGroups(ctx context.Context, options configs.Ru return []configs.RuleNamespace{}, err } + level.Debug(util.Logger).Log("msg", "listing rule groups", "handle", obj.Name) + + _, namespace, _, err := decomposeRuleHande(obj.Name) + if err != nil { + return []configs.RuleNamespace{}, err + } if obj.Name == options.Namespace && options.Namespace != "" { - level.Debug(util.Logger).Log("msg", "listing rule groups", "namespace", obj.Name) - ns, err := g.getRuleNamespace(ctx, options.UserID, obj.Name) - if err != nil { - return []configs.RuleNamespace{}, err - } - nss = append(nss, ns) + namespaces[namespace] = true } } + nss := []configs.RuleNamespace{} + + for ns := range namespaces { + ns, err := g.getRuleNamespace(ctx, options.UserID, ns) + if err != nil { + return []configs.RuleNamespace{}, err + } + nss = append(nss, ns) + } + return nss, nil } @@ -403,3 +417,14 @@ func generateRuleHandle(id, namespace, name string) string { } return prefix + namespace + "/" + name } + +func decomposeRuleHande(handle string) (string, string, string, error) { + components := strings.Split(handle, "/") + + if len(components) != 4 { + return "", "", "", ErrBadRuleGroup + } + + // Return `user, namespace, group_name` + return components[1], components[2], components[3], nil +} diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 804b0c27a65..1ace701d2a7 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -37,10 +37,10 @@ func (a *API) RegisterRoutes(r *mux.Router) { name, method, path string handler http.HandlerFunc }{ - {"list_rules", "GET", "/api/prom/rules/", a.listRules}, - {"list_rules_namespace", "GET", "/api/prom/rules/{namespace}/", a.listRules}, + {"list_rules", "GET", "/api/prom/rules", a.listRules}, + {"list_rules_namespace", "GET", "/api/prom/rules/{namespace}", a.listRules}, {"get_rulegroup", "GET", "/api/prom/rules/{namespace}/{groupName}", a.getRuleGroup}, - {"set_rulegroup", "POST", "/api/prom/rules/{namespace}/", a.setRuleGroup}, + {"set_rulegroup", "POST", "/api/prom/rules/{namespace}", a.setRuleGroup}, {"delete_rulegroup", "DELETE", "/api/prom/rules/{namespace}/{groupName}", a.deleteRuleGroup}, } { r.Handle(route.path, route.handler).Methods(route.method).Name(route.name) From 228a8987310b4eb1221218cf9273b767e807bb86 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Thu, 27 Jun 2019 00:32:50 -0400 Subject: [PATCH 14/60] fix listing rule namesaces Signed-off-by: Jacob Lisi --- pkg/configs/storage/clients/gcp/gcs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/configs/storage/clients/gcp/gcs.go b/pkg/configs/storage/clients/gcp/gcs.go index 022858b2258..93425faa65d 100644 --- a/pkg/configs/storage/clients/gcp/gcs.go +++ b/pkg/configs/storage/clients/gcp/gcs.go @@ -268,7 +268,7 @@ func (g *gcsConfigClient) ListRuleGroups(ctx context.Context, options configs.Ru if err != nil { return []configs.RuleNamespace{}, err } - if obj.Name == options.Namespace && options.Namespace != "" { + if obj.Name == options.Namespace || options.Namespace == "" { namespaces[namespace] = true } } From 502e70aeb153576207fca0452c94ff952588fe99 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Thu, 27 Jun 2019 00:46:54 -0400 Subject: [PATCH 15/60] fix polling and add xtra logging Signed-off-by: Jacob Lisi --- pkg/configs/storage/clients/client_test.go | 59 +++++++++++++++++++++ pkg/configs/storage/clients/gcp/fixtures.go | 37 +++++++++++++ pkg/configs/storage/clients/gcp/gcs.go | 18 ++++--- pkg/configs/storage/clients/utils.go | 38 +++++++++++++ 4 files changed, 146 insertions(+), 6 deletions(-) create mode 100644 pkg/configs/storage/clients/gcp/fixtures.go create mode 100644 pkg/configs/storage/clients/utils.go diff --git a/pkg/configs/storage/clients/client_test.go b/pkg/configs/storage/clients/client_test.go index 2ea39020bba..da6a7cb376e 100644 --- a/pkg/configs/storage/clients/client_test.go +++ b/pkg/configs/storage/clients/client_test.go @@ -1 +1,60 @@ package clients + +import ( + "context" + "math/rand" + "sort" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cortexproject/cortex/pkg/chunk" + "github.com/cortexproject/cortex/pkg/chunk/testutils" + "github.com/prometheus/common/model" +) + +func TestRuleStoreBasic(t *testing.T) { + forAllFixtures(t, func(t *testing.T, _ chunk.IndexClient, client chunk.ObjectClient) { + const batchSize = 5 + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + // Write a few batches of chunks. + written := []string{} + for i := 0; i < 5; i++ { + keys, chunks, err := testutils.CreateChunks(i, batchSize, model.Now()) + require.NoError(t, err) + written = append(written, keys...) + err = client.PutChunks(ctx, chunks) + require.NoError(t, err) + } + + // Get a few batches of chunks. + for batch := 0; batch < 50; batch++ { + keysToGet := map[string]struct{}{} + chunksToGet := []chunk.Chunk{} + for len(chunksToGet) < batchSize { + key := written[rand.Intn(len(written))] + if _, ok := keysToGet[key]; ok { + continue + } + keysToGet[key] = struct{}{} + chunk, err := chunk.ParseExternalKey(userID, key) + require.NoError(t, err) + chunksToGet = append(chunksToGet, chunk) + } + + chunksWeGot, err := client.GetChunks(ctx, chunksToGet) + require.NoError(t, err) + require.Equal(t, len(chunksToGet), len(chunksWeGot)) + + sort.Sort(ByKey(chunksToGet)) + sort.Sort(ByKey(chunksWeGot)) + for i := 0; i < len(chunksWeGot); i++ { + require.Equal(t, chunksToGet[i].ExternalKey(), chunksWeGot[i].ExternalKey(), strconv.Itoa(i)) + } + } + }) +} diff --git a/pkg/configs/storage/clients/gcp/fixtures.go b/pkg/configs/storage/clients/gcp/fixtures.go new file mode 100644 index 00000000000..82977d17a76 --- /dev/null +++ b/pkg/configs/storage/clients/gcp/fixtures.go @@ -0,0 +1,37 @@ +package gcp + +import ( + "context" + + "github.com/fsouza/fake-gcs-server/fakestorage" + + "github.com/cortexproject/cortex/pkg/configs" +) + +const ( + proj, instance = "proj", "instance" +) + +type fixture struct { + gcssrv *fakestorage.Server + + name string +} + +func (f *fixture) Name() string { + return f.name +} + +func (f *fixture) Clients() (store configs.ConfigStore, err error) { + f.gcssrv = fakestorage.NewServer(nil) + f.gcssrv.CreateBucket("configdb") + + return NewGCSConfigClient(context.Background(), GCSConfig{ + BucketName: "configdb", + }) +} + +func (f *fixture) Teardown() error { + f.gcssrv.Stop() + return nil +} diff --git a/pkg/configs/storage/clients/gcp/gcs.go b/pkg/configs/storage/clients/gcp/gcs.go index 93425faa65d..543c8d563d8 100644 --- a/pkg/configs/storage/clients/gcp/gcs.go +++ b/pkg/configs/storage/clients/gcp/gcs.go @@ -147,29 +147,34 @@ func (g *gcsConfigClient) DeleteAlertConfig(ctx context.Context, userID string) func (g *gcsConfigClient) PollRules(ctx context.Context) (map[string][]configs.RuleGroup, error) { objs := g.bucket.Objects(ctx, &storage.Query{ - Delimiter: "/", - Prefix: rulePrefix, + Prefix: rulePrefix, }) updatedUsers := []string{} for { - user, err := objs.Next() + handle, err := objs.Next() if err == iterator.Done { break } - level.Debug(util.Logger).Log("msg", "checking gcs for updated rules", "user", user.Name) + user, _, _, err := decomposeRuleHande(handle.Name) + if err != nil { + level.Error(util.Logger).Log("msg", "unable to poll user for rules", "user", user) + } + + level.Debug(util.Logger).Log("msg", "checking gcs for updated rules", "user", user) if err != nil { return nil, err } - updated, err := g.checkUser(ctx, user.Name) + updated, err := g.checkUser(ctx, user) if err != nil { return nil, err } if updated { - updatedUsers = append(updatedUsers, user.Name) + level.Info(util.Logger).Log("msg", "updated rules found", "user", user) + updatedUsers = append(updatedUsers, user) } } @@ -276,6 +281,7 @@ func (g *gcsConfigClient) ListRuleGroups(ctx context.Context, options configs.Ru nss := []configs.RuleNamespace{} for ns := range namespaces { + level.Debug(util.Logger).Log("msg", "retrieving rule namespace", "user", options.UserID, "namespace", ns) ns, err := g.getRuleNamespace(ctx, options.UserID, ns) if err != nil { return []configs.RuleNamespace{}, err diff --git a/pkg/configs/storage/clients/utils.go b/pkg/configs/storage/clients/utils.go new file mode 100644 index 00000000000..b7c48a255ef --- /dev/null +++ b/pkg/configs/storage/clients/utils.go @@ -0,0 +1,38 @@ +package clients + +import ( + "testing" + + "github.com/cortexproject/cortex/pkg/configs" + "github.com/cortexproject/cortex/pkg/configs/storage/clients/gcp" + "github.com/cortexproject/cortex/pkg/configs/storage/clients/client" + "github.com/cortexproject/cortex/pkg/chunk/testutils" + "github.com/stretchr/testify/require" +) + +const ( + userID = "userID" + tableName = "test" +) + +type configClientTest func(*testing.T, configs.AlertStore, configs.RuleStore) + +func forAllFixtures(t *testing.T, storageClientTest storageClientTest) { + var fixtures []testutils.Fixture + fixtures = append(fixtures, client.Fixtures...) + fixtures = append(fixtures, gcp.Fixtures...) + + cassandraFixtures, err := cassandra.Fixtures() + require.NoError(t, err) + fixtures = append(fixtures, cassandraFixtures...) + + for _, fixture := range fixtures { + t.Run(fixture.Name(), func(t *testing.T) { + indexClient, objectClient, err := testutils.Setup(fixture, tableName) + require.NoError(t, err) + defer fixture.Teardown() + + storageClientTest(t, indexClient, objectClient) + }) + } +} From e768c3197f296c7271f53f44f7780aa50c0f32cb Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Thu, 27 Jun 2019 00:57:21 -0400 Subject: [PATCH 16/60] fix double printing api response Signed-off-by: Jacob Lisi --- pkg/ruler/api.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 1ace701d2a7..5b2be3005ce 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -103,7 +103,6 @@ func (a *API) listRules(w http.ResponseWriter, r *http.Request) { } w.Header().Set("Content-Type", "application/yaml") - w.Write(d) if _, err := w.Write(d); err != nil { level.Error(logger).Log("msg", "error writing yaml response", "err", err) http.Error(w, err.Error(), http.StatusInternalServerError) @@ -159,7 +158,6 @@ func (a *API) getRuleGroup(w http.ResponseWriter, r *http.Request) { } w.Header().Set("Content-Type", "application/yaml") - w.Write(d) if _, err := w.Write(d); err != nil { level.Error(logger).Log("msg", "error writing yaml response", "err", err) http.Error(w, err.Error(), http.StatusInternalServerError) From a13f738e881d319b86d744d0f025b9a96b26783b Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Thu, 27 Jun 2019 01:11:19 -0400 Subject: [PATCH 17/60] fix listing rules with namespace Signed-off-by: Jacob Lisi --- pkg/ruler/api.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 5b2be3005ce..1d93a46e704 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -73,8 +73,8 @@ func (a *API) listRules(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - namespace, set := vars["namespace"] - if set { + namespace := vars["namespace"] + if namespace != "" { level.Debug(logger).Log("msg", "retrieving rule groups with namespace", "userID", userID, "namespace", namespace) options.Namespace = namespace } @@ -187,8 +187,8 @@ func (a *API) setRuleGroup(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - namespace, set := vars["namespace"] - if !set { + namespace := vars["namespace"] + if namespace != "" { level.Error(logger).Log("err", err.Error()) http.Error(w, ErrNoNamespace.Error(), http.StatusBadRequest) return From 3e683945214e15bf9b4f6caf99770b9c22771cdf Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Thu, 27 Jun 2019 01:21:25 -0400 Subject: [PATCH 18/60] fix gcs listing rules by namespace Signed-off-by: Jacob Lisi --- pkg/configs/storage/clients/gcp/gcs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/configs/storage/clients/gcp/gcs.go b/pkg/configs/storage/clients/gcp/gcs.go index 543c8d563d8..7888662a08a 100644 --- a/pkg/configs/storage/clients/gcp/gcs.go +++ b/pkg/configs/storage/clients/gcp/gcs.go @@ -273,7 +273,7 @@ func (g *gcsConfigClient) ListRuleGroups(ctx context.Context, options configs.Ru if err != nil { return []configs.RuleNamespace{}, err } - if obj.Name == options.Namespace || options.Namespace == "" { + if namespace == options.Namespace || options.Namespace == "" { namespaces[namespace] = true } } From a7621f4721d95b6f30976fcc6cf42728c2091a27 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Thu, 27 Jun 2019 12:56:20 -0400 Subject: [PATCH 19/60] fix test cases Signed-off-by: Jacob Lisi --- pkg/configs/storage/clients/client_test.go | 74 +++++++++------------ pkg/configs/storage/clients/gcp/fixtures.go | 9 +++ pkg/configs/storage/clients/utils.go | 38 ----------- pkg/configs/storage/testutils/testutils.go | 12 ++++ 4 files changed, 52 insertions(+), 81 deletions(-) delete mode 100644 pkg/configs/storage/clients/utils.go create mode 100644 pkg/configs/storage/testutils/testutils.go diff --git a/pkg/configs/storage/clients/client_test.go b/pkg/configs/storage/clients/client_test.go index da6a7cb376e..f97e3ef19a9 100644 --- a/pkg/configs/storage/clients/client_test.go +++ b/pkg/configs/storage/clients/client_test.go @@ -2,59 +2,47 @@ package clients import ( "context" - "math/rand" - "sort" - "strconv" "testing" "time" - "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/chunk/testutils" - "github.com/prometheus/common/model" + "github.com/cortexproject/cortex/pkg/configs" + "github.com/cortexproject/cortex/pkg/configs/storage/clients/gcp" + "github.com/cortexproject/cortex/pkg/configs/storage/testutils" + "github.com/prometheus/prometheus/pkg/rulefmt" +) + +const ( + userID = "userID" + namespace = "default" +) + +var ( + exampleRGOne = rulefmt.RuleGroup{ + Name: "example_rulegroup_one", + } + exampleRGTwo = rulefmt.RuleGroup{ + Name: "example_rulegroup_two", + } ) func TestRuleStoreBasic(t *testing.T) { - forAllFixtures(t, func(t *testing.T, _ chunk.IndexClient, client chunk.ObjectClient) { + forAllFixtures(t, func(t *testing.T, client configs.ConfigStore) { const batchSize = 5 ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() - // Write a few batches of chunks. - written := []string{} - for i := 0; i < 5; i++ { - keys, chunks, err := testutils.CreateChunks(i, batchSize, model.Now()) - require.NoError(t, err) - written = append(written, keys...) - err = client.PutChunks(ctx, chunks) - require.NoError(t, err) - } - - // Get a few batches of chunks. - for batch := 0; batch < 50; batch++ { - keysToGet := map[string]struct{}{} - chunksToGet := []chunk.Chunk{} - for len(chunksToGet) < batchSize { - key := written[rand.Intn(len(written))] - if _, ok := keysToGet[key]; ok { - continue - } - keysToGet[key] = struct{}{} - chunk, err := chunk.ParseExternalKey(userID, key) - require.NoError(t, err) - chunksToGet = append(chunksToGet, chunk) - } - - chunksWeGot, err := client.GetChunks(ctx, chunksToGet) - require.NoError(t, err) - require.Equal(t, len(chunksToGet), len(chunksWeGot)) - - sort.Sort(ByKey(chunksToGet)) - sort.Sort(ByKey(chunksWeGot)) - for i := 0; i < len(chunksWeGot); i++ { - require.Equal(t, chunksToGet[i].ExternalKey(), chunksWeGot[i].ExternalKey(), strconv.Itoa(i)) - } - } + err := client.SetRuleGroup(ctx, userID, namespace, exampleRGOne) + assert.NoError(t, err) + + rg, err := client.GetRuleGroup(ctx, userID, namespace, exampleRGOne.Name) + assert.NoError(t, err) + assert.Equal(t, exampleRGOne.Name, rg.Name) }) } + +func forAllFixtures(t *testing.T, f func(t *testing.T, client configs.ConfigStore)) { + var fixtures []testutils.Fixture + fixtures = append(fixtures, gcp.Fixtures...) +} diff --git a/pkg/configs/storage/clients/gcp/fixtures.go b/pkg/configs/storage/clients/gcp/fixtures.go index 82977d17a76..d18a66677f0 100644 --- a/pkg/configs/storage/clients/gcp/fixtures.go +++ b/pkg/configs/storage/clients/gcp/fixtures.go @@ -6,6 +6,7 @@ import ( "github.com/fsouza/fake-gcs-server/fakestorage" "github.com/cortexproject/cortex/pkg/configs" + "github.com/cortexproject/cortex/pkg/configs/storage/testutils" ) const ( @@ -35,3 +36,11 @@ func (f *fixture) Teardown() error { f.gcssrv.Stop() return nil } + +// Fixtures for unit testing GCP storage. +var Fixtures = func() []testutils.Fixture { + fixtures := []testutils.Fixture{ + &fixture{name: "gcs"}, + } + return fixtures +}() diff --git a/pkg/configs/storage/clients/utils.go b/pkg/configs/storage/clients/utils.go deleted file mode 100644 index b7c48a255ef..00000000000 --- a/pkg/configs/storage/clients/utils.go +++ /dev/null @@ -1,38 +0,0 @@ -package clients - -import ( - "testing" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/configs/storage/clients/gcp" - "github.com/cortexproject/cortex/pkg/configs/storage/clients/client" - "github.com/cortexproject/cortex/pkg/chunk/testutils" - "github.com/stretchr/testify/require" -) - -const ( - userID = "userID" - tableName = "test" -) - -type configClientTest func(*testing.T, configs.AlertStore, configs.RuleStore) - -func forAllFixtures(t *testing.T, storageClientTest storageClientTest) { - var fixtures []testutils.Fixture - fixtures = append(fixtures, client.Fixtures...) - fixtures = append(fixtures, gcp.Fixtures...) - - cassandraFixtures, err := cassandra.Fixtures() - require.NoError(t, err) - fixtures = append(fixtures, cassandraFixtures...) - - for _, fixture := range fixtures { - t.Run(fixture.Name(), func(t *testing.T) { - indexClient, objectClient, err := testutils.Setup(fixture, tableName) - require.NoError(t, err) - defer fixture.Teardown() - - storageClientTest(t, indexClient, objectClient) - }) - } -} diff --git a/pkg/configs/storage/testutils/testutils.go b/pkg/configs/storage/testutils/testutils.go new file mode 100644 index 00000000000..96b79fca50f --- /dev/null +++ b/pkg/configs/storage/testutils/testutils.go @@ -0,0 +1,12 @@ +package testutils + +import ( + "github.com/cortexproject/cortex/pkg/configs" +) + +// Fixture type for per-backend testing. +type Fixture interface { + Name() string + Clients() (configs.ConfigStore, error) + Teardown() error +} From 2dbc85ee6dc9e89300ab2e824fdd09ce02b36eed Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 28 Jun 2019 11:00:33 -0400 Subject: [PATCH 20/60] return map instead of array for rules api Signed-off-by: Jacob Lisi --- pkg/configs/configs.go | 2 +- pkg/configs/storage/clients/client/client.go | 2 +- pkg/configs/storage/clients/gcp/gcs.go | 18 +++++++++--------- pkg/configs/storage/instrumented.go | 4 ++-- pkg/ruler/ruler_test.go | 6 +++--- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pkg/configs/configs.go b/pkg/configs/configs.go index ed415b3c9f3..15a3d9072b6 100644 --- a/pkg/configs/configs.go +++ b/pkg/configs/configs.go @@ -44,7 +44,7 @@ type RuleStoreConditions struct { type RuleStore interface { PollRules(ctx context.Context) (map[string][]RuleGroup, error) - ListRuleGroups(ctx context.Context, options RuleStoreConditions) ([]RuleNamespace, error) + ListRuleGroups(ctx context.Context, options RuleStoreConditions) (map[string]RuleNamespace, error) GetRuleGroup(ctx context.Context, userID, namespace, group string) (rulefmt.RuleGroup, error) SetRuleGroup(ctx context.Context, userID, namespace string, group rulefmt.RuleGroup) error DeleteRuleGroup(ctx context.Context, userID, namespace string, group string) error diff --git a/pkg/configs/storage/clients/client/client.go b/pkg/configs/storage/clients/client/client.go index e15f66af866..36fd2e45d46 100644 --- a/pkg/configs/storage/clients/client/client.go +++ b/pkg/configs/storage/clients/client/client.go @@ -185,7 +185,7 @@ func (c *configsClient) DeleteAlertConfig(ctx context.Context, userID string) er return errors.New("remote configdb client does not implement DeleteAlertConfig") } -func (c *configsClient) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) ([]configs.RuleNamespace, error) { +func (c *configsClient) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) (map[string]configs.RuleNamespace, error) { return nil, errors.New("remote configdb client does not implement ListRule") } diff --git a/pkg/configs/storage/clients/gcp/gcs.go b/pkg/configs/storage/clients/gcp/gcs.go index 7888662a08a..fdf3b678a52 100644 --- a/pkg/configs/storage/clients/gcp/gcs.go +++ b/pkg/configs/storage/clients/gcp/gcs.go @@ -251,7 +251,7 @@ func (g *gcsConfigClient) getAllRuleGroups(ctx context.Context, userID string) ( return rgs, nil } -func (g *gcsConfigClient) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) ([]configs.RuleNamespace, error) { +func (g *gcsConfigClient) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) (map[string]configs.RuleNamespace, error) { it := g.bucket.Objects(ctx, &storage.Query{ Prefix: generateRuleHandle(options.UserID, options.Namespace, ""), }) @@ -264,29 +264,29 @@ func (g *gcsConfigClient) ListRuleGroups(ctx context.Context, options configs.Ru } if err != nil { - return []configs.RuleNamespace{}, err + return nil, err } level.Debug(util.Logger).Log("msg", "listing rule groups", "handle", obj.Name) _, namespace, _, err := decomposeRuleHande(obj.Name) if err != nil { - return []configs.RuleNamespace{}, err + return nil, err } if namespace == options.Namespace || options.Namespace == "" { namespaces[namespace] = true } } - nss := []configs.RuleNamespace{} + nss := map[string]configs.RuleNamespace{} - for ns := range namespaces { - level.Debug(util.Logger).Log("msg", "retrieving rule namespace", "user", options.UserID, "namespace", ns) - ns, err := g.getRuleNamespace(ctx, options.UserID, ns) + for namespace := range namespaces { + level.Debug(util.Logger).Log("msg", "retrieving rule namespace", "user", options.UserID, "namespace", namespace) + ns, err := g.getRuleNamespace(ctx, options.UserID, namespace) if err != nil { - return []configs.RuleNamespace{}, err + return nil, err } - nss = append(nss, ns) + nss[namespace] = ns } return nss, nil diff --git a/pkg/configs/storage/instrumented.go b/pkg/configs/storage/instrumented.go index 0aabf394c02..2cc212af237 100644 --- a/pkg/configs/storage/instrumented.go +++ b/pkg/configs/storage/instrumented.go @@ -71,8 +71,8 @@ func (i instrumented) PollRules(ctx context.Context) (map[string][]configs.RuleG return cfgs, err } -func (i instrumented) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) ([]configs.RuleNamespace, error) { - var cfgs []configs.RuleNamespace +func (i instrumented) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) (map[string]configs.RuleNamespace, error) { + var cfgs map[string]configs.RuleNamespace err := instrument.CollectedRequest(context.Background(), "Configs.ListRuleGroups", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { var err error cfgs, err = i.next.ListRuleGroups(ctx, options) diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index 95518a60fa4..f81936ed288 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -36,11 +36,11 @@ func (m *mockRuleStore) PollRules(ctx context.Context) (map[string][]configs.Rul return pollPayload, nil } -func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) ([]configs.RuleNamespace, error) { +func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) (map[string]configs.RuleNamespace, error) { groupPrefix := userID + ":" namespaces := []string{} - nss := []configs.RuleNamespace{} + nss := map[string]configs.RuleNamespace{} for n := range m.rules { if strings.HasPrefix(n, groupPrefix) { components := strings.Split(n, ":") @@ -61,7 +61,7 @@ func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options configs.Rule continue } - nss = append(nss, ns) + nss[n] = ns } return nss, nil From cd902a28edfb2f4ce743851658cb7420d5da77a0 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 28 Jun 2019 11:54:25 -0400 Subject: [PATCH 21/60] fix alert prefix for gcs config backend Signed-off-by: Jacob Lisi --- pkg/configs/configs.go | 3 +-- pkg/configs/storage/clients/gcp/gcs.go | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/configs/configs.go b/pkg/configs/configs.go index 15a3d9072b6..c6fea9df066 100644 --- a/pkg/configs/configs.go +++ b/pkg/configs/configs.go @@ -67,8 +67,7 @@ type RuleGroup interface { // rule file format, if no namespace is set, the default namespace // is used type RuleNamespace struct { - Namespace string `yaml:"namespace"` - Groups []rulefmt.RuleGroup `yaml:"groups"` + Groups []rulefmt.RuleGroup `yaml:"groups"` } // Validate each rule in the rule namespace is valid diff --git a/pkg/configs/storage/clients/gcp/gcs.go b/pkg/configs/storage/clients/gcp/gcs.go index fdf3b678a52..be0a2688ed8 100644 --- a/pkg/configs/storage/clients/gcp/gcs.go +++ b/pkg/configs/storage/clients/gcp/gcs.go @@ -88,7 +88,7 @@ func (g *gcsConfigClient) getAlertConfig(ctx context.Context, obj string) (confi func (g *gcsConfigClient) PollAlerts(ctx context.Context) (map[string]configs.AlertConfig, error) { objs := g.bucket.Objects(ctx, &storage.Query{ - Prefix: rulePrefix, + Prefix: alertPrefix, }) alertMap := map[string]configs.AlertConfig{} @@ -298,8 +298,7 @@ func (g *gcsConfigClient) getRuleNamespace(ctx context.Context, userID string, n }) ns := configs.RuleNamespace{ - Namespace: namespace, - Groups: []rulefmt.RuleGroup{}, + Groups: []rulefmt.RuleGroup{}, } for { From 343bf9ba2dda7f42f83c59c7fa319dbbcb62d4e9 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 28 Jun 2019 18:32:34 -0400 Subject: [PATCH 22/60] fix ruler test Signed-off-by: Jacob Lisi --- pkg/ruler/ruler_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index f81936ed288..fa38763ac3e 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -71,8 +71,7 @@ func (m *mockRuleStore) getRuleNamespace(ctx context.Context, userID string, nam groupPrefix := userID + ":" + namespace + ":" ns := configs.RuleNamespace{ - Namespace: namespace, - Groups: []rulefmt.RuleGroup{}, + Groups: []rulefmt.RuleGroup{}, } for n, g := range m.rules { if strings.HasPrefix(n, groupPrefix) { From 13c6b001b8e9bd1fe3ab33cb083a03fba6a3493c Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Sun, 30 Jun 2019 10:44:15 -0400 Subject: [PATCH 23/60] fix setting namespace for rule creation Signed-off-by: Jacob Lisi --- pkg/ruler/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 1d93a46e704..cdd99fa3132 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -188,8 +188,8 @@ func (a *API) setRuleGroup(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) namespace := vars["namespace"] - if namespace != "" { - level.Error(logger).Log("err", err.Error()) + if namespace == "" { + level.Error(logger).Log("err", "no namespace provided with rule group") http.Error(w, ErrNoNamespace.Error(), http.StatusBadRequest) return } From 4bdee8320c4c3e5d8538424443994f54e1377a13 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Tue, 2 Jul 2019 13:51:56 -0400 Subject: [PATCH 24/60] update rule group creation endpoint Signed-off-by: Jacob Lisi --- pkg/ruler/api.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index cdd99fa3132..2fada572122 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -40,7 +40,7 @@ func (a *API) RegisterRoutes(r *mux.Router) { {"list_rules", "GET", "/api/prom/rules", a.listRules}, {"list_rules_namespace", "GET", "/api/prom/rules/{namespace}", a.listRules}, {"get_rulegroup", "GET", "/api/prom/rules/{namespace}/{groupName}", a.getRuleGroup}, - {"set_rulegroup", "POST", "/api/prom/rules/{namespace}", a.setRuleGroup}, + {"set_rulegroup", "POST", "/api/prom/rules/{namespace}", a.createRuleGroup}, {"delete_rulegroup", "DELETE", "/api/prom/rules/{namespace}/{groupName}", a.deleteRuleGroup}, } { r.Handle(route.path, route.handler).Methods(route.method).Name(route.name) @@ -165,7 +165,7 @@ func (a *API) getRuleGroup(w http.ResponseWriter, r *http.Request) { } } -func (a *API) setRuleGroup(w http.ResponseWriter, r *http.Request) { +func (a *API) createRuleGroup(w http.ResponseWriter, r *http.Request) { userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) if err != nil { http.Error(w, err.Error(), http.StatusUnauthorized) @@ -225,7 +225,8 @@ func (a *API) setRuleGroup(w http.ResponseWriter, r *http.Request) { return } - w.WriteHeader(http.StatusOK) + // Return a status accepted because the rule has been stored and queued for polling, but is not currently active + w.WriteHeader(http.StatusAccepted) } func (a *API) deleteRuleGroup(w http.ResponseWriter, r *http.Request) { From 2b474f9f59b046493f9ce89691c05926db750b94 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 5 Jul 2019 22:56:19 -0400 Subject: [PATCH 25/60] remove unused metrics and add user metric for evaluation failures Signed-off-by: Jacob Lisi --- pkg/ruler/ruler.go | 6 ++++++ pkg/ruler/scheduler.go | 10 ---------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 123903c74f7..833e5978377 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -48,6 +48,11 @@ var ( Name: "ruler_ring_check_errors_total", Help: "Number of errors that have occurred when checking the ring for ownership", }) + evalFailures = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "cortex", + Name: "ruler_evalauation_failures_total", + Help: "Total number of rule groups failed to be evaluated.", + }, []string{"id"}) ruleMetrics *rules.Metrics ) @@ -313,6 +318,7 @@ func (r *Ruler) Evaluate(userID string, item *workItem) { if err := ctx.Err(); err == nil { cancelTimeout() // release resources } else { + evalFailures.WithLabelValues(userID).Inc() level.Warn(logger).Log("msg", "context error", "error", err) } diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index 8c0dfa2544b..bc37f168083 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -29,21 +29,11 @@ const ( ) var ( - totalConfigs = promauto.NewGauge(prometheus.GaugeOpts{ - Namespace: "cortex", - Name: "scheduler_configs_total", - Help: "How many configs the scheduler knows about.", - }) totalRuleGroups = promauto.NewGauge(prometheus.GaugeOpts{ Namespace: "cortex", Name: "scheduler_groups_total", Help: "How many rule groups the scheduler is currently evaluating", }) - configUpdates = promauto.NewCounter(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "scheduler_config_updates_total", - Help: "How many config updates the scheduler has made.", - }) ) type workItem struct { From 71984c9ee6d23ca8bbafd490f69d3efe6caae4e0 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Sun, 7 Jul 2019 19:29:54 -0400 Subject: [PATCH 26/60] add comment for later group instrumentation Signed-off-by: Jacob Lisi --- pkg/ruler/rulegroup/rulegroup.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/ruler/rulegroup/rulegroup.go b/pkg/ruler/rulegroup/rulegroup.go index dba13b8c1a5..4cb3299b8bb 100644 --- a/pkg/ruler/rulegroup/rulegroup.go +++ b/pkg/ruler/rulegroup/rulegroup.go @@ -8,6 +8,8 @@ import ( ) // TODO: Add a lazy rule group that only loads rules when they are needed +// TODO: The cortex project should implement a separate Group struct from +// the prometheus project. This will allow for more precise instrumentation type ruleGroup struct { name string From 299f4d257772de3eab7c8c2e63071357223f3576 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Sun, 7 Jul 2019 19:48:28 -0400 Subject: [PATCH 27/60] use promauto to register metrics Signed-off-by: Jacob Lisi --- pkg/ruler/rulegroup/rulegroup.go | 2 +- pkg/ruler/worker.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/ruler/rulegroup/rulegroup.go b/pkg/ruler/rulegroup/rulegroup.go index 4cb3299b8bb..c6e8e4819ef 100644 --- a/pkg/ruler/rulegroup/rulegroup.go +++ b/pkg/ruler/rulegroup/rulegroup.go @@ -24,7 +24,7 @@ func (rg *ruleGroup) Rules(ctx context.Context) ([]rules.Rule, error) { } func (rg *ruleGroup) Name() string { - return rg.user + "/" + rg.namespace + "/" + rg.name + return rg.namespace + "/" + rg.name } // NewRuleGroup returns a rulegroup diff --git a/pkg/ruler/worker.go b/pkg/ruler/worker.go index a56eae0f707..e269876698d 100644 --- a/pkg/ruler/worker.go +++ b/pkg/ruler/worker.go @@ -6,20 +6,21 @@ import ( "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) var ( - blockedWorkers = prometheus.NewGauge(prometheus.GaugeOpts{ + blockedWorkers = promauto.NewGauge(prometheus.GaugeOpts{ Namespace: "cortex", Name: "blocked_workers", Help: "How many workers are waiting on an item to be ready.", }) - workerIdleTime = prometheus.NewCounter(prometheus.CounterOpts{ + workerIdleTime = promauto.NewCounter(prometheus.CounterOpts{ Namespace: "cortex", Name: "worker_idle_seconds_total", Help: "How long workers have spent waiting for work.", }) - evalLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ + evalLatency = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: "cortex", Name: "group_evaluation_latency_seconds", Help: "How far behind the target time each rule group executed.", From 5fbc094a424f1f185c1bfae11ffb5502b197f5e2 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Sun, 7 Jul 2019 20:44:05 -0400 Subject: [PATCH 28/60] fix log messages for owning rules Signed-off-by: Jacob Lisi --- pkg/ruler/ruler.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 833e5978377..f932b8d684c 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -336,9 +336,10 @@ func (r *Ruler) ownsRule(hash uint32) bool { return true } if rlrs.Ingesters[0].Addr == r.lifecycler.Addr { + level.Debug(util.Logger).Log("msg", "rule group owned", "owner_addr", rlrs.Ingesters[0].Addr, "addr", r.lifecycler.Addr) return true } - level.Debug(util.Logger).Log("msg", "rule group not owned, address does not match", "owner", rlrs.Ingesters[0].Addr, "current", r.cfg.LifecyclerConfig.Addr) + level.Debug(util.Logger).Log("msg", "rule group not owned, address does not match", "owner_addr", rlrs.Ingesters[0].Addr, "addr", r.lifecycler.Addr) return false } From 516da88fd29d18282a7ae479e7a00eb13715e288 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Mon, 8 Jul 2019 11:46:27 -0400 Subject: [PATCH 29/60] fix typos Signed-off-by: Jacob Lisi --- pkg/configs/configs.go | 2 +- pkg/ruler/scheduler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/configs/configs.go b/pkg/configs/configs.go index c6fea9df066..64d087d257a 100644 --- a/pkg/configs/configs.go +++ b/pkg/configs/configs.go @@ -36,7 +36,7 @@ type RuleStoreConditions struct { UserID string // Namespaces filters results only rule groups with the specified namespace - // are retreived + // are retrieved Namespace string } diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index bc37f168083..8d50da9575b 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -190,7 +190,7 @@ func (s *scheduler) addUserConfig(ctx context.Context, userID string, rgs []conf } func (s *scheduler) updateUserConfig(ctx context.Context, cfg userConfig) { - // Retreive any previous configuration and update to the new configuration + // Retrieve any previous configuration and update to the new configuration s.Lock() curr, exists := s.cfgs[cfg.id] s.cfgs[cfg.id] = cfg From 0a9f1846e6e2ba66228c4049b39dadc713e30dbd Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Mon, 8 Jul 2019 15:26:18 -0400 Subject: [PATCH 30/60] keep namespace in struct for file parsing, unexport gcs error Signed-off-by: Jacob Lisi --- pkg/configs/configs.go | 3 +++ pkg/configs/storage/clients/gcp/gcs.go | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/configs/configs.go b/pkg/configs/configs.go index 64d087d257a..2ad6832b4a1 100644 --- a/pkg/configs/configs.go +++ b/pkg/configs/configs.go @@ -67,6 +67,9 @@ type RuleGroup interface { // rule file format, if no namespace is set, the default namespace // is used type RuleNamespace struct { + // Namespace field only exists for setting namespace in namespace body instead of file name + Namespace string `yaml:"namespace,omitempty"` + Groups []rulefmt.RuleGroup `yaml:"groups"` } diff --git a/pkg/configs/storage/clients/gcp/gcs.go b/pkg/configs/storage/clients/gcp/gcs.go index be0a2688ed8..47d20a20d96 100644 --- a/pkg/configs/storage/clients/gcp/gcs.go +++ b/pkg/configs/storage/clients/gcp/gcs.go @@ -26,7 +26,7 @@ const ( ) var ( - ErrBadRuleGroup = errors.New("unable to decompose handle for rule object") + errBadRuleGroup = errors.New("unable to decompose handle for rule object") ) // GCSConfig is config for the GCS Chunk Client. @@ -427,7 +427,7 @@ func decomposeRuleHande(handle string) (string, string, string, error) { components := strings.Split(handle, "/") if len(components) != 4 { - return "", "", "", ErrBadRuleGroup + return "", "", "", errBadRuleGroup } // Return `user, namespace, group_name` From ae19f69c0b9b7924717c2bfb502786d25c579ac9 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Tue, 9 Jul 2019 13:44:45 -0400 Subject: [PATCH 31/60] use rulegroup in groupfn function Signed-off-by: Jacob Lisi --- pkg/configs/configs.go | 1 + pkg/ruler/rulegroup/rulegroup.go | 4 ++++ pkg/ruler/ruler.go | 11 ++++++++--- pkg/ruler/scheduler.go | 34 +++++++++++++++++--------------- 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/pkg/configs/configs.go b/pkg/configs/configs.go index 2ad6832b4a1..1f37eff5f29 100644 --- a/pkg/configs/configs.go +++ b/pkg/configs/configs.go @@ -61,6 +61,7 @@ type AlertConfig struct { type RuleGroup interface { Rules(ctx context.Context) ([]rules.Rule, error) Name() string + User() string } // RuleNamespace is used to parse a slightly modified prometheus diff --git a/pkg/ruler/rulegroup/rulegroup.go b/pkg/ruler/rulegroup/rulegroup.go index c6e8e4819ef..c5bcce214c4 100644 --- a/pkg/ruler/rulegroup/rulegroup.go +++ b/pkg/ruler/rulegroup/rulegroup.go @@ -27,6 +27,10 @@ func (rg *ruleGroup) Name() string { return rg.namespace + "/" + rg.name } +func (rg *ruleGroup) User() string { + return rg.user +} + // NewRuleGroup returns a rulegroup func NewRuleGroup(name, namespace, user string, rules []rules.Rule) configs.RuleGroup { return &ruleGroup{ diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index f932b8d684c..def7e6589d1 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -213,12 +213,17 @@ func (r *Ruler) Stop() { } } -func (r *Ruler) newGroup(userID string, groupName string, rls []rules.Rule) (*group, error) { +func (r *Ruler) newGroup(ctx context.Context, g configs.RuleGroup) (*group, error) { appendable := &appendableAppender{pusher: r.pusher} - notifier, err := r.getOrCreateNotifier(userID) + notifier, err := r.getOrCreateNotifier(g.User()) if err != nil { return nil, err } + rls, err := g.Rules(ctx) + if err != nil { + return nil, err + } + opts := &rules.ManagerOptions{ Appendable: appendable, QueryFunc: rules.EngineQueryFunc(r.engine, r.queryable), @@ -228,7 +233,7 @@ func (r *Ruler) newGroup(userID string, groupName string, rls []rules.Rule) (*gr Logger: util.Logger, Metrics: ruleMetrics, } - return newGroup(groupName, rls, appendable, opts), nil + return newGroup(g.Name(), rls, appendable, opts), nil } // sendAlerts implements a rules.NotifyFunc for a Notifier. diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index 8d50da9575b..97fd22571dd 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -15,7 +15,6 @@ import ( "github.com/jonboulle/clockwork" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/prometheus/rules" ) var backoffConfig = util.BackoffConfig{ @@ -34,6 +33,11 @@ var ( Name: "scheduler_groups_total", Help: "How many rule groups the scheduler is currently evaluating", }) + scheduleFailures = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "cortex", + Name: "scheduler_update_failures_total", + Help: "Number of failures when updating rule groups", + }) ) type workItem struct { @@ -71,7 +75,7 @@ type userConfig struct { rules []configs.RuleGroup } -type groupFactory func(userID string, groupName string, rls []rules.Rule) (*group, error) +type groupFactory func(context.Context, configs.RuleGroup) (*group, error) type scheduler struct { store configs.RuleStore @@ -145,35 +149,27 @@ func (s *scheduler) updateConfigs(ctx context.Context) error { } func (s *scheduler) addUserConfig(ctx context.Context, userID string, rgs []configs.RuleGroup) { - now := time.Now() - hasher := fnv.New64a() - level.Info(util.Logger).Log("msg", "scheduler: updating rules for user", "user_id", userID, "num_groups", len(rgs)) // create a new userchan for rulegroups of this user userChan := make(chan struct{}) ringHasher := fnv.New32a() - evalTime := computeNextEvalTime(hasher, now, float64(s.evaluationInterval.Nanoseconds()), userID) workItems := []workItem{} - for _, rg := range rgs { - rules, err := rg.Rules(ctx) - if err != nil { - level.Warn(util.Logger).Log("msg", "scheduler: failed to create group for user", "user_id", userID, "group", rg.Name(), "err", err) - return - } + evalTime := s.determineEvalTime(userID) - level.Debug(util.Logger).Log("msg", "scheduler: updating rules for user and group", "user_id", userID, "group", rg.Name(), "num_rules", len(rules)) - g, err := s.groupFn(userID, rg.Name(), rules) + for _, rg := range rgs { + level.Debug(util.Logger).Log("msg", "scheduler: updating rules for user and group", "user_id", userID, "group", rg.Name()) + grp, err := s.groupFn(ctx, rg) if err != nil { - level.Warn(util.Logger).Log("msg", "scheduler: failed to create group for user", "user_id", userID, "group", rg.Name(), "err", err) + level.Error(util.Logger).Log("msg", "scheduler: failed to create group for user", "user_id", userID, "group", rg.Name(), "err", err) return } ringHasher.Reset() ringHasher.Write([]byte(rg.Name())) hash := ringHasher.Sum32() - workItems = append(workItems, workItem{userID, rg.Name(), hash, g, evalTime, userChan}) + workItems = append(workItems, workItem{userID, rg.Name(), hash, grp, evalTime, userChan}) } s.updateUserConfig(ctx, userConfig{ @@ -203,6 +199,12 @@ func (s *scheduler) updateUserConfig(ctx context.Context, cfg userConfig) { return } +func (s *scheduler) determineEvalTime(userID string) time.Time { + now := time.Now() + hasher := fnv.New64a() + return computeNextEvalTime(hasher, now, float64(s.evaluationInterval.Nanoseconds()), userID) +} + // computeNextEvalTime Computes when a user's rules should be next evaluated, based on how far we are through an evaluation cycle func computeNextEvalTime(hasher hash.Hash64, now time.Time, intervalNanos float64, userID string) time.Time { // Compute how far we are into the current evaluation cycle From 325861dca531cfd57bd573e2561bae132df28408 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Tue, 9 Jul 2019 14:10:48 -0400 Subject: [PATCH 32/60] refactor package names and function names Signed-off-by: Jacob Lisi --- Makefile | 2 +- pkg/alertmanager/multitenant.go | 2 +- pkg/configs/configs.go | 2 +- pkg/configs/storage/clients/client/client.go | 6 +- pkg/configs/storage/clients/gcp/gcs.go | 28 ++++---- pkg/configs/storage/instrumented.go | 6 +- pkg/ruler/{rulegroup => group}/compat.go | 2 +- pkg/ruler/{rulegroup => group}/rulegroup.go | 2 +- .../{rulegroup => group}/rulegroup.pb.go | 68 +++++++++---------- .../{rulegroup => group}/rulegroup.proto | 4 +- pkg/ruler/scheduler_test.go | 6 +- 11 files changed, 64 insertions(+), 64 deletions(-) rename pkg/ruler/{rulegroup => group}/compat.go (99%) rename pkg/ruler/{rulegroup => group}/rulegroup.go (98%) rename pkg/ruler/{rulegroup => group}/rulegroup.pb.go (90%) rename pkg/ruler/{rulegroup => group}/rulegroup.proto (94%) diff --git a/Makefile b/Makefile index b0727189893..94966908111 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ pkg/ring/ring.pb.go: pkg/ring/ring.proto pkg/querier/frontend/frontend.pb.go: pkg/querier/frontend/frontend.proto pkg/chunk/storage/caching_index_client.pb.go: pkg/chunk/storage/caching_index_client.proto pkg/distributor/ha_tracker.pb.go: pkg/distributor/ha_tracker.proto -pkg/ruler/rulegroup/rulegroup.pb.go: pkg/ruler/rulegroup/rulegroup.proto +pkg/ruler/group/group.pb.go: pkg/ruler/group/group.proto all: $(UPTODATE_FILES) test: protos mod-check: protos diff --git a/pkg/alertmanager/multitenant.go b/pkg/alertmanager/multitenant.go index 0410c6ba092..ec0f20304e4 100644 --- a/pkg/alertmanager/multitenant.go +++ b/pkg/alertmanager/multitenant.go @@ -340,7 +340,7 @@ func (am *MultitenantAlertmanager) updateConfigs(now time.Time) error { // poll the configuration server. Not re-entrant. func (am *MultitenantAlertmanager) poll() (map[string]configs.AlertConfig, error) { - cfgs, err := am.store.PollAlerts(context.Background()) + cfgs, err := am.store.PollAlertConfigs(context.Background()) if err != nil { level.Warn(util.Logger).Log("msg", "MultitenantAlertmanager: configs server poll failed", "err", err) return nil, err diff --git a/pkg/configs/configs.go b/pkg/configs/configs.go index 1f37eff5f29..5ea974ee2f4 100644 --- a/pkg/configs/configs.go +++ b/pkg/configs/configs.go @@ -23,7 +23,7 @@ type ConfigStore interface { // AlertStore stores config information and template files to configure alertmanager tenants type AlertStore interface { - PollAlerts(ctx context.Context) (map[string]AlertConfig, error) + PollAlertConfigs(ctx context.Context) (map[string]AlertConfig, error) GetAlertConfig(ctx context.Context, userID string) (AlertConfig, error) SetAlertConfig(ctx context.Context, userID string, config AlertConfig) error diff --git a/pkg/configs/storage/clients/client/client.go b/pkg/configs/storage/clients/client/client.go index 36fd2e45d46..5102c57562e 100644 --- a/pkg/configs/storage/clients/client/client.go +++ b/pkg/configs/storage/clients/client/client.go @@ -10,7 +10,7 @@ import ( "strings" "time" - "github.com/cortexproject/cortex/pkg/ruler/rulegroup" + "github.com/cortexproject/cortex/pkg/ruler/group" "github.com/cortexproject/cortex/pkg/configs" legacy_configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" @@ -115,7 +115,7 @@ func (c ConfigsResponse) GetLatestConfigID() legacy_configs.ID { return latest } -func (c *configsClient) PollAlerts(ctx context.Context) (map[string]configs.AlertConfig, error) { +func (c *configsClient) PollAlertConfigs(ctx context.Context) (map[string]configs.AlertConfig, error) { resp, err := c.GetAlerts(ctx, c.lastPoll) if err != nil { return nil, err @@ -152,7 +152,7 @@ func (c *configsClient) PollRules(ctx context.Context) (map[string][]configs.Rul } for groupSlug, r := range rMap { name, file := decomposeGroupSlug(groupSlug) - userRules = append(userRules, rulegroup.NewRuleGroup(name, file, user, r)) + userRules = append(userRules, group.NewRuleGroup(name, file, user, r)) } newRules[user] = userRules } diff --git a/pkg/configs/storage/clients/gcp/gcs.go b/pkg/configs/storage/clients/gcp/gcs.go index 47d20a20d96..bfa0f973fc3 100644 --- a/pkg/configs/storage/clients/gcp/gcs.go +++ b/pkg/configs/storage/clients/gcp/gcs.go @@ -9,11 +9,11 @@ import ( "strings" "time" - "github.com/cortexproject/cortex/pkg/ruler/rulegroup" - - "cloud.google.com/go/storage" "github.com/cortexproject/cortex/pkg/configs" + "github.com/cortexproject/cortex/pkg/ruler/group" "github.com/cortexproject/cortex/pkg/util" + + "cloud.google.com/go/storage" "github.com/go-kit/kit/log/level" "github.com/golang/protobuf/proto" "github.com/prometheus/prometheus/pkg/rulefmt" @@ -86,7 +86,7 @@ func (g *gcsConfigClient) getAlertConfig(ctx context.Context, obj string) (confi return config, nil } -func (g *gcsConfigClient) PollAlerts(ctx context.Context) (map[string]configs.AlertConfig, error) { +func (g *gcsConfigClient) PollAlertConfigs(ctx context.Context) (map[string]configs.AlertConfig, error) { objs := g.bucket.Objects(ctx, &storage.Query{ Prefix: alertPrefix, }) @@ -240,7 +240,7 @@ func (g *gcsConfigClient) getAllRuleGroups(ctx context.Context, userID string) ( return []configs.RuleGroup{}, err } - rg, err := rulegroup.GenerateRuleGroup(userID, rgProto) + rg, err := group.GenerateRuleGroup(userID, rgProto) if err != nil { return []configs.RuleGroup{}, err } @@ -316,14 +316,14 @@ func (g *gcsConfigClient) getRuleNamespace(ctx context.Context, userID string, n return configs.RuleNamespace{}, err } - ns.Groups = append(ns.Groups, rulegroup.FromProto(rg)) + ns.Groups = append(ns.Groups, group.FromProto(rg)) } return ns, nil } -func (g *gcsConfigClient) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (rulefmt.RuleGroup, error) { - handle := generateRuleHandle(userID, namespace, group) +func (g *gcsConfigClient) GetRuleGroup(ctx context.Context, userID string, namespace string, grp string) (rulefmt.RuleGroup, error) { + handle := generateRuleHandle(userID, namespace, grp) rg, err := g.getRuleGroup(ctx, handle) if err != nil { return rulefmt.RuleGroup{}, err @@ -332,10 +332,10 @@ func (g *gcsConfigClient) GetRuleGroup(ctx context.Context, userID string, names if rg == nil { return rulefmt.RuleGroup{}, configs.ErrGroupNotFound } - return rulegroup.FromProto(rg), nil + return group.FromProto(rg), nil } -func (g *gcsConfigClient) getRuleGroup(ctx context.Context, handle string) (*rulegroup.RuleGroup, error) { +func (g *gcsConfigClient) getRuleGroup(ctx context.Context, handle string) (*group.RuleGroup, error) { reader, err := g.bucket.Object(handle).NewReader(ctx) if err == storage.ErrObjectNotExist { level.Debug(util.Logger).Log("msg", "rule group does not exist", "name", handle) @@ -351,7 +351,7 @@ func (g *gcsConfigClient) getRuleGroup(ctx context.Context, handle string) (*rul return nil, err } - rg := &rulegroup.RuleGroup{} + rg := &group.RuleGroup{} err = proto.Unmarshal(buf, rg) if err != nil { @@ -361,14 +361,14 @@ func (g *gcsConfigClient) getRuleGroup(ctx context.Context, handle string) (*rul return rg, nil } -func (g *gcsConfigClient) SetRuleGroup(ctx context.Context, userID string, namespace string, group rulefmt.RuleGroup) error { - rg := rulegroup.ToProto(namespace, group) +func (g *gcsConfigClient) SetRuleGroup(ctx context.Context, userID string, namespace string, grp rulefmt.RuleGroup) error { + rg := group.ToProto(namespace, grp) rgBytes, err := proto.Marshal(&rg) if err != nil { return err } - handle := generateRuleHandle(userID, namespace, group.Name) + handle := generateRuleHandle(userID, namespace, grp.Name) objHandle := g.bucket.Object(handle) writer := objHandle.NewWriter(ctx) diff --git a/pkg/configs/storage/instrumented.go b/pkg/configs/storage/instrumented.go index 2cc212af237..9a29a20e8c2 100644 --- a/pkg/configs/storage/instrumented.go +++ b/pkg/configs/storage/instrumented.go @@ -24,11 +24,11 @@ type instrumented struct { next configs.ConfigStore } -func (i instrumented) PollAlerts(ctx context.Context) (map[string]configs.AlertConfig, error) { +func (i instrumented) PollAlertConfigs(ctx context.Context) (map[string]configs.AlertConfig, error) { var cfgs map[string]configs.AlertConfig - err := instrument.CollectedRequest(context.Background(), "Configs.PollAlerts", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { + err := instrument.CollectedRequest(context.Background(), "Configs.PollAlertConfigs", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { var err error - cfgs, err = i.next.PollAlerts(ctx) + cfgs, err = i.next.PollAlertConfigs(ctx) return err }) return cfgs, err diff --git a/pkg/ruler/rulegroup/compat.go b/pkg/ruler/group/compat.go similarity index 99% rename from pkg/ruler/rulegroup/compat.go rename to pkg/ruler/group/compat.go index 3298181a7d0..f26700b921b 100644 --- a/pkg/ruler/rulegroup/compat.go +++ b/pkg/ruler/group/compat.go @@ -1,4 +1,4 @@ -package rulegroup +package group import ( time "time" diff --git a/pkg/ruler/rulegroup/rulegroup.go b/pkg/ruler/group/rulegroup.go similarity index 98% rename from pkg/ruler/rulegroup/rulegroup.go rename to pkg/ruler/group/rulegroup.go index c5bcce214c4..94e4269dd8d 100644 --- a/pkg/ruler/rulegroup/rulegroup.go +++ b/pkg/ruler/group/rulegroup.go @@ -1,4 +1,4 @@ -package rulegroup +package group import ( "context" diff --git a/pkg/ruler/rulegroup/rulegroup.pb.go b/pkg/ruler/group/rulegroup.pb.go similarity index 90% rename from pkg/ruler/rulegroup/rulegroup.pb.go rename to pkg/ruler/group/rulegroup.pb.go index 3d23ac5871f..49317a79d0f 100644 --- a/pkg/ruler/rulegroup/rulegroup.pb.go +++ b/pkg/ruler/group/rulegroup.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: rulegroup.proto -package rulegroup +package group import ( fmt "fmt" @@ -175,42 +175,42 @@ func (m *Rule) GetFor() *time.Duration { } func init() { - proto.RegisterType((*RuleGroup)(nil), "rulegroup.RuleGroup") - proto.RegisterType((*Rule)(nil), "rulegroup.Rule") + proto.RegisterType((*RuleGroup)(nil), "group.RuleGroup") + proto.RegisterType((*Rule)(nil), "group.Rule") } func init() { proto.RegisterFile("rulegroup.proto", fileDescriptor_3324c775b6f77a4a) } var fileDescriptor_3324c775b6f77a4a = []byte{ - // 439 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x93, 0xb1, 0x8e, 0xd4, 0x30, - 0x10, 0x86, 0xe3, 0xdb, 0xec, 0x72, 0xf1, 0x16, 0x27, 0x2c, 0x84, 0xcc, 0x09, 0x79, 0xa3, 0x93, - 0x90, 0xd2, 0x90, 0x88, 0xa3, 0xa4, 0x00, 0x56, 0x48, 0x50, 0x50, 0xa0, 0x94, 0x74, 0x4e, 0x32, - 0x67, 0x02, 0xbe, 0x38, 0x72, 0x1c, 0x44, 0x83, 0xc4, 0x23, 0x50, 0xf2, 0x08, 0xbc, 0x03, 0x2f, - 0x70, 0xe5, 0x96, 0x27, 0x8a, 0x83, 0xcd, 0x36, 0x74, 0xdc, 0x23, 0x20, 0x3b, 0xc9, 0xe5, 0x4a, - 0x84, 0x44, 0x95, 0xf9, 0x67, 0xc6, 0x33, 0x9f, 0xff, 0xc8, 0xf8, 0x40, 0xb7, 0x12, 0x84, 0x56, - 0x6d, 0x1d, 0xd7, 0x5a, 0x19, 0x45, 0x82, 0xab, 0xc4, 0xe1, 0x7d, 0x51, 0x9a, 0x37, 0x6d, 0x16, - 0xe7, 0xea, 0x34, 0x11, 0x4a, 0xa8, 0xc4, 0x75, 0x64, 0xed, 0x89, 0x53, 0x4e, 0xb8, 0xa8, 0x3f, - 0x79, 0xc8, 0x84, 0x52, 0x42, 0xc2, 0xd4, 0x55, 0xb4, 0x9a, 0x9b, 0x52, 0x55, 0x43, 0xfd, 0xc9, - 0xb5, 0x71, 0xb9, 0xd2, 0x06, 0x3e, 0xd4, 0x5a, 0xbd, 0x85, 0xdc, 0x0c, 0x2a, 0xa9, 0xdf, 0x89, - 0xa4, 0xac, 0x04, 0x34, 0x06, 0x74, 0x92, 0xcb, 0x12, 0xaa, 0xb1, 0xd4, 0x4f, 0x38, 0xfa, 0x86, - 0x70, 0x90, 0xb6, 0x12, 0x9e, 0x5b, 0x3c, 0x42, 0xb0, 0x5f, 0xf1, 0x53, 0xa0, 0x28, 0x44, 0x51, - 0x90, 0xba, 0x98, 0xdc, 0xc5, 0x81, 0xfd, 0x36, 0x35, 0xcf, 0x81, 0xee, 0xb9, 0xc2, 0x94, 0x20, - 0x8f, 0xf0, 0x7e, 0x59, 0x19, 0xd0, 0xef, 0xb9, 0xa4, 0xb3, 0x10, 0x45, 0xcb, 0xe3, 0x3b, 0x71, - 0x0f, 0x1d, 0x8f, 0xd0, 0xf1, 0xb3, 0x01, 0x7a, 0xed, 0x7f, 0xf9, 0xb1, 0x42, 0xe9, 0xd5, 0x01, - 0x72, 0x0f, 0xcf, 0xad, 0x35, 0x0d, 0xf5, 0xc3, 0x59, 0xb4, 0x3c, 0x3e, 0x88, 0x27, 0xe7, 0x2c, - 0x53, 0xda, 0x57, 0x09, 0xc5, 0x37, 0x0a, 0x90, 0x60, 0xa0, 0xa0, 0xf3, 0x10, 0x45, 0xfb, 0xe9, - 0x28, 0x8f, 0x7e, 0xef, 0x61, 0xdf, 0x76, 0x5a, 0x70, 0x7b, 0xf7, 0x11, 0xdc, 0xc6, 0xe4, 0x36, - 0x5e, 0x68, 0xc8, 0x95, 0x2e, 0x06, 0xea, 0x41, 0x91, 0x5b, 0x78, 0xce, 0x25, 0x68, 0xe3, 0x78, - 0x83, 0xb4, 0x17, 0xe4, 0x01, 0x9e, 0x9d, 0x28, 0x4d, 0xfd, 0xbf, 0xbb, 0x83, 0xed, 0x25, 0x0d, - 0x5e, 0x48, 0x9e, 0x81, 0x6c, 0xe8, 0xdc, 0xf1, 0xdf, 0x8c, 0x07, 0x6b, 0x5f, 0xda, 0xec, 0x2b, - 0x5e, 0xea, 0xf5, 0x8b, 0xb3, 0x8b, 0x95, 0xf7, 0xfd, 0x62, 0xf5, 0x2f, 0x3f, 0xaa, 0x1f, 0xf3, - 0xb4, 0xe0, 0xb5, 0x01, 0x9d, 0x0e, 0xab, 0xc8, 0x47, 0xbc, 0xe4, 0x55, 0xa5, 0x8c, 0xa3, 0x69, - 0xe8, 0xe2, 0xff, 0x6f, 0xbe, 0xbe, 0x6f, 0xfd, 0x78, 0xb3, 0x65, 0xde, 0xf9, 0x96, 0x79, 0x97, - 0x5b, 0x86, 0x3e, 0x75, 0x0c, 0x7d, 0xed, 0x18, 0x3a, 0xeb, 0x18, 0xda, 0x74, 0x0c, 0xfd, 0xec, - 0x18, 0xfa, 0xd5, 0x31, 0xef, 0xb2, 0x63, 0xe8, 0xf3, 0x8e, 0x79, 0x9b, 0x1d, 0xf3, 0xce, 0x77, - 0xcc, 0x7b, 0x3d, 0xbd, 0x80, 0x6c, 0xe1, 0x2c, 0x7d, 0xf8, 0x27, 0x00, 0x00, 0xff, 0xff, 0x04, - 0xbb, 0x4d, 0x8a, 0x26, 0x03, 0x00, 0x00, + // 436 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x92, 0xb1, 0x6e, 0xd4, 0x30, + 0x18, 0xc7, 0xe3, 0x5e, 0x72, 0xf4, 0x7c, 0x03, 0xc2, 0x42, 0xc8, 0x54, 0xc8, 0x17, 0x3a, 0x65, + 0x21, 0x11, 0x65, 0xec, 0x02, 0x27, 0x24, 0x18, 0x18, 0x50, 0x46, 0x36, 0x27, 0xf9, 0x6a, 0x02, + 0x6e, 0x1c, 0x39, 0x0e, 0x62, 0x41, 0xe2, 0x11, 0x18, 0x79, 0x04, 0xde, 0x80, 0x57, 0xe8, 0x78, + 0x63, 0xc5, 0x50, 0xb8, 0xdc, 0xc2, 0x46, 0x1f, 0x01, 0xd9, 0x4e, 0x68, 0x47, 0x84, 0xc4, 0x94, + 0xef, 0xef, 0xff, 0xe7, 0xef, 0xfb, 0xe5, 0x9f, 0xe0, 0x9b, 0xba, 0x97, 0x20, 0xb4, 0xea, 0xdb, + 0xb4, 0xd5, 0xca, 0x28, 0x12, 0x39, 0x71, 0xf0, 0x40, 0xd4, 0xe6, 0x75, 0x5f, 0xa4, 0xa5, 0x3a, + 0xcd, 0x84, 0x12, 0x2a, 0x73, 0x6e, 0xd1, 0x9f, 0x38, 0xe5, 0x84, 0xab, 0xfc, 0xad, 0x03, 0x26, + 0x94, 0x12, 0x12, 0xae, 0xba, 0xaa, 0x5e, 0x73, 0x53, 0xab, 0x66, 0xf4, 0x1f, 0x5f, 0x1b, 0x57, + 0x2a, 0x6d, 0xe0, 0x7d, 0xab, 0xd5, 0x1b, 0x28, 0xcd, 0xa8, 0xb2, 0xf6, 0xad, 0xc8, 0xea, 0x46, + 0x40, 0x67, 0x40, 0x67, 0xa5, 0xac, 0xa1, 0x99, 0x2c, 0x3f, 0xe1, 0xf0, 0x2b, 0xc2, 0x8b, 0xbc, + 0x97, 0xf0, 0xcc, 0xe2, 0x11, 0x82, 0xc3, 0x86, 0x9f, 0x02, 0x45, 0x31, 0x4a, 0x16, 0xb9, 0xab, + 0xc9, 0x3d, 0xbc, 0xb0, 0xcf, 0xae, 0xe5, 0x25, 0xd0, 0x3d, 0x67, 0x5c, 0x1d, 0x90, 0x63, 0xbc, + 0x5f, 0x37, 0x06, 0xf4, 0x3b, 0x2e, 0xe9, 0x2c, 0x46, 0xc9, 0xf2, 0xe8, 0x6e, 0xea, 0xa1, 0xd3, + 0x09, 0x3a, 0x7d, 0x3a, 0x42, 0xaf, 0xc3, 0xcf, 0xdf, 0x57, 0x28, 0xff, 0x73, 0x81, 0xdc, 0xc7, + 0x91, 0xcd, 0xa9, 0xa3, 0x61, 0x3c, 0x4b, 0x96, 0x47, 0xcb, 0xd4, 0x27, 0x66, 0x79, 0x72, 0xef, + 0x10, 0x8a, 0x6f, 0x54, 0x20, 0xc1, 0x40, 0x45, 0xa3, 0x18, 0x25, 0xfb, 0xf9, 0x24, 0x0f, 0x7f, + 0xed, 0xe1, 0xd0, 0x76, 0x5a, 0x68, 0xfb, 0xde, 0x13, 0xb4, 0xad, 0xc9, 0x1d, 0x3c, 0xd7, 0x50, + 0x2a, 0x5d, 0x8d, 0xc4, 0xa3, 0x22, 0xb7, 0x71, 0xc4, 0x25, 0x68, 0xe3, 0x58, 0x17, 0xb9, 0x17, + 0xe4, 0x21, 0x9e, 0x9d, 0x28, 0x4d, 0xc3, 0xbf, 0xe3, 0xb7, 0xbd, 0xa4, 0xc3, 0x73, 0xc9, 0x0b, + 0x90, 0x1d, 0x8d, 0x1c, 0xfb, 0xad, 0x74, 0x8c, 0xf5, 0x85, 0x3d, 0x7d, 0xc9, 0x6b, 0xbd, 0x7e, + 0x7e, 0x76, 0xb1, 0x0a, 0xbe, 0x5d, 0xac, 0xfe, 0xe5, 0x23, 0xf9, 0x31, 0x4f, 0x2a, 0xde, 0x1a, + 0xd0, 0xf9, 0xb8, 0x8a, 0x7c, 0xc0, 0x4b, 0xde, 0x34, 0xca, 0x38, 0x9a, 0x8e, 0xce, 0xff, 0xff, + 0xe6, 0xeb, 0xfb, 0xd6, 0xc7, 0x9b, 0x2d, 0x0b, 0xce, 0xb7, 0x2c, 0xb8, 0xdc, 0x32, 0xf4, 0x71, + 0x60, 0xe8, 0xcb, 0xc0, 0xd0, 0xd9, 0xc0, 0xd0, 0x66, 0x60, 0xe8, 0xc7, 0xc0, 0xd0, 0xcf, 0x81, + 0x05, 0x97, 0x03, 0x43, 0x9f, 0x76, 0x2c, 0xd8, 0xec, 0x58, 0x70, 0xbe, 0x63, 0xc1, 0x2b, 0xff, + 0xe7, 0x17, 0x73, 0x17, 0xe7, 0xa3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x7f, 0xbc, 0xc2, 0x52, + 0x1a, 0x03, 0x00, 0x00, } func (this *RuleGroup) Equal(that interface{}) bool { @@ -320,7 +320,7 @@ func (this *RuleGroup) GoString() string { return "nil" } s := make([]string, 0, 9) - s = append(s, "&rulegroup.RuleGroup{") + s = append(s, "&group.RuleGroup{") s = append(s, "Name: "+fmt.Sprintf("%#v", this.Name)+",\n") s = append(s, "Namespace: "+fmt.Sprintf("%#v", this.Namespace)+",\n") s = append(s, "Interval: "+fmt.Sprintf("%#v", this.Interval)+",\n") @@ -336,7 +336,7 @@ func (this *Rule) GoString() string { return "nil" } s := make([]string, 0, 10) - s = append(s, "&rulegroup.Rule{") + s = append(s, "&group.Rule{") s = append(s, "Expr: "+fmt.Sprintf("%#v", this.Expr)+",\n") s = append(s, "Record: "+fmt.Sprintf("%#v", this.Record)+",\n") s = append(s, "Alert: "+fmt.Sprintf("%#v", this.Alert)+",\n") diff --git a/pkg/ruler/rulegroup/rulegroup.proto b/pkg/ruler/group/rulegroup.proto similarity index 94% rename from pkg/ruler/rulegroup/rulegroup.proto rename to pkg/ruler/group/rulegroup.proto index 9d7973b100b..ea6a6eede7c 100644 --- a/pkg/ruler/rulegroup/rulegroup.proto +++ b/pkg/ruler/group/rulegroup.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package rulegroup; +package group; -option go_package = "rulegroup"; +option go_package = "group"; import "github.com/gogo/protobuf/gogoproto/gogo.proto"; import "google/protobuf/duration.proto"; diff --git a/pkg/ruler/scheduler_test.go b/pkg/ruler/scheduler_test.go index 82c0cec8755..b4430e4888a 100644 --- a/pkg/ruler/scheduler_test.go +++ b/pkg/ruler/scheduler_test.go @@ -7,7 +7,7 @@ import ( "time" "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/ruler/rulegroup" + "github.com/cortexproject/cortex/pkg/ruler/group" "github.com/prometheus/prometheus/rules" "github.com/stretchr/testify/assert" ) @@ -78,11 +78,11 @@ func TestSchedulerRulesOverlap(t *testing.T) { next := time.Now() ruleSetsOne := []configs.RuleGroup{ - rulegroup.NewRuleGroup(groupOne, "default", userID, []rules.Rule{nil}), + group.NewRuleGroup(groupOne, "default", userID, []rules.Rule{nil}), } ruleSetsTwo := []configs.RuleGroup{ - rulegroup.NewRuleGroup(groupTwo, "default", userID, []rules.Rule{nil}), + group.NewRuleGroup(groupTwo, "default", userID, []rules.Rule{nil}), } userChanOne := make(chan struct{}) userChanTwo := make(chan struct{}) From 4d6b5e6ad2d2da81f3c760a937cae26d21e88a08 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Wed, 10 Jul 2019 16:47:29 -0400 Subject: [PATCH 33/60] rename things Signed-off-by: Jacob Lisi --- pkg/configs/{configs.go => storage.go} | 1 + .../storage/clients/aws/s3_config_client.go | 114 ++++++++++++++++++ .../clients/gcp/{gcs.go => config_client.go} | 49 +++----- pkg/ruler/api.go | 4 +- 4 files changed, 138 insertions(+), 30 deletions(-) rename pkg/configs/{configs.go => storage.go} (96%) create mode 100644 pkg/configs/storage/clients/aws/s3_config_client.go rename pkg/configs/storage/clients/gcp/{gcs.go => config_client.go} (91%) diff --git a/pkg/configs/configs.go b/pkg/configs/storage.go similarity index 96% rename from pkg/configs/configs.go rename to pkg/configs/storage.go index 5ea974ee2f4..f62737a03b9 100644 --- a/pkg/configs/configs.go +++ b/pkg/configs/storage.go @@ -16,6 +16,7 @@ var ( ) // ConfigStore unifies the AlertStore and ConfigStore interface +// TODO: Remove polling from the configstorage backend. A notification queue system would work better for this use case. type ConfigStore interface { AlertStore RuleStore diff --git a/pkg/configs/storage/clients/aws/s3_config_client.go b/pkg/configs/storage/clients/aws/s3_config_client.go new file mode 100644 index 00000000000..77258e4c620 --- /dev/null +++ b/pkg/configs/storage/clients/aws/s3_config_client.go @@ -0,0 +1,114 @@ +package aws + +import ( + "context" + "flag" + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3iface" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pkg/rulefmt" + + "github.com/cortexproject/cortex/pkg/configs" + "github.com/cortexproject/cortex/pkg/util/flagext" + awscommon "github.com/weaveworks/common/aws" + "github.com/weaveworks/common/instrument" +) + +var ( + s3RequestDuration = instrument.NewHistogramCollector(prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "s3_request_duration_seconds", + Help: "Time spent doing S3 requests.", + Buckets: []float64{.025, .05, .1, .25, .5, 1, 2}, + }, []string{"operation", "status_code"})) +) + +func init() { + s3RequestDuration.Register() +} + +// S3ClientConfig is config for the GCS Chunk Client. +type S3ClientConfig struct { + BucketName string `yaml:"bucket_name"` + S3 flagext.URLValue `yaml:"url"` + S3ForcePathStyle bool +} + +// RegisterFlagsWithPrefix registers flags. +func (cfg *S3ClientConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.StringVar(&cfg.BucketName, prefix+"s3.bucketname", "", "Name of S3 bucket to put chunks in.") +} + +type s3ConfigClient struct { + bucketName string + S3 s3iface.S3API + + alertPolled time.Time + rulePolled time.Time +} + +// NewS3ConfigClient makes a new S3-backed ObjectClient. +func NewS3ConfigClient(cfg S3ClientConfig) (configs.ConfigStore, error) { + if cfg.S3.URL == nil { + return nil, fmt.Errorf("no URL specified for S3") + } + s3Config, err := awscommon.ConfigFromURL(cfg.S3.URL) + if err != nil { + return nil, err + } + + s3Config = s3Config.WithS3ForcePathStyle(cfg.S3ForcePathStyle) // support for Path Style S3 url if has the flag + + s3Config = s3Config.WithMaxRetries(0) // We do our own retries, so we can monitor them + s3Client := s3.New(session.New(s3Config)) + bucketName := strings.TrimPrefix(cfg.S3.URL.Path, "/") + client := &s3ConfigClient{ + S3: s3Client, + bucketName: bucketName, + + alertPolled: time.Unix(0, 0), + rulePolled: time.Unix(0, 0), + } + return client, nil +} + +func (a *s3ConfigClient) PollAlertConfigs(ctx context.Context) (map[string]configs.AlertConfig, error) { + a.S3.ListObjectsV2() +} + +func (a *s3ConfigClient) GetAlertConfig(ctx context.Context, userID string) (configs.AlertConfig, error) { + panic("not implemented") +} + +func (a *s3ConfigClient) SetAlertConfig(ctx context.Context, userID string, config configs.AlertConfig) error { + panic("not implemented") +} + +func (a *s3ConfigClient) DeleteAlertConfig(ctx context.Context, userID string) error { + panic("not implemented") +} + +func (a *s3ConfigClient) PollRules(ctx context.Context) (map[string][]configs.RuleGroup, error) { + panic("not implemented") +} + +func (a *s3ConfigClient) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) (map[string]configs.RuleNamespace, error) { + panic("not implemented") +} + +func (a *s3ConfigClient) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (rulefmt.RuleGroup, error) { + panic("not implemented") +} + +func (a *s3ConfigClient) SetRuleGroup(ctx context.Context, userID string, namespace string, group rulefmt.RuleGroup) error { + panic("not implemented") +} + +func (a *s3ConfigClient) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { + panic("not implemented") +} diff --git a/pkg/configs/storage/clients/gcp/gcs.go b/pkg/configs/storage/clients/gcp/config_client.go similarity index 91% rename from pkg/configs/storage/clients/gcp/gcs.go rename to pkg/configs/storage/clients/gcp/config_client.go index bfa0f973fc3..9bb4e833cf0 100644 --- a/pkg/configs/storage/clients/gcp/gcs.go +++ b/pkg/configs/storage/clients/gcp/config_client.go @@ -39,11 +39,14 @@ func (cfg *GCSConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.StringVar(&cfg.BucketName, prefix+"gcs.bucketname", "", "Name of GCS bucket to put chunks in.") } +// gcsConfigClient acts as a config backend. It is not safe to use concurrently when polling for rules. +// This is not an issue with the current scheduler architecture, but must be noted. type gcsConfigClient struct { client *storage.Client bucket *storage.BucketHandle - lastPolled time.Time + alertPolled time.Time + rulePolled time.Time } // NewGCSConfigClient makes a new chunk.ObjectClient that writes chunks to GCS. @@ -57,7 +60,8 @@ func NewGCSConfigClient(ctx context.Context, cfg GCSConfig) (configs.ConfigStore client: client, bucket: bucket, - lastPolled: time.Unix(0, 0), + alertPolled: time.Unix(0, 0), + rulePolled: time.Unix(0, 0), }, nil } @@ -86,6 +90,7 @@ func (g *gcsConfigClient) getAlertConfig(ctx context.Context, obj string) (confi return config, nil } +// PollAlertConfigs returns any recently updated alert configurations func (g *gcsConfigClient) PollAlertConfigs(ctx context.Context) (map[string]configs.AlertConfig, error) { objs := g.bucket.Objects(ctx, &storage.Query{ Prefix: alertPrefix, @@ -103,7 +108,7 @@ func (g *gcsConfigClient) PollAlertConfigs(ctx context.Context) (map[string]conf return nil, err } - if objAttrs.Updated.After(g.lastPolled) { + if objAttrs.Updated.After(g.alertPolled) { level.Debug(util.Logger).Log("msg", "adding updated gcs config", "config", objAttrs.Name) rls, err := g.getAlertConfig(ctx, objAttrs.Name) if err != nil { @@ -113,14 +118,16 @@ func (g *gcsConfigClient) PollAlertConfigs(ctx context.Context) (map[string]conf } } - g.lastPolled = time.Now() + g.alertPolled = time.Now() return alertMap, nil } +// GetAlertConfig returns a specified users alertmanager configuration func (g *gcsConfigClient) GetAlertConfig(ctx context.Context, userID string) (configs.AlertConfig, error) { return g.getAlertConfig(ctx, alertPrefix+userID) } +// SetAlertConfig sets a specified users alertmanager configuration func (g *gcsConfigClient) SetAlertConfig(ctx context.Context, userID string, cfg configs.AlertConfig) error { cfgBytes, err := json.Marshal(cfg) if err != nil { @@ -141,10 +148,16 @@ func (g *gcsConfigClient) SetAlertConfig(ctx context.Context, userID string, cfg return nil } +// DeleteAlertConfig deletes a specified users alertmanager configuration func (g *gcsConfigClient) DeleteAlertConfig(ctx context.Context, userID string) error { + err := g.bucket.Object(alertPrefix + userID).Delete(ctx) + if err != nil { + return err + } return nil } +// PollRules returns a users recently updated rule set func (g *gcsConfigClient) PollRules(ctx context.Context) (map[string][]configs.RuleGroup, error) { objs := g.bucket.Objects(ctx, &storage.Query{ Prefix: rulePrefix, @@ -189,7 +202,7 @@ func (g *gcsConfigClient) PollRules(ctx context.Context) (map[string][]configs.R ruleMap[user] = rgs } - g.lastPolled = time.Now() + g.rulePolled = time.Now() return ruleMap, nil } @@ -209,7 +222,7 @@ func (g *gcsConfigClient) checkUser(ctx context.Context, userID string) (bool, e return false, err } - if rg.Updated.After(g.lastPolled) { + if rg.Updated.After(g.rulePolled) { level.Debug(util.Logger).Log("msg", "updated rulegroups found", "user", userID) return true, nil } @@ -385,33 +398,11 @@ func (g *gcsConfigClient) SetRuleGroup(ctx context.Context, userID string, names func (g *gcsConfigClient) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { handle := generateRuleHandle(userID, namespace, group) - rg, err := g.getRuleGroup(ctx, handle) - if err != nil { - return nil - } - - if rg == nil { - return configs.ErrGroupNotFound - } - - rg.Deleted = true - - rgBytes, err := proto.Marshal(rg) + err := g.bucket.Object(handle).Delete(ctx) if err != nil { return err } - objHandle := g.bucket.Object(handle) - - writer := objHandle.NewWriter(ctx) - if _, err := writer.Write(rgBytes); err != nil { - return err - } - - if err := writer.Close(); err != nil { - return err - } - return nil } diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 2fada572122..a3c457c1484 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -144,8 +144,10 @@ func (a *API) getRuleGroup(w http.ResponseWriter, r *http.Request) { } rg, err := a.store.GetRuleGroup(r.Context(), userID, ns, gn) - if err != nil { + if err == configs.ErrGroupNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } http.Error(w, err.Error(), http.StatusBadRequest) return } From f9763e57dcbf424418045585eb84c4c6f8b4a72d Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Thu, 11 Jul 2019 14:05:38 -0400 Subject: [PATCH 34/60] fix s3 client to panic until implemented Signed-off-by: Jacob Lisi --- pkg/configs/storage/clients/aws/s3_config_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/configs/storage/clients/aws/s3_config_client.go b/pkg/configs/storage/clients/aws/s3_config_client.go index 77258e4c620..fbc835e98b6 100644 --- a/pkg/configs/storage/clients/aws/s3_config_client.go +++ b/pkg/configs/storage/clients/aws/s3_config_client.go @@ -78,7 +78,7 @@ func NewS3ConfigClient(cfg S3ClientConfig) (configs.ConfigStore, error) { } func (a *s3ConfigClient) PollAlertConfigs(ctx context.Context) (map[string]configs.AlertConfig, error) { - a.S3.ListObjectsV2() + panic("not implemented") } func (a *s3ConfigClient) GetAlertConfig(ctx context.Context, userID string) (configs.AlertConfig, error) { From f374d6309c3b80653b627514e4b5ff15bd7adcf5 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Thu, 11 Jul 2019 19:04:57 -0400 Subject: [PATCH 35/60] refactor configdb and config storage clients Signed-off-by: Jacob Lisi --- pkg/alertmanager/alert_store.go | 20 ++ pkg/alertmanager/api.go | 7 +- pkg/alertmanager/multitenant.go | 17 +- pkg/configs/api/api.go | 2 +- pkg/configs/api/api_test.go | 2 +- pkg/configs/api/helpers_test.go | 2 +- pkg/configs/client/client.go | 165 ++++++++++ pkg/configs/client/config.go | 70 ++++ .../clients => }/client/configs_test.go | 2 +- pkg/configs/configs.go | 298 ++++++++++++++++++ pkg/configs/configs_test.go | 105 ++++++ pkg/configs/db/db.go | 2 +- pkg/configs/db/memory/memory.go | 2 +- pkg/configs/db/postgres/postgres.go | 2 +- pkg/configs/db/timed.go | 2 +- pkg/configs/db/traced.go | 2 +- pkg/configs/storage/clients/client/config.go | 30 -- pkg/configs/storage/clients/client_test.go | 48 --- pkg/configs/storage/factory.go | 46 --- pkg/configs/storage/instrumented.go | 108 ------- pkg/configs/storage/testutils/testutils.go | 12 - pkg/cortex/cortex.go | 6 +- pkg/cortex/modules.go | 12 +- pkg/ruler/api.go | 11 +- pkg/ruler/group.go | 12 +- pkg/ruler/group/compat.go | 8 +- pkg/ruler/group/rulegroup.go | 13 +- .../storage.go => ruler/rule_store.go} | 26 +- pkg/ruler/ruler.go | 5 +- pkg/ruler/ruler_test.go | 32 +- pkg/ruler/scheduler.go | 13 +- pkg/ruler/scheduler_test.go | 5 +- .../storage/clients/aws/s3_config_client.go | 2 +- pkg/storage/clients/client_test.go | 59 ++++ .../clients/configdb}/client.go | 72 ++--- pkg/storage/clients/configdb/config.go | 28 ++ pkg/storage/clients/configdb/configs_test.go | 54 ++++ .../storage/clients/gcp/config_client.go | 109 ++++--- .../storage/clients/gcp/fixtures.go | 17 +- pkg/storage/factory.go | 75 +++++ pkg/storage/testutils/testutils.go | 13 + 41 files changed, 1068 insertions(+), 448 deletions(-) create mode 100644 pkg/alertmanager/alert_store.go create mode 100644 pkg/configs/client/client.go create mode 100644 pkg/configs/client/config.go rename pkg/configs/{storage/clients => }/client/configs_test.go (94%) create mode 100644 pkg/configs/configs.go create mode 100644 pkg/configs/configs_test.go delete mode 100644 pkg/configs/storage/clients/client/config.go delete mode 100644 pkg/configs/storage/clients/client_test.go delete mode 100644 pkg/configs/storage/factory.go delete mode 100644 pkg/configs/storage/instrumented.go delete mode 100644 pkg/configs/storage/testutils/testutils.go rename pkg/{configs/storage.go => ruler/rule_store.go} (74%) rename pkg/{configs => }/storage/clients/aws/s3_config_client.go (99%) create mode 100644 pkg/storage/clients/client_test.go rename pkg/{configs/storage/clients/client => storage/clients/configdb}/client.go (65%) create mode 100644 pkg/storage/clients/configdb/config.go create mode 100644 pkg/storage/clients/configdb/configs_test.go rename pkg/{configs => }/storage/clients/gcp/config_client.go (68%) rename pkg/{configs => }/storage/clients/gcp/fixtures.go (65%) create mode 100644 pkg/storage/factory.go create mode 100644 pkg/storage/testutils/testutils.go diff --git a/pkg/alertmanager/alert_store.go b/pkg/alertmanager/alert_store.go new file mode 100644 index 00000000000..0a674b04b26 --- /dev/null +++ b/pkg/alertmanager/alert_store.go @@ -0,0 +1,20 @@ +package alertmanager + +import ( + "context" +) + +// AlertConfig is used to configure user alert managers +type AlertConfig struct { + TemplateFiles map[string]string `json:"template_files"` + AlertmanagerConfig string `json:"alertmanager_config"` +} + +// AlertStore stores config information and template files to configure alertmanager tenants +type AlertStore interface { + PollAlertConfigs(ctx context.Context) (map[string]AlertConfig, error) + + GetAlertConfig(ctx context.Context, id string) (AlertConfig, error) + SetAlertConfig(ctx context.Context, id string, cfg AlertConfig) error + DeleteAlertConfig(ctx context.Context, id string) error +} diff --git a/pkg/alertmanager/api.go b/pkg/alertmanager/api.go index 02153dffc6d..1489d8d7eee 100644 --- a/pkg/alertmanager/api.go +++ b/pkg/alertmanager/api.go @@ -4,7 +4,6 @@ import ( "io/ioutil" "net/http" - "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "github.com/gorilla/mux" @@ -14,11 +13,11 @@ import ( // API is used to provided endpoints to directly interact with the ruler type API struct { - store configs.AlertStore + store AlertStore } // NewAPI returns a ruler API -func NewAPI(store configs.AlertStore) *API { +func NewAPI(store AlertStore) *API { return &API{store} } @@ -104,7 +103,7 @@ func (a *API) setConfig(w http.ResponseWriter, r *http.Request) { return } - cfg := configs.AlertConfig{} + cfg := AlertConfig{} err = yaml.Unmarshal(payload, &cfg) if err != nil { level.Error(logger).Log("err", err.Error()) diff --git a/pkg/alertmanager/multitenant.go b/pkg/alertmanager/multitenant.go index ec0f20304e4..7a63efba0b5 100644 --- a/pkg/alertmanager/multitenant.go +++ b/pkg/alertmanager/multitenant.go @@ -22,7 +22,6 @@ import ( "github.com/weaveworks/common/user" "github.com/weaveworks/mesh" - "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" ) @@ -213,7 +212,7 @@ func (cfg *MultitenantAlertmanagerConfig) RegisterFlags(f *flag.FlagSet) { type MultitenantAlertmanager struct { cfg *MultitenantAlertmanagerConfig - store configs.AlertStore + store AlertStore // The fallback config is stored as a string and parsed every time it's needed // because we mutate the parsed results and don't want those changes to take @@ -221,7 +220,7 @@ type MultitenantAlertmanager struct { fallbackConfig string // All the organization configurations that we have. Only used for instrumentation. - cfgs map[string]configs.AlertConfig + cfgs map[string]AlertConfig alertmanagersMtx sync.Mutex alertmanagers map[string]*Alertmanager @@ -234,7 +233,7 @@ type MultitenantAlertmanager struct { } // NewMultitenantAlertmanager creates a new MultitenantAlertmanager. -func NewMultitenantAlertmanager(cfg *MultitenantAlertmanagerConfig, store configs.AlertStore) (*MultitenantAlertmanager, error) { +func NewMultitenantAlertmanager(cfg *MultitenantAlertmanagerConfig, store AlertStore) (*MultitenantAlertmanager, error) { err := os.MkdirAll(cfg.DataDir, 0777) if err != nil { return nil, fmt.Errorf("unable to create Alertmanager data directory %q: %s", cfg.DataDir, err) @@ -316,7 +315,7 @@ func (am *MultitenantAlertmanager) Stop() { // Load the full set of configurations from the server, retrying with backoff // until we can get them. -func (am *MultitenantAlertmanager) loadAllConfigs() map[string]configs.AlertConfig { +func (am *MultitenantAlertmanager) loadAllConfigs() map[string]AlertConfig { backoff := util.NewBackoff(context.Background(), backoffConfig) for { cfgs, err := am.poll() @@ -339,7 +338,7 @@ func (am *MultitenantAlertmanager) updateConfigs(now time.Time) error { } // poll the configuration server. Not re-entrant. -func (am *MultitenantAlertmanager) poll() (map[string]configs.AlertConfig, error) { +func (am *MultitenantAlertmanager) poll() (map[string]AlertConfig, error) { cfgs, err := am.store.PollAlertConfigs(context.Background()) if err != nil { level.Warn(util.Logger).Log("msg", "MultitenantAlertmanager: configs server poll failed", "err", err) @@ -348,7 +347,7 @@ func (am *MultitenantAlertmanager) poll() (map[string]configs.AlertConfig, error return cfgs, nil } -func (am *MultitenantAlertmanager) addNewConfigs(cfgs map[string]configs.AlertConfig) { +func (am *MultitenantAlertmanager) addNewConfigs(cfgs map[string]AlertConfig) { // TODO: instrument how many configs we have, both valid & invalid. level.Debug(util.Logger).Log("msg", "adding configurations", "num_configs", len(cfgs)) for userID, config := range cfgs { @@ -412,7 +411,7 @@ func (am *MultitenantAlertmanager) createTemplatesFile(userID, fn, content strin // setConfig applies the given configuration to the alertmanager for `userID`, // creating an alertmanager if it doesn't already exist. -func (am *MultitenantAlertmanager) setConfig(userID string, config configs.AlertConfig) error { +func (am *MultitenantAlertmanager) setConfig(userID string, config AlertConfig) error { _, hasExisting := am.alertmanagers[userID] var amConfig *amconfig.Config var err error @@ -475,7 +474,7 @@ func (am *MultitenantAlertmanager) setConfig(userID string, config configs.Alert } // alertmanagerConfigFromConfig returns the Alertmanager config from the Cortex configuration. -func alertmanagerConfigFromConfig(c configs.AlertConfig) (*amconfig.Config, error) { +func alertmanagerConfigFromConfig(c AlertConfig) (*amconfig.Config, error) { cfg, err := amconfig.Load(c.AlertmanagerConfig) if err != nil { return nil, fmt.Errorf("error parsing Alertmanager config: %s", err) diff --git a/pkg/configs/api/api.go b/pkg/configs/api/api.go index 55998f394c6..b72a5d6d47d 100644 --- a/pkg/configs/api/api.go +++ b/pkg/configs/api/api.go @@ -13,8 +13,8 @@ import ( "github.com/gorilla/mux" amconfig "github.com/prometheus/alertmanager/config" + "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/configs/db" - configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" "github.com/cortexproject/cortex/pkg/util" "github.com/weaveworks/common/user" ) diff --git a/pkg/configs/api/api_test.go b/pkg/configs/api/api_test.go index 1a3ba222b4f..45abef4d3b6 100644 --- a/pkg/configs/api/api_test.go +++ b/pkg/configs/api/api_test.go @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/configs/api" - configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" ) const ( diff --git a/pkg/configs/api/helpers_test.go b/pkg/configs/api/helpers_test.go index 96286d9ea7e..b22359c96c4 100644 --- a/pkg/configs/api/helpers_test.go +++ b/pkg/configs/api/helpers_test.go @@ -11,10 +11,10 @@ import ( "github.com/stretchr/testify/require" + "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/configs/api" "github.com/cortexproject/cortex/pkg/configs/db" "github.com/cortexproject/cortex/pkg/configs/db/dbtest" - configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" "github.com/weaveworks/common/user" ) diff --git a/pkg/configs/client/client.go b/pkg/configs/client/client.go new file mode 100644 index 00000000000..a5528b29ecc --- /dev/null +++ b/pkg/configs/client/client.go @@ -0,0 +1,165 @@ +package client + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/cortexproject/cortex/pkg/configs" + "github.com/cortexproject/cortex/pkg/configs/db" + "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log/level" +) + +// Client is what the ruler and altermanger needs from a config store to process rules. +type Client interface { + // GetRules returns all Cortex configurations from a configs API server + // that have been updated after the given configs.ID was last updated. + GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) + + // GetAlerts fetches all the alerts that have changes since since. + GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) +} + +// New creates a new ConfigClient. +func New(cfg Config) (Client, error) { + // All of this falderal is to allow for a smooth transition away from + // using the configs server and toward directly connecting to the database. + // See https://github.com/cortexproject/cortex/issues/619 + if cfg.ConfigsAPIURL.URL != nil { + return instrumented{ + next: configsClient{ + URL: cfg.ConfigsAPIURL.URL, + Timeout: cfg.ClientTimeout, + }, + }, nil + } + + db, err := db.New(cfg.DBConfig) + if err != nil { + return nil, err + } + return instrumented{ + next: dbStore{ + db: db, + }, + }, nil +} + +// configsClient allows retrieving recording and alerting rules from the configs server. +type configsClient struct { + URL *url.URL + Timeout time.Duration +} + +// GetRules implements ConfigClient. +func (c configsClient) GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { + suffix := "" + if since != 0 { + suffix = fmt.Sprintf("?since=%d", since) + } + endpoint := fmt.Sprintf("%s/private/api/prom/configs/rules%s", c.URL.String(), suffix) + response, err := doRequest(endpoint, c.Timeout, since) + if err != nil { + return nil, err + } + configs := map[string]configs.VersionedRulesConfig{} + for id, view := range response.Configs { + cfg := view.GetVersionedRulesConfig() + if cfg != nil { + configs[id] = *cfg + } + } + return configs, nil +} + +// GetAlerts implements ConfigClient. +func (c configsClient) GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) { + suffix := "" + if since != 0 { + suffix = fmt.Sprintf("?since=%d", since) + } + endpoint := fmt.Sprintf("%s/private/api/prom/configs/alertmanager%s", c.URL.String(), suffix) + return doRequest(endpoint, c.Timeout, since) +} + +func doRequest(endpoint string, timeout time.Duration, since configs.ID) (*ConfigsResponse, error) { + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return nil, err + } + + client := &http.Client{Timeout: timeout} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Invalid response from configs server: %v", resp.StatusCode) + } + + var config ConfigsResponse + if err := json.NewDecoder(resp.Body).Decode(&config); err != nil { + level.Error(util.Logger).Log("msg", "configs: couldn't decode JSON body", "err", err) + return nil, err + } + + config.since = since + return &config, nil +} + +type dbStore struct { + db db.DB +} + +// GetRules implements ConfigClient. +func (d dbStore) GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { + if since == 0 { + return d.db.GetAllRulesConfigs(ctx) + } + return d.db.GetRulesConfigs(ctx, since) +} + +// GetAlerts implements ConfigClient. +func (d dbStore) GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) { + var resp map[string]configs.View + var err error + if since == 0 { + resp, err = d.db.GetAllConfigs(ctx) + + } + resp, err = d.db.GetConfigs(ctx, since) + if err != nil { + return nil, err + } + + return &ConfigsResponse{ + since: since, + Configs: resp, + }, nil +} + +// ConfigsResponse is a response from server for GetConfigs. +type ConfigsResponse struct { + // The version since which these configs were changed + since configs.ID + + // Configs maps user ID to their latest configs.View. + Configs map[string]configs.View `json:"configs"` +} + +// GetLatestConfigID returns the last config ID from a set of configs. +func (c ConfigsResponse) GetLatestConfigID() configs.ID { + latest := c.since + for _, config := range c.Configs { + if config.ID > latest { + latest = config.ID + } + } + return latest +} diff --git a/pkg/configs/client/config.go b/pkg/configs/client/config.go new file mode 100644 index 00000000000..3850b967c68 --- /dev/null +++ b/pkg/configs/client/config.go @@ -0,0 +1,70 @@ +package client + +import ( + "context" + "flag" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/weaveworks/common/instrument" + + "github.com/cortexproject/cortex/pkg/configs" + "github.com/cortexproject/cortex/pkg/configs/db" + "github.com/cortexproject/cortex/pkg/util/flagext" +) + +// Config says where we can find the ruler configs. +type Config struct { + DBConfig db.Config + + // DEPRECATED + ConfigsAPIURL flagext.URLValue + + // DEPRECATED. HTTP timeout duration for requests made to the Weave Cloud + // configs service. + ClientTimeout time.Duration +} + +// RegisterFlags adds the flags required to config this to the given FlagSet +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + cfg.DBConfig.RegisterFlags(f) + f.Var(&cfg.ConfigsAPIURL, "ruler.configs.url", "DEPRECATED. URL of configs API server.") + f.DurationVar(&cfg.ClientTimeout, "ruler.client-timeout", 5*time.Second, "DEPRECATED. Timeout for requests to Weave Cloud configs service.") + flag.Var(&cfg.ConfigsAPIURL, "alertmanager.configs.url", "URL of configs API server.") + flag.DurationVar(&cfg.ClientTimeout, "alertmanager.configs.client-timeout", 5*time.Second, "Timeout for requests to Weave Cloud configs service.") +} + +var configsRequestDuration = instrument.NewHistogramCollector(prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "configs_request_duration_seconds", + Help: "Time spent requesting configs.", + Buckets: prometheus.DefBuckets, +}, []string{"operation", "status_code"})) + +func init() { + configsRequestDuration.Register() +} + +type instrumented struct { + next Client +} + +func (i instrumented) GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { + var cfgs map[string]configs.VersionedRulesConfig + err := instrument.CollectedRequest(context.Background(), "Configs.GetConfigs", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { + var err error + cfgs, err = i.next.GetRules(ctx, since) // Warning: this will produce an incorrect result if the configID ever overflows + return err + }) + return cfgs, err +} + +func (i instrumented) GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) { + var cfgs *ConfigsResponse + err := instrument.CollectedRequest(context.Background(), "Configs.GetConfigs", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { + var err error + cfgs, err = i.next.GetAlerts(ctx, since) // Warning: this will produce an incorrect result if the configID ever overflows + return err + }) + return cfgs, err +} diff --git a/pkg/configs/storage/clients/client/configs_test.go b/pkg/configs/client/configs_test.go similarity index 94% rename from pkg/configs/storage/clients/client/configs_test.go rename to pkg/configs/client/configs_test.go index 20ae24b2b73..af7acf25b45 100644 --- a/pkg/configs/storage/clients/client/configs_test.go +++ b/pkg/configs/client/configs_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" - configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" + "github.com/cortexproject/cortex/pkg/configs" "github.com/stretchr/testify/assert" ) diff --git a/pkg/configs/configs.go b/pkg/configs/configs.go new file mode 100644 index 00000000000..fcf9c593125 --- /dev/null +++ b/pkg/configs/configs.go @@ -0,0 +1,298 @@ +package configs + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/go-kit/kit/log" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/rulefmt" + "github.com/prometheus/prometheus/promql" + "github.com/prometheus/prometheus/rules" + + legacy_promql "github.com/cortexproject/cortex/pkg/configs/legacy_promql" + "github.com/cortexproject/cortex/pkg/util" +) + +// An ID is the ID of a single users's Cortex configuration. When a +// configuration changes, it gets a new ID. +type ID int + +// RuleFormatVersion indicates which Prometheus rule format (v1 vs. v2) to use in parsing. +type RuleFormatVersion int + +const ( + // RuleFormatV1 is the Prometheus 1.x rule format. + RuleFormatV1 RuleFormatVersion = iota + // RuleFormatV2 is the Prometheus 2.x rule format. + RuleFormatV2 RuleFormatVersion = iota +) + +// IsValid returns whether the rules format version is a valid (known) version. +func (v RuleFormatVersion) IsValid() bool { + switch v { + case RuleFormatV1, RuleFormatV2: + return true + default: + return false + } +} + +// MarshalJSON implements json.Marshaler. +func (v RuleFormatVersion) MarshalJSON() ([]byte, error) { + switch v { + case RuleFormatV1: + return json.Marshal("1") + case RuleFormatV2: + return json.Marshal("2") + default: + return nil, fmt.Errorf("unknown rule format version %d", v) + } +} + +// UnmarshalJSON implements json.Unmarshaler. +func (v *RuleFormatVersion) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + switch s { + case "1": + *v = RuleFormatV1 + case "2": + *v = RuleFormatV2 + default: + return fmt.Errorf("unknown rule format version %q", string(data)) + } + return nil +} + +// A Config is a Cortex configuration for a single user. +type Config struct { + // RulesFiles maps from a rules filename to file contents. + RulesConfig RulesConfig + TemplateFiles map[string]string + AlertmanagerConfig string +} + +// configCompat is a compatibility struct to support old JSON config blobs +// saved in the config DB that didn't have a rule format version yet and +// just had a top-level field for the rule files. +type configCompat struct { + RulesFiles map[string]string `json:"rules_files"` + RuleFormatVersion RuleFormatVersion `json:"rule_format_version"` + TemplateFiles map[string]string `json:"template_files"` + AlertmanagerConfig string `json:"alertmanager_config"` +} + +// MarshalJSON implements json.Marshaler. +func (c Config) MarshalJSON() ([]byte, error) { + compat := &configCompat{ + RulesFiles: c.RulesConfig.Files, + RuleFormatVersion: c.RulesConfig.FormatVersion, + TemplateFiles: c.TemplateFiles, + AlertmanagerConfig: c.AlertmanagerConfig, + } + + return json.Marshal(compat) +} + +// UnmarshalJSON implements json.Unmarshaler. +func (c *Config) UnmarshalJSON(data []byte) error { + compat := configCompat{} + if err := json.Unmarshal(data, &compat); err != nil { + return err + } + *c = Config{ + RulesConfig: RulesConfig{ + Files: compat.RulesFiles, + FormatVersion: compat.RuleFormatVersion, + }, + TemplateFiles: compat.TemplateFiles, + AlertmanagerConfig: compat.AlertmanagerConfig, + } + return nil +} + +// View is what's returned from the Weave Cloud configs service +// when we ask for all Cortex configurations. +// +// The configs service is essentially a JSON blob store that gives each +// _version_ of a configuration a unique ID and guarantees that later versions +// have greater IDs. +type View struct { + ID ID `json:"id"` + Config Config `json:"config"` + DeletedAt time.Time `json:"deleted_at"` +} + +// GetVersionedRulesConfig specializes the view to just the rules config. +func (v View) GetVersionedRulesConfig() *VersionedRulesConfig { + if v.Config.RulesConfig.Files == nil { + return nil + } + return &VersionedRulesConfig{ + ID: v.ID, + Config: v.Config.RulesConfig, + DeletedAt: v.DeletedAt, + } +} + +// RulesConfig is the rules configuration for a particular organization. +type RulesConfig struct { + FormatVersion RuleFormatVersion `json:"format_version"` + Files map[string]string `json:"files"` +} + +// Equal compares two RulesConfigs for equality. +// +// instance Eq RulesConfig +func (c RulesConfig) Equal(o RulesConfig) bool { + if c.FormatVersion != o.FormatVersion { + return false + } + if len(o.Files) != len(c.Files) { + return false + } + for k, v1 := range c.Files { + v2, ok := o.Files[k] + if !ok || v1 != v2 { + return false + } + } + return true +} + +// Parse parses and validates the content of the rule files in a RulesConfig +// according to the passed rule format version. +func (c RulesConfig) Parse() (map[string][]rules.Rule, error) { + switch c.FormatVersion { + case RuleFormatV1: + return c.parseV1() + case RuleFormatV2: + return c.parseV2() + default: + return nil, fmt.Errorf("unknown rule format version %v", c.FormatVersion) + } +} + +// parseV2 parses and validates the content of the rule files in a RulesConfig +// according to the Prometheus 2.x rule format. +// +// NOTE: On one hand, we cannot return fully-fledged lists of rules.Group +// here yet, as creating a rules.Group requires already +// passing in rules.ManagerOptions options (which in turn require a +// notifier, appender, etc.), which we do not want to create simply +// for parsing. On the other hand, we should not return barebones +// rulefmt.RuleGroup sets here either, as only a fully-converted rules.Rule +// is able to track alert states over multiple rule evaluations. The caller +// would otherwise have to ensure to convert the rulefmt.RuleGroup only exactly +// once, not for every evaluation (or risk losing alert pending states). So +// it's probably better to just return a set of rules.Rule here. +func (c RulesConfig) parseV2() (map[string][]rules.Rule, error) { + groups := map[string][]rules.Rule{} + + for fn, content := range c.Files { + rgs, errs := rulefmt.Parse([]byte(content)) + if len(errs) > 0 { + return nil, fmt.Errorf("error parsing %s: %v", fn, errs[0]) + } + + for _, rg := range rgs.Groups { + rls := make([]rules.Rule, 0, len(rg.Rules)) + for _, rl := range rg.Rules { + expr, err := promql.ParseExpr(rl.Expr) + if err != nil { + return nil, err + } + + if rl.Alert != "" { + rls = append(rls, rules.NewAlertingRule( + rl.Alert, + expr, + time.Duration(rl.For), + labels.FromMap(rl.Labels), + labels.FromMap(rl.Annotations), + true, + log.With(util.Logger, "alert", rl.Alert), + )) + continue + } + rls = append(rls, rules.NewRecordingRule( + rl.Record, + expr, + labels.FromMap(rl.Labels), + )) + } + + // Group names have to be unique in Prometheus, but only within one rules file. + groups[rg.Name+";"+fn] = rls + } + } + + return groups, nil +} + +// parseV1 parses and validates the content of the rule files in a RulesConfig +// according to the Prometheus 1.x rule format. +// +// The same comment about rule groups as on ParseV2() applies here. +func (c RulesConfig) parseV1() (map[string][]rules.Rule, error) { + result := map[string][]rules.Rule{} + for fn, content := range c.Files { + stmts, err := legacy_promql.ParseStmts(content) + if err != nil { + return nil, fmt.Errorf("error parsing %s: %s", fn, err) + } + ra := []rules.Rule{} + for _, stmt := range stmts { + var rule rules.Rule + + switch r := stmt.(type) { + case *legacy_promql.AlertStmt: + // legacy_promql.ParseStmts has parsed the whole rule for us. + // Ideally we'd just use r.Expr and pass that to rules.NewAlertingRule, + // but it is of the type legacy_proql.Expr and not promql.Expr. + // So we convert it back to a string, and then parse it again with the + // upstream parser to get it into the right type. + expr, err := promql.ParseExpr(r.Expr.String()) + if err != nil { + return nil, err + } + + rule = rules.NewAlertingRule( + r.Name, expr, r.Duration, r.Labels, r.Annotations, true, + log.With(util.Logger, "alert", r.Name), + ) + + case *legacy_promql.RecordStmt: + expr, err := promql.ParseExpr(r.Expr.String()) + if err != nil { + return nil, err + } + + rule = rules.NewRecordingRule(r.Name, expr, r.Labels) + + default: + return nil, fmt.Errorf("ruler.GetRules: unknown statement type") + } + ra = append(ra, rule) + } + result[fn] = ra + } + return result, nil +} + +// VersionedRulesConfig is a RulesConfig together with a version. +// `data Versioned a = Versioned { id :: ID , config :: a }` +type VersionedRulesConfig struct { + ID ID `json:"id"` + Config RulesConfig `json:"config"` + DeletedAt time.Time `json:"deleted_at"` +} + +// IsDeleted tells you if the config is deleted. +func (vr VersionedRulesConfig) IsDeleted() bool { + return !vr.DeletedAt.IsZero() +} diff --git a/pkg/configs/configs_test.go b/pkg/configs/configs_test.go new file mode 100644 index 00000000000..d5c83c80041 --- /dev/null +++ b/pkg/configs/configs_test.go @@ -0,0 +1,105 @@ +package configs + +import ( + "encoding/json" + "strconv" + "testing" + "time" + + "github.com/go-kit/kit/log" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/promql" + "github.com/prometheus/prometheus/rules" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cortexproject/cortex/pkg/util" +) + +func TestUnmarshalLegacyConfigWithMissingRuleFormatVersionSucceeds(t *testing.T) { + actual := Config{} + buf := []byte(`{"rules_files": {"a": "b"}}`) + assert.Nil(t, json.Unmarshal(buf, &actual)) + + expected := Config{ + RulesConfig: RulesConfig{ + Files: map[string]string{ + "a": "b", + }, + FormatVersion: RuleFormatV1, + }, + } + + assert.Equal(t, expected, actual) +} + +func TestParseLegacyAlerts(t *testing.T) { + parsed, err := promql.ParseExpr("up == 0") + require.NoError(t, err) + rule := rules.NewAlertingRule( + "TestAlert", + parsed, + 5*time.Minute, + labels.Labels{ + labels.Label{Name: "severity", Value: "critical"}, + }, + labels.Labels{ + labels.Label{Name: "message", Value: "I am a message"}, + }, + true, + log.With(util.Logger, "alert", "TestAlert"), + ) + + for i, tc := range []struct { + cfg RulesConfig + expected map[string][]rules.Rule + }{ + { + cfg: RulesConfig{ + FormatVersion: RuleFormatV1, + Files: map[string]string{ + "legacy.rules": ` + ALERT TestAlert + IF up == 0 + FOR 5m + LABELS { severity = "critical" } + ANNOTATIONS { + message = "I am a message" + } + `, + }, + }, + expected: map[string][]rules.Rule{ + "legacy.rules": {rule}, + }, + }, + { + cfg: RulesConfig{ + FormatVersion: RuleFormatV2, + Files: map[string]string{ + "alerts.yaml": ` +groups: +- name: example + rules: + - alert: TestAlert + expr: up == 0 + for: 5m + labels: + severity: critical + annotations: + message: I am a message +`, + }, + }, + expected: map[string][]rules.Rule{ + "example;alerts.yaml": {rule}, + }, + }, + } { + t.Run(strconv.Itoa(i), func(t *testing.T) { + rules, err := tc.cfg.Parse() + require.NoError(t, err) + require.Equal(t, tc.expected, rules) + }) + } +} diff --git a/pkg/configs/db/db.go b/pkg/configs/db/db.go index 15452d44949..b04fcd36978 100644 --- a/pkg/configs/db/db.go +++ b/pkg/configs/db/db.go @@ -7,9 +7,9 @@ import ( "io/ioutil" "net/url" + "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/configs/db/memory" "github.com/cortexproject/cortex/pkg/configs/db/postgres" - configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" ) // Config configures the database. diff --git a/pkg/configs/db/memory/memory.go b/pkg/configs/db/memory/memory.go index 5d3498441d5..c2f8bbe2c0c 100644 --- a/pkg/configs/db/memory/memory.go +++ b/pkg/configs/db/memory/memory.go @@ -6,7 +6,7 @@ import ( "fmt" "time" - configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" + "github.com/cortexproject/cortex/pkg/configs" ) // DB is an in-memory database for testing, and local development diff --git a/pkg/configs/db/postgres/postgres.go b/pkg/configs/db/postgres/postgres.go index 52dd57b2c60..12f8e9b6aa5 100644 --- a/pkg/configs/db/postgres/postgres.go +++ b/pkg/configs/db/postgres/postgres.go @@ -8,7 +8,7 @@ import ( "time" "github.com/Masterminds/squirrel" - configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" + "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "github.com/lib/pq" diff --git a/pkg/configs/db/timed.go b/pkg/configs/db/timed.go index afc8784e6dc..54826582dfb 100644 --- a/pkg/configs/db/timed.go +++ b/pkg/configs/db/timed.go @@ -3,7 +3,7 @@ package db import ( "context" - configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" + "github.com/cortexproject/cortex/pkg/configs" "github.com/prometheus/client_golang/prometheus" "github.com/weaveworks/common/instrument" ) diff --git a/pkg/configs/db/traced.go b/pkg/configs/db/traced.go index 5a54a605eb7..a328b64adaa 100644 --- a/pkg/configs/db/traced.go +++ b/pkg/configs/db/traced.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" + "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" ) diff --git a/pkg/configs/storage/clients/client/config.go b/pkg/configs/storage/clients/client/config.go deleted file mode 100644 index 210abc9124f..00000000000 --- a/pkg/configs/storage/clients/client/config.go +++ /dev/null @@ -1,30 +0,0 @@ -package client - -import ( - "flag" - "time" - - "github.com/cortexproject/cortex/pkg/configs/db" - "github.com/cortexproject/cortex/pkg/util/flagext" -) - -// Config says where we can find the ruler configs. -type Config struct { - DBConfig db.Config - - // DEPRECATED - ConfigsAPIURL flagext.URLValue - - // DEPRECATED. HTTP timeout duration for requests made to the Weave Cloud - // configs service. - ClientTimeout time.Duration -} - -// RegisterFlags adds the flags required to config this to the given FlagSet -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - cfg.DBConfig.RegisterFlags(f) - f.Var(&cfg.ConfigsAPIURL, "ruler.configs.url", "DEPRECATED. URL of configs API server.") - f.DurationVar(&cfg.ClientTimeout, "ruler.client-timeout", 5*time.Second, "DEPRECATED. Timeout for requests to Weave Cloud configs service.") - flag.Var(&cfg.ConfigsAPIURL, "alertmanager.configs.url", "URL of configs API server.") - flag.DurationVar(&cfg.ClientTimeout, "alertmanager.configs.client-timeout", 5*time.Second, "Timeout for requests to Weave Cloud configs service.") -} diff --git a/pkg/configs/storage/clients/client_test.go b/pkg/configs/storage/clients/client_test.go deleted file mode 100644 index f97e3ef19a9..00000000000 --- a/pkg/configs/storage/clients/client_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package clients - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/configs/storage/clients/gcp" - "github.com/cortexproject/cortex/pkg/configs/storage/testutils" - "github.com/prometheus/prometheus/pkg/rulefmt" -) - -const ( - userID = "userID" - namespace = "default" -) - -var ( - exampleRGOne = rulefmt.RuleGroup{ - Name: "example_rulegroup_one", - } - exampleRGTwo = rulefmt.RuleGroup{ - Name: "example_rulegroup_two", - } -) - -func TestRuleStoreBasic(t *testing.T) { - forAllFixtures(t, func(t *testing.T, client configs.ConfigStore) { - const batchSize = 5 - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - - err := client.SetRuleGroup(ctx, userID, namespace, exampleRGOne) - assert.NoError(t, err) - - rg, err := client.GetRuleGroup(ctx, userID, namespace, exampleRGOne.Name) - assert.NoError(t, err) - assert.Equal(t, exampleRGOne.Name, rg.Name) - }) -} - -func forAllFixtures(t *testing.T, f func(t *testing.T, client configs.ConfigStore)) { - var fixtures []testutils.Fixture - fixtures = append(fixtures, gcp.Fixtures...) -} diff --git a/pkg/configs/storage/factory.go b/pkg/configs/storage/factory.go deleted file mode 100644 index 847917fe8fc..00000000000 --- a/pkg/configs/storage/factory.go +++ /dev/null @@ -1,46 +0,0 @@ -package storage - -import ( - "context" - "flag" - "fmt" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/configs/storage/clients/client" - "github.com/cortexproject/cortex/pkg/configs/storage/clients/gcp" -) - -// Config is used to config an alertstore -type Config struct { - BackendType string - - ClientConfig client.Config - GCSConfig gcp.GCSConfig -} - -// RegisterFlags registers flags. -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - f.StringVar(&cfg.BackendType, "configdb.backend", "client", "backend to use for storing and retrieving alerts") - cfg.ClientConfig.RegisterFlags(f) - cfg.GCSConfig.RegisterFlagsWithPrefix("configs.", f) -} - -// New returns a configstore -func New(cfg Config) (configs.ConfigStore, error) { - var ( - store configs.ConfigStore - err error - ) - switch cfg.BackendType { - case "client": - store, err = client.New(cfg.ClientConfig) - case "gcp": - store, err = gcp.NewGCSConfigClient(context.Background(), cfg.GCSConfig) - default: - return nil, fmt.Errorf("Unrecognized config storage client %v, choose one of: client, gcp", cfg.BackendType) - } - - return &instrumented{ - next: store, - }, err -} diff --git a/pkg/configs/storage/instrumented.go b/pkg/configs/storage/instrumented.go deleted file mode 100644 index 9a29a20e8c2..00000000000 --- a/pkg/configs/storage/instrumented.go +++ /dev/null @@ -1,108 +0,0 @@ -package storage - -import ( - "context" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/prometheus/pkg/rulefmt" - "github.com/weaveworks/common/instrument" -) - -var configsRequestDuration = instrument.NewHistogramCollector(prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "cortex", - Name: "configs_request_duration_seconds", - Help: "Time spent requesting configs.", - Buckets: prometheus.DefBuckets, -}, []string{"operation", "status_code"})) - -func init() { - configsRequestDuration.Register() -} - -type instrumented struct { - next configs.ConfigStore -} - -func (i instrumented) PollAlertConfigs(ctx context.Context) (map[string]configs.AlertConfig, error) { - var cfgs map[string]configs.AlertConfig - err := instrument.CollectedRequest(context.Background(), "Configs.PollAlertConfigs", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - cfgs, err = i.next.PollAlertConfigs(ctx) - return err - }) - return cfgs, err -} - -func (i instrumented) GetAlertConfig(ctx context.Context, userID string) (configs.AlertConfig, error) { - var cfg configs.AlertConfig - err := instrument.CollectedRequest(context.Background(), "Configs.GetAlertConfig", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - cfg, err = i.next.GetAlertConfig(ctx, userID) - return err - }) - return cfg, err -} - -func (i instrumented) SetAlertConfig(ctx context.Context, userID string, config configs.AlertConfig) error { - return instrument.CollectedRequest(context.Background(), "Configs.SetAlertConfig", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - err = i.next.SetAlertConfig(ctx, userID, config) - return err - }) -} - -func (i instrumented) DeleteAlertConfig(ctx context.Context, userID string) error { - return instrument.CollectedRequest(context.Background(), "Configs.DeleteAlertConfig", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - err = i.next.DeleteAlertConfig(ctx, userID) - return err - }) -} - -func (i instrumented) PollRules(ctx context.Context) (map[string][]configs.RuleGroup, error) { - var cfgs map[string][]configs.RuleGroup - err := instrument.CollectedRequest(context.Background(), "Configs.PollRules", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - cfgs, err = i.next.PollRules(ctx) - return err - }) - - return cfgs, err -} - -func (i instrumented) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) (map[string]configs.RuleNamespace, error) { - var cfgs map[string]configs.RuleNamespace - err := instrument.CollectedRequest(context.Background(), "Configs.ListRuleGroups", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - cfgs, err = i.next.ListRuleGroups(ctx, options) - return err - }) - return cfgs, err -} - -func (i instrumented) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (rulefmt.RuleGroup, error) { - var cfg rulefmt.RuleGroup - err := instrument.CollectedRequest(context.Background(), "Configs.GetRuleGroup", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - cfg, err = i.next.GetRuleGroup(ctx, userID, namespace, group) - return err - }) - return cfg, err -} - -func (i instrumented) SetRuleGroup(ctx context.Context, userID string, namespace string, group rulefmt.RuleGroup) error { - return instrument.CollectedRequest(context.Background(), "Configs.SetRuleGroup", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - err = i.next.SetRuleGroup(ctx, userID, namespace, group) - return err - }) -} - -func (i instrumented) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { - return instrument.CollectedRequest(context.Background(), "Configs.DeleteRuleGroup", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - err = i.next.DeleteRuleGroup(ctx, userID, namespace, group) - return err - }) -} diff --git a/pkg/configs/storage/testutils/testutils.go b/pkg/configs/storage/testutils/testutils.go deleted file mode 100644 index 96b79fca50f..00000000000 --- a/pkg/configs/storage/testutils/testutils.go +++ /dev/null @@ -1,12 +0,0 @@ -package testutils - -import ( - "github.com/cortexproject/cortex/pkg/configs" -) - -// Fixture type for per-backend testing. -type Fixture interface { - Name() string - Clients() (configs.ConfigStore, error) - Teardown() error -} diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index b1144143bff..30271445a4c 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -19,7 +19,6 @@ import ( chunk_util "github.com/cortexproject/cortex/pkg/chunk/util" "github.com/cortexproject/cortex/pkg/configs/api" "github.com/cortexproject/cortex/pkg/configs/db" - config_storage "github.com/cortexproject/cortex/pkg/configs/storage" "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/ingester" "github.com/cortexproject/cortex/pkg/ingester/client" @@ -27,6 +26,7 @@ import ( "github.com/cortexproject/cortex/pkg/querier/frontend" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ruler" + config_storage "github.com/cortexproject/cortex/pkg/storage" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -69,8 +69,9 @@ type Config struct { TableManager chunk.TableManagerConfig `yaml:"table_manager,omitempty"` Encoding encoding.Config `yaml:"-"` // No yaml for this, it only works with flags. - Ruler ruler.Config `yaml:"ruler,omitempty"` + ConfigDB db.Config `yaml:"configdb,omitempty"` ConfigStore config_storage.Config `yaml:"config_store,omitempty"` + Ruler ruler.Config `yaml:"ruler,omitempty"` Alertmanager alertmanager.MultitenantAlertmanagerConfig `yaml:"alertmanager,omitempty"` } @@ -99,6 +100,7 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { c.Encoding.RegisterFlags(f) c.Ruler.RegisterFlags(f) + c.ConfigDB.RegisterFlags(f) c.ConfigStore.RegisterFlags(f) c.Alertmanager.RegisterFlags(f) diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index ec293631b98..5fb727040be 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -22,7 +22,6 @@ import ( "github.com/cortexproject/cortex/pkg/chunk/storage" "github.com/cortexproject/cortex/pkg/configs/api" "github.com/cortexproject/cortex/pkg/configs/db" - config_storage "github.com/cortexproject/cortex/pkg/configs/storage" "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/ingester" "github.com/cortexproject/cortex/pkg/ingester/client" @@ -30,6 +29,7 @@ import ( "github.com/cortexproject/cortex/pkg/querier/frontend" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ruler" + config_storage "github.com/cortexproject/cortex/pkg/storage" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -320,7 +320,7 @@ func (t *Cortex) initRuler(cfg *Config) (err error) { cfg.Ruler.LifecyclerConfig.ListenPort = &cfg.Server.GRPCListenPort queryable, engine := querier.New(cfg.Querier, t.distributor, t.store) - store, err := config_storage.New(cfg.ConfigStore) + store, err := config_storage.NewRuleStore(cfg.ConfigStore) if err != nil { return err } @@ -333,7 +333,7 @@ func (t *Cortex) initRuler(cfg *Config) (err error) { // Only serve the API for setting & getting rules configs if we're not // serving configs from the configs API. Allows for smoother // migration. See https://github.com/cortexproject/cortex/issues/619 - if cfg.ConfigStore.BackendType != "client" { + if cfg.ConfigStore.RuleStoreConfig.BackendType != "configdb" { a := ruler.NewAPI(store) a.RegisterRoutes(t.server.HTTP) } @@ -348,7 +348,7 @@ func (t *Cortex) stopRuler() error { } func (t *Cortex) initConfigs(cfg *Config) (err error) { - t.configDB, err = db.New(cfg.ConfigStore.ClientConfig.DBConfig) + t.configDB, err = db.New(cfg.ConfigDB) if err != nil { return } @@ -364,7 +364,7 @@ func (t *Cortex) stopConfigs() error { } func (t *Cortex) initAlertmanager(cfg *Config) (err error) { - store, err := config_storage.New(cfg.ConfigStore) + store, err := config_storage.NewAlertStore(cfg.ConfigStore) if err != nil { return err } @@ -377,7 +377,7 @@ func (t *Cortex) initAlertmanager(cfg *Config) (err error) { // Only serve the API for setting & getting alert configs if we're not // serving configs from the configs API. Allows for smoother // migration. See https://github.com/cortexproject/cortex/issues/619 - if cfg.ConfigStore.BackendType != "client" { + if cfg.ConfigStore.AlertStoreConfig.BackendType != "configdb" { a := alertmanager.NewAPI(store) a.RegisterRoutes(t.server.HTTP) } diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index a3c457c1484..d542e2c3a4a 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -7,7 +7,6 @@ import ( "github.com/prometheus/prometheus/pkg/rulefmt" - "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "github.com/gorilla/mux" @@ -23,11 +22,11 @@ var ( // API is used to provided endpoints to directly interact with the ruler type API struct { - store configs.RuleStore + store RuleStore } // NewAPI returns a ruler API -func NewAPI(store configs.RuleStore) *API { +func NewAPI(store RuleStore) *API { return &API{store} } @@ -67,7 +66,7 @@ func (a *API) listRules(w http.ResponseWriter, r *http.Request) { return } - options := configs.RuleStoreConditions{ + options := RuleStoreConditions{ UserID: userID, } @@ -145,7 +144,7 @@ func (a *API) getRuleGroup(w http.ResponseWriter, r *http.Request) { rg, err := a.store.GetRuleGroup(r.Context(), userID, ns, gn) if err != nil { - if err == configs.ErrGroupNotFound { + if err == ErrGroupNotFound { http.Error(w, err.Error(), http.StatusNotFound) } http.Error(w, err.Error(), http.StatusBadRequest) @@ -213,7 +212,7 @@ func (a *API) createRuleGroup(w http.ResponseWriter, r *http.Request) { return } - errs := configs.ValidateRuleGroup(rg) + errs := ValidateRuleGroup(rg) if len(errs) > 0 { level.Error(logger).Log("err", err.Error()) http.Error(w, errs[0].Error(), http.StatusBadRequest) diff --git a/pkg/ruler/group.go b/pkg/ruler/group.go index a9a9d34678e..be56338f35d 100644 --- a/pkg/ruler/group.go +++ b/pkg/ruler/group.go @@ -7,24 +7,24 @@ import ( "github.com/prometheus/prometheus/rules" ) -// group is a wrapper around a prometheus rules.Group, with a mutable appendable +// wrappedGroup is a wrapper around a prometheus rules.Group, with a mutable appendable // appendable stored here will be the same appendable as in promGroup.opts.Appendable -type group struct { +type wrappedGroup struct { promGroup *rules.Group appendable *appendableAppender } -func newGroup(name string, rls []rules.Rule, appendable *appendableAppender, opts *rules.ManagerOptions) *group { +func newGroup(name string, rls []rules.Rule, appendable *appendableAppender, opts *rules.ManagerOptions) *wrappedGroup { delay := 0 * time.Second // Unused, so 0 value is fine. promGroup := rules.NewGroup(name, "none", delay, rls, false, opts) - return &group{promGroup, appendable} + return &wrappedGroup{promGroup, appendable} } -func (g *group) Eval(ctx context.Context, ts time.Time) { +func (g *wrappedGroup) Eval(ctx context.Context, ts time.Time) { g.appendable.ctx = ctx g.promGroup.Eval(ctx, ts) } -func (g *group) Rules() []rules.Rule { +func (g *wrappedGroup) Rules() []rules.Rule { return g.promGroup.Rules() } diff --git a/pkg/ruler/group/compat.go b/pkg/ruler/group/compat.go index f26700b921b..f891cec0f66 100644 --- a/pkg/ruler/group/compat.go +++ b/pkg/ruler/group/compat.go @@ -3,9 +3,9 @@ package group import ( time "time" - "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" @@ -44,7 +44,7 @@ func ToProto(namespace string, rl rulefmt.RuleGroup) RuleGroup { } // FromProto generates a rulefmt RuleGroup -func FromProto(rg *RuleGroup) rulefmt.RuleGroup { +func FromProto(rg *RuleGroup) *rulefmt.RuleGroup { formattedRuleGroup := rulefmt.RuleGroup{ Name: rg.GetName(), Interval: model.Duration(*rg.Interval), @@ -62,11 +62,11 @@ func FromProto(rg *RuleGroup) rulefmt.RuleGroup { } } - return formattedRuleGroup + return &formattedRuleGroup } // GenerateRuleGroup returns a functional rulegroup from a proto -func GenerateRuleGroup(userID string, rg *RuleGroup) (configs.RuleGroup, error) { +func GenerateRuleGroup(userID string, rg *RuleGroup) (*Group, error) { rls := make([]rules.Rule, 0, len(rg.Rules)) for _, rl := range rg.Rules { expr, err := promql.ParseExpr(rl.GetExpr()) diff --git a/pkg/ruler/group/rulegroup.go b/pkg/ruler/group/rulegroup.go index 94e4269dd8d..6f17e197c50 100644 --- a/pkg/ruler/group/rulegroup.go +++ b/pkg/ruler/group/rulegroup.go @@ -3,7 +3,6 @@ package group import ( "context" - "github.com/cortexproject/cortex/pkg/configs" "github.com/prometheus/prometheus/rules" ) @@ -11,7 +10,7 @@ import ( // TODO: The cortex project should implement a separate Group struct from // the prometheus project. This will allow for more precise instrumentation -type ruleGroup struct { +type Group struct { name string namespace string user string @@ -19,21 +18,21 @@ type ruleGroup struct { rules []rules.Rule } -func (rg *ruleGroup) Rules(ctx context.Context) ([]rules.Rule, error) { +func (rg *Group) Rules(ctx context.Context) ([]rules.Rule, error) { return rg.rules, nil } -func (rg *ruleGroup) Name() string { +func (rg *Group) Name() string { return rg.namespace + "/" + rg.name } -func (rg *ruleGroup) User() string { +func (rg *Group) User() string { return rg.user } // NewRuleGroup returns a rulegroup -func NewRuleGroup(name, namespace, user string, rules []rules.Rule) configs.RuleGroup { - return &ruleGroup{ +func NewRuleGroup(name, namespace, user string, rules []rules.Rule) *Group { + return &Group{ name: name, namespace: namespace, user: user, diff --git a/pkg/configs/storage.go b/pkg/ruler/rule_store.go similarity index 74% rename from pkg/configs/storage.go rename to pkg/ruler/rule_store.go index f62737a03b9..a728e0b6858 100644 --- a/pkg/configs/storage.go +++ b/pkg/ruler/rule_store.go @@ -1,4 +1,4 @@ -package configs +package ruler import ( "context" @@ -15,22 +15,6 @@ var ( ErrUserNotFound = errors.New("no rule groups found for user") ) -// ConfigStore unifies the AlertStore and ConfigStore interface -// TODO: Remove polling from the configstorage backend. A notification queue system would work better for this use case. -type ConfigStore interface { - AlertStore - RuleStore -} - -// AlertStore stores config information and template files to configure alertmanager tenants -type AlertStore interface { - PollAlertConfigs(ctx context.Context) (map[string]AlertConfig, error) - - GetAlertConfig(ctx context.Context, userID string) (AlertConfig, error) - SetAlertConfig(ctx context.Context, userID string, config AlertConfig) error - DeleteAlertConfig(ctx context.Context, userID string) error -} - // RuleStoreConditions are used to filter retrieived results from a rule store type RuleStoreConditions struct { // UserID specifies to only retrieve rules with this ID @@ -46,17 +30,11 @@ type RuleStore interface { PollRules(ctx context.Context) (map[string][]RuleGroup, error) ListRuleGroups(ctx context.Context, options RuleStoreConditions) (map[string]RuleNamespace, error) - GetRuleGroup(ctx context.Context, userID, namespace, group string) (rulefmt.RuleGroup, error) + GetRuleGroup(ctx context.Context, userID, namespace, group string) (*rulefmt.RuleGroup, error) SetRuleGroup(ctx context.Context, userID, namespace string, group rulefmt.RuleGroup) error DeleteRuleGroup(ctx context.Context, userID, namespace string, group string) error } -// AlertConfig is used to configure user alert managers -type AlertConfig struct { - TemplateFiles map[string]string `json:"template_files"` - AlertmanagerConfig string `json:"alertmanager_config"` -} - // RuleGroup is used to retrieve rules from the database to evaluate, // an interface is used to allow for lazy evaluation implementations type RuleGroup interface { diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index def7e6589d1..5a9a7774755 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -22,7 +22,6 @@ import ( "golang.org/x/net/context" "golang.org/x/net/context/ctxhttp" - "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/util" @@ -135,7 +134,7 @@ type Ruler struct { } // NewRuler creates a new ruler from a distributor and chunk store. -func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d *distributor.Distributor, store configs.RuleStore) (*Ruler, error) { +func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d *distributor.Distributor, store RuleStore) (*Ruler, error) { if cfg.NumWorkers <= 0 { return nil, fmt.Errorf("must have at least 1 worker, got %d", cfg.NumWorkers) } @@ -213,7 +212,7 @@ func (r *Ruler) Stop() { } } -func (r *Ruler) newGroup(ctx context.Context, g configs.RuleGroup) (*group, error) { +func (r *Ruler) newGroup(ctx context.Context, g RuleGroup) (*wrappedGroup, error) { appendable := &appendableAppender{pusher: r.pusher} notifier, err := r.getOrCreateNotifier(g.User()) if err != nil { diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index fa38763ac3e..a194af7fa6b 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -9,17 +9,15 @@ import ( "testing" "time" - "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/pkg/rulefmt" - "github.com/prometheus/prometheus/promql" - - "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/querier" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ring/kv/codec" "github.com/cortexproject/cortex/pkg/ring/kv/consul" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/prometheus/prometheus/notifier" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/rulefmt" + "github.com/prometheus/prometheus/promql" "github.com/stretchr/testify/assert" "github.com/weaveworks/common/user" ) @@ -27,20 +25,20 @@ import ( type mockRuleStore struct { rules map[string]rulefmt.RuleGroup - pollPayload map[string][]configs.RuleGroup + pollPayload map[string][]RuleGroup } -func (m *mockRuleStore) PollRules(ctx context.Context) (map[string][]configs.RuleGroup, error) { +func (m *mockRuleStore) PollRules(ctx context.Context) (map[string][]RuleGroup, error) { pollPayload := m.pollPayload - m.pollPayload = map[string][]configs.RuleGroup{} + m.pollPayload = map[string][]RuleGroup{} return pollPayload, nil } -func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) (map[string]configs.RuleNamespace, error) { +func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options RuleStoreConditions) (map[string]RuleNamespace, error) { groupPrefix := userID + ":" namespaces := []string{} - nss := map[string]configs.RuleNamespace{} + nss := map[string]RuleNamespace{} for n := range m.rules { if strings.HasPrefix(n, groupPrefix) { components := strings.Split(n, ":") @@ -52,7 +50,7 @@ func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options configs.Rule } if len(namespaces) == 0 { - return nss, configs.ErrUserNotFound + return nss, ErrUserNotFound } for _, n := range namespaces { @@ -67,10 +65,10 @@ func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options configs.Rule return nss, nil } -func (m *mockRuleStore) getRuleNamespace(ctx context.Context, userID string, namespace string) (configs.RuleNamespace, error) { +func (m *mockRuleStore) getRuleNamespace(ctx context.Context, userID string, namespace string) (RuleNamespace, error) { groupPrefix := userID + ":" + namespace + ":" - ns := configs.RuleNamespace{ + ns := RuleNamespace{ Groups: []rulefmt.RuleGroup{}, } for n, g := range m.rules { @@ -80,21 +78,21 @@ func (m *mockRuleStore) getRuleNamespace(ctx context.Context, userID string, nam } if len(ns.Groups) == 0 { - return ns, configs.ErrGroupNamespaceNotFound + return ns, ErrGroupNamespaceNotFound } return ns, nil } -func (m *mockRuleStore) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (rulefmt.RuleGroup, error) { +func (m *mockRuleStore) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (*rulefmt.RuleGroup, error) { groupID := userID + ":" + namespace + ":" + group g, ok := m.rules[groupID] if !ok { - return rulefmt.RuleGroup{}, configs.ErrGroupNotFound + return nil, ErrGroupNotFound } - return g, nil + return &g, nil } diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index 97fd22571dd..973f4fc7906 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -9,7 +9,6 @@ import ( "sync" "time" - "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "github.com/jonboulle/clockwork" @@ -44,7 +43,7 @@ type workItem struct { userID string groupName string hash uint32 - group *group + group *wrappedGroup scheduled time.Time done chan struct{} @@ -72,13 +71,13 @@ func (w workItem) String() string { type userConfig struct { done chan struct{} id string - rules []configs.RuleGroup + rules []RuleGroup } -type groupFactory func(context.Context, configs.RuleGroup) (*group, error) +type groupFactory func(context.Context, RuleGroup) (*wrappedGroup, error) type scheduler struct { - store configs.RuleStore + store RuleStore evaluationInterval time.Duration // how often we re-evaluate each rule set q *SchedulingQueue @@ -91,7 +90,7 @@ type scheduler struct { } // newScheduler makes a new scheduler. -func newScheduler(store configs.RuleStore, evaluationInterval, pollInterval time.Duration, groupFn groupFactory) *scheduler { +func newScheduler(store RuleStore, evaluationInterval, pollInterval time.Duration, groupFn groupFactory) *scheduler { return &scheduler{ store: store, evaluationInterval: evaluationInterval, @@ -148,7 +147,7 @@ func (s *scheduler) updateConfigs(ctx context.Context) error { return nil } -func (s *scheduler) addUserConfig(ctx context.Context, userID string, rgs []configs.RuleGroup) { +func (s *scheduler) addUserConfig(ctx context.Context, userID string, rgs []RuleGroup) { level.Info(util.Logger).Log("msg", "scheduler: updating rules for user", "user_id", userID, "num_groups", len(rgs)) // create a new userchan for rulegroups of this user diff --git a/pkg/ruler/scheduler_test.go b/pkg/ruler/scheduler_test.go index b4430e4888a..d7b6cdca4cc 100644 --- a/pkg/ruler/scheduler_test.go +++ b/pkg/ruler/scheduler_test.go @@ -6,7 +6,6 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/pkg/configs" "github.com/cortexproject/cortex/pkg/ruler/group" "github.com/prometheus/prometheus/rules" "github.com/stretchr/testify/assert" @@ -77,11 +76,11 @@ func TestSchedulerRulesOverlap(t *testing.T) { groupTwo := "test2" next := time.Now() - ruleSetsOne := []configs.RuleGroup{ + ruleSetsOne := []RuleGroup{ group.NewRuleGroup(groupOne, "default", userID, []rules.Rule{nil}), } - ruleSetsTwo := []configs.RuleGroup{ + ruleSetsTwo := []RuleGroup{ group.NewRuleGroup(groupTwo, "default", userID, []rules.Rule{nil}), } userChanOne := make(chan struct{}) diff --git a/pkg/configs/storage/clients/aws/s3_config_client.go b/pkg/storage/clients/aws/s3_config_client.go similarity index 99% rename from pkg/configs/storage/clients/aws/s3_config_client.go rename to pkg/storage/clients/aws/s3_config_client.go index fbc835e98b6..2765e55031d 100644 --- a/pkg/configs/storage/clients/aws/s3_config_client.go +++ b/pkg/storage/clients/aws/s3_config_client.go @@ -93,7 +93,7 @@ func (a *s3ConfigClient) DeleteAlertConfig(ctx context.Context, userID string) e panic("not implemented") } -func (a *s3ConfigClient) PollRules(ctx context.Context) (map[string][]configs.RuleGroup, error) { +func (a *s3ConfigClient) PollRules(ctx context.Context) (map[string][]storage.RuleGroup error) { panic("not implemented") } diff --git a/pkg/storage/clients/client_test.go b/pkg/storage/clients/client_test.go new file mode 100644 index 00000000000..b4039b7bba9 --- /dev/null +++ b/pkg/storage/clients/client_test.go @@ -0,0 +1,59 @@ +package clients + +import ( + "context" + "testing" + "time" + + "github.com/cortexproject/cortex/pkg/storage" + "github.com/cortexproject/cortex/pkg/storage/clients/gcp" + "github.com/cortexproject/cortex/pkg/storage/testutils" + "github.com/prometheus/prometheus/pkg/rulefmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + userID = "userID" + namespace = "default" +) + +var ( + exampleRuleGrp = rulefmt.RuleGroup{ + Name: "example_rulegroup_one", + } +) + +func TestRuleStoreBasic(t *testing.T) { + forAllFixtures(t, func(t *testing.T, _ storage.AlertStore, client storage.RuleStore) { + const batchSize = 5 + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + err := client.SetRuleGroup(ctx, userID, namespace, exampleRuleGrp) + require.NoError(t, err) + + rg, err := client.GetRuleGroup(ctx, userID, namespace, exampleRuleGrp.Name) + require.NoError(t, err) + assert.Equal(t, exampleRuleGrp.Name, rg.Name) + + err = client.DeleteRuleGroup(ctx, userID, namespace, exampleRuleGrp.Name) + require.NoError(t, err) + + rg, err = client.GetRuleGroup(ctx, userID, namespace, exampleRuleGrp.Name) + require.Error(t, err) + assert.Nil(t, rg) + }) +} + +func forAllFixtures(t *testing.T, clientTest func(*testing.T, storage.AlertStore, storage.RuleStore)) { + var fixtures []testutils.Fixture + fixtures = append(fixtures, gcp.Fixtures...) + + for _, fixture := range fixtures { + a, r, err := fixture.Clients() + require.NoError(t, err) + + clientTest(t, a, r) + } +} diff --git a/pkg/configs/storage/clients/client/client.go b/pkg/storage/clients/configdb/client.go similarity index 65% rename from pkg/configs/storage/clients/client/client.go rename to pkg/storage/clients/configdb/client.go index 5102c57562e..ed3dc39689f 100644 --- a/pkg/configs/storage/clients/client/client.go +++ b/pkg/storage/clients/configdb/client.go @@ -1,4 +1,4 @@ -package client +package configdb import ( "context" @@ -10,18 +10,26 @@ import ( "strings" "time" - "github.com/cortexproject/cortex/pkg/ruler/group" - + "github.com/cortexproject/cortex/pkg/alertmanager" "github.com/cortexproject/cortex/pkg/configs" - legacy_configs "github.com/cortexproject/cortex/pkg/configs/legacy_configs" + "github.com/cortexproject/cortex/pkg/ruler" + "github.com/cortexproject/cortex/pkg/ruler/group" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "github.com/prometheus/prometheus/pkg/rulefmt" ) +// ConfigsClient allows retrieving recording and alerting rules from the configs server. +type ConfigsClient struct { + URL *url.URL + Timeout time.Duration + + lastPoll configs.ID +} + // New creates a new ConfigClient. -func New(cfg Config) (configs.ConfigStore, error) { - return &configsClient{ +func New(cfg Config) (*ConfigsClient, error) { + return &ConfigsClient{ URL: cfg.ConfigsAPIURL.URL, Timeout: cfg.ClientTimeout, @@ -29,16 +37,8 @@ func New(cfg Config) (configs.ConfigStore, error) { }, nil } -// configsClient allows retrieving recording and alerting rules from the configs server. -type configsClient struct { - URL *url.URL - Timeout time.Duration - - lastPoll legacy_configs.ID -} - // GetRules implements ConfigClient. -func (c *configsClient) GetRules(ctx context.Context, since legacy_configs.ID) (map[string]legacy_configs.VersionedRulesConfig, error) { +func (c *ConfigsClient) GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { suffix := "" if since != 0 { suffix = fmt.Sprintf("?since=%d", since) @@ -48,7 +48,7 @@ func (c *configsClient) GetRules(ctx context.Context, since legacy_configs.ID) ( if err != nil { return nil, err } - configs := map[string]legacy_configs.VersionedRulesConfig{} + configs := map[string]configs.VersionedRulesConfig{} for id, view := range response.Configs { cfg := view.GetVersionedRulesConfig() if cfg != nil { @@ -59,7 +59,7 @@ func (c *configsClient) GetRules(ctx context.Context, since legacy_configs.ID) ( } // GetAlerts implements ConfigClient. -func (c *configsClient) GetAlerts(ctx context.Context, since legacy_configs.ID) (*ConfigsResponse, error) { +func (c *ConfigsClient) GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) { suffix := "" if since != 0 { suffix = fmt.Sprintf("?since=%d", since) @@ -68,7 +68,7 @@ func (c *configsClient) GetAlerts(ctx context.Context, since legacy_configs.ID) return doRequest(endpoint, c.Timeout, since) } -func doRequest(endpoint string, timeout time.Duration, since legacy_configs.ID) (*ConfigsResponse, error) { +func doRequest(endpoint string, timeout time.Duration, since configs.ID) (*ConfigsResponse, error) { req, err := http.NewRequest("GET", endpoint, nil) if err != nil { return nil, err @@ -98,14 +98,14 @@ func doRequest(endpoint string, timeout time.Duration, since legacy_configs.ID) // ConfigsResponse is a response from server for GetConfigs. type ConfigsResponse struct { // The version since which these configs were changed - since legacy_configs.ID + since configs.ID // Configs maps user ID to their latest configs.View. - Configs map[string]legacy_configs.View `json:"configs"` + Configs map[string]configs.View `json:"configs"` } // GetLatestConfigID returns the last config ID from a set of configs. -func (c ConfigsResponse) GetLatestConfigID() legacy_configs.ID { +func (c ConfigsResponse) GetLatestConfigID() configs.ID { latest := c.since for _, config := range c.Configs { if config.ID > latest { @@ -115,15 +115,15 @@ func (c ConfigsResponse) GetLatestConfigID() legacy_configs.ID { return latest } -func (c *configsClient) PollAlertConfigs(ctx context.Context) (map[string]configs.AlertConfig, error) { +func (c *ConfigsClient) PollAlertConfigs(ctx context.Context) (map[string]alertmanager.AlertConfig, error) { resp, err := c.GetAlerts(ctx, c.lastPoll) if err != nil { return nil, err } - newConfigs := map[string]configs.AlertConfig{} + newConfigs := map[string]alertmanager.AlertConfig{} for user, c := range resp.Configs { - newConfigs[user] = configs.AlertConfig{ + newConfigs[user] = alertmanager.AlertConfig{ TemplateFiles: c.Config.TemplateFiles, AlertmanagerConfig: c.Config.AlertmanagerConfig, } @@ -135,16 +135,16 @@ func (c *configsClient) PollAlertConfigs(ctx context.Context) (map[string]config } // PollRules polls the configdb server and returns the updated rule groups -func (c *configsClient) PollRules(ctx context.Context) (map[string][]configs.RuleGroup, error) { +func (c *ConfigsClient) PollRules(ctx context.Context) (map[string][]ruler.RuleGroup, error) { resp, err := c.GetAlerts(ctx, c.lastPoll) if err != nil { return nil, err } - newRules := map[string][]configs.RuleGroup{} + newRules := map[string][]ruler.RuleGroup{} for user, cfg := range resp.Configs { - userRules := []configs.RuleGroup{} + userRules := []ruler.RuleGroup{} rls := cfg.GetVersionedRulesConfig() rMap, err := rls.Config.Parse() if err != nil { @@ -173,30 +173,30 @@ func decomposeGroupSlug(slug string) (string, string) { return components[0], components[1] } -func (c *configsClient) GetAlertConfig(ctx context.Context, userID string) (configs.AlertConfig, error) { - return configs.AlertConfig{}, errors.New("remote configdb client does not implement GetAlertConfig") +func (c *ConfigsClient) GetAlertConfig(ctx context.Context, userID string) (alertmanager.AlertConfig, error) { + return alertmanager.AlertConfig{}, errors.New("remote configdb client does not implement GetAlertConfig") } -func (c *configsClient) SetAlertConfig(ctx context.Context, userID string, config configs.AlertConfig) error { +func (c *ConfigsClient) SetAlertConfig(ctx context.Context, userID string, config alertmanager.AlertConfig) error { return errors.New("remote configdb client does not implement SetAlertConfig") } -func (c *configsClient) DeleteAlertConfig(ctx context.Context, userID string) error { +func (c *ConfigsClient) DeleteAlertConfig(ctx context.Context, userID string) error { return errors.New("remote configdb client does not implement DeleteAlertConfig") } -func (c *configsClient) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) (map[string]configs.RuleNamespace, error) { +func (c *ConfigsClient) ListRuleGroups(ctx context.Context, options ruler.RuleStoreConditions) (map[string]ruler.RuleNamespace, error) { return nil, errors.New("remote configdb client does not implement ListRule") } -func (c *configsClient) GetRuleGroup(ctx context.Context, userID, namespace, group string) (rulefmt.RuleGroup, error) { - return rulefmt.RuleGroup{}, errors.New("remote configdb client does not implement GetRuleGroup") +func (c *ConfigsClient) GetRuleGroup(ctx context.Context, userID, namespace, group string) (*rulefmt.RuleGroup, error) { + return nil, errors.New("remote configdb client does not implement GetRuleGroup") } -func (c *configsClient) SetRuleGroup(ctx context.Context, userID, namespace string, group rulefmt.RuleGroup) error { +func (c *ConfigsClient) SetRuleGroup(ctx context.Context, userID, namespace string, group rulefmt.RuleGroup) error { return errors.New("remote configdb client does not implement SetRuleGroup") } -func (c *configsClient) DeleteRuleGroup(ctx context.Context, userID, namespace string, group string) error { +func (c *ConfigsClient) DeleteRuleGroup(ctx context.Context, userID, namespace string, group string) error { return errors.New("remote configdb client does not implement DeleteRuleGroup") } diff --git a/pkg/storage/clients/configdb/config.go b/pkg/storage/clients/configdb/config.go new file mode 100644 index 00000000000..c344c15f4ca --- /dev/null +++ b/pkg/storage/clients/configdb/config.go @@ -0,0 +1,28 @@ +package configdb + +import ( + "flag" + "time" + + "github.com/cortexproject/cortex/pkg/configs/db" + "github.com/cortexproject/cortex/pkg/util/flagext" +) + +// Config says where we can find the ruler configs. +type Config struct { + DBConfig db.Config + + // DEPRECATED + ConfigsAPIURL flagext.URLValue + + // DEPRECATED. HTTP timeout duration for requests made to the Weave Cloud + // configs service. + ClientTimeout time.Duration +} + +// RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet +func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + cfg.DBConfig.RegisterFlags(f) + f.Var(&cfg.ConfigsAPIURL, prefix+".configs.url", "DEPRECATED. URL of configs API server.") + f.DurationVar(&cfg.ClientTimeout, prefix+".client-timeout", 5*time.Second, "DEPRECATED. Timeout for requests to Weave Cloud configs service.") +} diff --git a/pkg/storage/clients/configdb/configs_test.go b/pkg/storage/clients/configdb/configs_test.go new file mode 100644 index 00000000000..bf0d38cdbb5 --- /dev/null +++ b/pkg/storage/clients/configdb/configs_test.go @@ -0,0 +1,54 @@ +package configdb + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cortexproject/cortex/pkg/configs" + "github.com/stretchr/testify/assert" +) + +var response = `{ + "configs": { + "2": { + "id": 1, + "config": { + "rules_files": { + "recording.rules": "groups:\n- name: demo-service-alerts\n interval: 15s\n rules:\n - alert: SomethingIsUp\n expr: up == 1\n" + }, + "rule_format_version": "2" + } + } + } +} +` + +func TestDoRequest(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, err := w.Write([]byte(response)) + require.NoError(t, err) + })) + defer server.Close() + + resp, err := doRequest(server.URL, 1*time.Second, 0) + assert.Nil(t, err) + + expected := ConfigsResponse{Configs: map[string]configs.View{ + "2": { + ID: 1, + Config: configs.Config{ + RulesConfig: configs.RulesConfig{ + Files: map[string]string{ + "recording.rules": "groups:\n- name: demo-service-alerts\n interval: 15s\n rules:\n - alert: SomethingIsUp\n expr: up == 1\n", + }, + FormatVersion: configs.RuleFormatV2, + }, + }, + }, + }} + assert.Equal(t, &expected, resp) +} diff --git a/pkg/configs/storage/clients/gcp/config_client.go b/pkg/storage/clients/gcp/config_client.go similarity index 68% rename from pkg/configs/storage/clients/gcp/config_client.go rename to pkg/storage/clients/gcp/config_client.go index 9bb4e833cf0..da1d4d5ad17 100644 --- a/pkg/configs/storage/clients/gcp/config_client.go +++ b/pkg/storage/clients/gcp/config_client.go @@ -9,11 +9,12 @@ import ( "strings" "time" - "github.com/cortexproject/cortex/pkg/configs" + "github.com/cortexproject/cortex/pkg/alertmanager" + "github.com/cortexproject/cortex/pkg/ruler" "github.com/cortexproject/cortex/pkg/ruler/group" "github.com/cortexproject/cortex/pkg/util" - "cloud.google.com/go/storage" + gstorage "cloud.google.com/go/storage" "github.com/go-kit/kit/log/level" "github.com/golang/protobuf/proto" "github.com/prometheus/prometheus/pkg/rulefmt" @@ -39,64 +40,70 @@ func (cfg *GCSConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.StringVar(&cfg.BucketName, prefix+"gcs.bucketname", "", "Name of GCS bucket to put chunks in.") } -// gcsConfigClient acts as a config backend. It is not safe to use concurrently when polling for rules. +// GCSClient acts as a config backend. It is not safe to use concurrently when polling for rules. // This is not an issue with the current scheduler architecture, but must be noted. -type gcsConfigClient struct { - client *storage.Client - bucket *storage.BucketHandle +type GCSClient struct { + client *gstorage.Client + bucket *gstorage.BucketHandle alertPolled time.Time rulePolled time.Time } -// NewGCSConfigClient makes a new chunk.ObjectClient that writes chunks to GCS. -func NewGCSConfigClient(ctx context.Context, cfg GCSConfig) (configs.ConfigStore, error) { - client, err := storage.NewClient(ctx) +// NewGCSClient makes a new chunk.ObjectClient that writes chunks to GCS. +func NewGCSClient(ctx context.Context, cfg GCSConfig) (*GCSClient, error) { + client, err := gstorage.NewClient(ctx) if err != nil { return nil, err } + + return newGCSClient(cfg, client), nil +} + +// newGCSClient makes a new chunk.ObjectClient that writes chunks to GCS. +func newGCSClient(cfg GCSConfig, client *gstorage.Client) *GCSClient { bucket := client.Bucket(cfg.BucketName) - return &gcsConfigClient{ + return &GCSClient{ client: client, bucket: bucket, alertPolled: time.Unix(0, 0), rulePolled: time.Unix(0, 0), - }, nil + } } -func (g *gcsConfigClient) getAlertConfig(ctx context.Context, obj string) (configs.AlertConfig, error) { +func (g *GCSClient) getAlertConfig(ctx context.Context, obj string) (alertmanager.AlertConfig, error) { reader, err := g.bucket.Object(obj).NewReader(ctx) - if err == storage.ErrObjectNotExist { + if err == gstorage.ErrObjectNotExist { level.Debug(util.Logger).Log("msg", "object does not exist", "name", obj) - return configs.AlertConfig{}, nil + return alertmanager.AlertConfig{}, nil } if err != nil { - return configs.AlertConfig{}, err + return alertmanager.AlertConfig{}, err } defer reader.Close() buf, err := ioutil.ReadAll(reader) if err != nil { - return configs.AlertConfig{}, err + return alertmanager.AlertConfig{}, err } - config := configs.AlertConfig{} + config := alertmanager.AlertConfig{} err = json.Unmarshal(buf, &config) if err != nil { - return configs.AlertConfig{}, err + return alertmanager.AlertConfig{}, err } return config, nil } // PollAlertConfigs returns any recently updated alert configurations -func (g *gcsConfigClient) PollAlertConfigs(ctx context.Context) (map[string]configs.AlertConfig, error) { - objs := g.bucket.Objects(ctx, &storage.Query{ +func (g *GCSClient) PollAlertConfigs(ctx context.Context) (map[string]alertmanager.AlertConfig, error) { + objs := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: alertPrefix, }) - alertMap := map[string]configs.AlertConfig{} + alertMap := map[string]alertmanager.AlertConfig{} for { objAttrs, err := objs.Next() if err == iterator.Done { @@ -123,12 +130,12 @@ func (g *gcsConfigClient) PollAlertConfigs(ctx context.Context) (map[string]conf } // GetAlertConfig returns a specified users alertmanager configuration -func (g *gcsConfigClient) GetAlertConfig(ctx context.Context, userID string) (configs.AlertConfig, error) { +func (g *GCSClient) GetAlertConfig(ctx context.Context, userID string) (alertmanager.AlertConfig, error) { return g.getAlertConfig(ctx, alertPrefix+userID) } // SetAlertConfig sets a specified users alertmanager configuration -func (g *gcsConfigClient) SetAlertConfig(ctx context.Context, userID string, cfg configs.AlertConfig) error { +func (g *GCSClient) SetAlertConfig(ctx context.Context, userID string, cfg alertmanager.AlertConfig) error { cfgBytes, err := json.Marshal(cfg) if err != nil { return err @@ -149,7 +156,7 @@ func (g *gcsConfigClient) SetAlertConfig(ctx context.Context, userID string, cfg } // DeleteAlertConfig deletes a specified users alertmanager configuration -func (g *gcsConfigClient) DeleteAlertConfig(ctx context.Context, userID string) error { +func (g *GCSClient) DeleteAlertConfig(ctx context.Context, userID string) error { err := g.bucket.Object(alertPrefix + userID).Delete(ctx) if err != nil { return err @@ -158,8 +165,8 @@ func (g *gcsConfigClient) DeleteAlertConfig(ctx context.Context, userID string) } // PollRules returns a users recently updated rule set -func (g *gcsConfigClient) PollRules(ctx context.Context) (map[string][]configs.RuleGroup, error) { - objs := g.bucket.Objects(ctx, &storage.Query{ +func (g *GCSClient) PollRules(ctx context.Context) (map[string][]ruler.RuleGroup, error) { + objs := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: rulePrefix, }) @@ -191,7 +198,7 @@ func (g *gcsConfigClient) PollRules(ctx context.Context) (map[string][]configs.R } } - ruleMap := map[string][]configs.RuleGroup{} + ruleMap := map[string][]ruler.RuleGroup{} for _, user := range updatedUsers { rgs, err := g.getAllRuleGroups(ctx, user) @@ -206,8 +213,8 @@ func (g *gcsConfigClient) PollRules(ctx context.Context) (map[string][]configs.R return ruleMap, nil } -func (g *gcsConfigClient) checkUser(ctx context.Context, userID string) (bool, error) { - objs := g.bucket.Objects(ctx, &storage.Query{ +func (g *GCSClient) checkUser(ctx context.Context, userID string) (bool, error) { + objs := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: generateRuleHandle(userID, "", ""), }) @@ -231,12 +238,12 @@ func (g *gcsConfigClient) checkUser(ctx context.Context, userID string) (bool, e return false, nil } -func (g *gcsConfigClient) getAllRuleGroups(ctx context.Context, userID string) ([]configs.RuleGroup, error) { - it := g.bucket.Objects(ctx, &storage.Query{ +func (g *GCSClient) getAllRuleGroups(ctx context.Context, userID string) ([]ruler.RuleGroup, error) { + it := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: generateRuleHandle(userID, "", ""), }) - rgs := []configs.RuleGroup{} + rgs := []ruler.RuleGroup{} for { obj, err := it.Next() @@ -245,17 +252,17 @@ func (g *gcsConfigClient) getAllRuleGroups(ctx context.Context, userID string) ( } if err != nil { - return []configs.RuleGroup{}, err + return []ruler.RuleGroup{}, err } rgProto, err := g.getRuleGroup(ctx, obj.Name) if err != nil { - return []configs.RuleGroup{}, err + return []ruler.RuleGroup{}, err } rg, err := group.GenerateRuleGroup(userID, rgProto) if err != nil { - return []configs.RuleGroup{}, err + return []ruler.RuleGroup{}, err } rgs = append(rgs, rg) @@ -264,8 +271,8 @@ func (g *gcsConfigClient) getAllRuleGroups(ctx context.Context, userID string) ( return rgs, nil } -func (g *gcsConfigClient) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) (map[string]configs.RuleNamespace, error) { - it := g.bucket.Objects(ctx, &storage.Query{ +func (g *GCSClient) ListRuleGroups(ctx context.Context, options ruler.RuleStoreConditions) (map[string]ruler.RuleNamespace, error) { + it := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: generateRuleHandle(options.UserID, options.Namespace, ""), }) @@ -291,7 +298,7 @@ func (g *gcsConfigClient) ListRuleGroups(ctx context.Context, options configs.Ru } } - nss := map[string]configs.RuleNamespace{} + nss := map[string]ruler.RuleNamespace{} for namespace := range namespaces { level.Debug(util.Logger).Log("msg", "retrieving rule namespace", "user", options.UserID, "namespace", namespace) @@ -305,12 +312,12 @@ func (g *gcsConfigClient) ListRuleGroups(ctx context.Context, options configs.Ru return nss, nil } -func (g *gcsConfigClient) getRuleNamespace(ctx context.Context, userID string, namespace string) (configs.RuleNamespace, error) { - it := g.bucket.Objects(ctx, &storage.Query{ +func (g *GCSClient) getRuleNamespace(ctx context.Context, userID string, namespace string) (ruler.RuleNamespace, error) { + it := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: generateRuleHandle(userID, namespace, ""), }) - ns := configs.RuleNamespace{ + ns := ruler.RuleNamespace{ Groups: []rulefmt.RuleGroup{}, } @@ -321,36 +328,36 @@ func (g *gcsConfigClient) getRuleNamespace(ctx context.Context, userID string, n } if err != nil { - return configs.RuleNamespace{}, err + return ruler.RuleNamespace{}, err } rg, err := g.getRuleGroup(ctx, obj.Name) if err != nil { - return configs.RuleNamespace{}, err + return ruler.RuleNamespace{}, err } - ns.Groups = append(ns.Groups, group.FromProto(rg)) + ns.Groups = append(ns.Groups, *group.FromProto(rg)) } return ns, nil } -func (g *gcsConfigClient) GetRuleGroup(ctx context.Context, userID string, namespace string, grp string) (rulefmt.RuleGroup, error) { +func (g *GCSClient) GetRuleGroup(ctx context.Context, userID string, namespace string, grp string) (*rulefmt.RuleGroup, error) { handle := generateRuleHandle(userID, namespace, grp) rg, err := g.getRuleGroup(ctx, handle) if err != nil { - return rulefmt.RuleGroup{}, err + return nil, err } if rg == nil { - return rulefmt.RuleGroup{}, configs.ErrGroupNotFound + return nil, ruler.ErrGroupNotFound } return group.FromProto(rg), nil } -func (g *gcsConfigClient) getRuleGroup(ctx context.Context, handle string) (*group.RuleGroup, error) { +func (g *GCSClient) getRuleGroup(ctx context.Context, handle string) (*group.RuleGroup, error) { reader, err := g.bucket.Object(handle).NewReader(ctx) - if err == storage.ErrObjectNotExist { + if err == gstorage.ErrObjectNotExist { level.Debug(util.Logger).Log("msg", "rule group does not exist", "name", handle) return nil, nil } @@ -374,7 +381,7 @@ func (g *gcsConfigClient) getRuleGroup(ctx context.Context, handle string) (*gro return rg, nil } -func (g *gcsConfigClient) SetRuleGroup(ctx context.Context, userID string, namespace string, grp rulefmt.RuleGroup) error { +func (g *GCSClient) SetRuleGroup(ctx context.Context, userID string, namespace string, grp rulefmt.RuleGroup) error { rg := group.ToProto(namespace, grp) rgBytes, err := proto.Marshal(&rg) if err != nil { @@ -396,7 +403,7 @@ func (g *gcsConfigClient) SetRuleGroup(ctx context.Context, userID string, names return nil } -func (g *gcsConfigClient) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { +func (g *GCSClient) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { handle := generateRuleHandle(userID, namespace, group) err := g.bucket.Object(handle).Delete(ctx) if err != nil { diff --git a/pkg/configs/storage/clients/gcp/fixtures.go b/pkg/storage/clients/gcp/fixtures.go similarity index 65% rename from pkg/configs/storage/clients/gcp/fixtures.go rename to pkg/storage/clients/gcp/fixtures.go index d18a66677f0..bb5462d16f4 100644 --- a/pkg/configs/storage/clients/gcp/fixtures.go +++ b/pkg/storage/clients/gcp/fixtures.go @@ -1,12 +1,10 @@ package gcp import ( - "context" - + "github.com/cortexproject/cortex/pkg/alertmanager" + "github.com/cortexproject/cortex/pkg/ruler" + "github.com/cortexproject/cortex/pkg/storage/testutils" "github.com/fsouza/fake-gcs-server/fakestorage" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/configs/storage/testutils" ) const ( @@ -23,13 +21,14 @@ func (f *fixture) Name() string { return f.name } -func (f *fixture) Clients() (store configs.ConfigStore, err error) { +func (f *fixture) Clients() (alertmanager.AlertStore, ruler.RuleStore, error) { f.gcssrv = fakestorage.NewServer(nil) f.gcssrv.CreateBucket("configdb") - - return NewGCSConfigClient(context.Background(), GCSConfig{ + cli := newGCSClient(GCSConfig{ BucketName: "configdb", - }) + }, f.gcssrv.Client()) + + return cli, cli, nil } func (f *fixture) Teardown() error { diff --git a/pkg/storage/factory.go b/pkg/storage/factory.go new file mode 100644 index 00000000000..0cd49da11cc --- /dev/null +++ b/pkg/storage/factory.go @@ -0,0 +1,75 @@ +package storage + +import ( + "context" + "flag" + "fmt" + + "github.com/cortexproject/cortex/pkg/alertmanager" + "github.com/cortexproject/cortex/pkg/ruler" + "github.com/cortexproject/cortex/pkg/storage/clients/configdb" + "github.com/cortexproject/cortex/pkg/storage/clients/gcp" +) + +// Config is used to configure the alert and rule store config clients +type Config struct { + AlertStoreConfig ClientConfig + RuleStoreConfig ClientConfig +} + +// RegisterFlags registers flags. +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + cfg.AlertStoreConfig.RegisterFlagsWithPrefix("alertmanager", f) + cfg.RuleStoreConfig.RegisterFlagsWithPrefix("ruler", f) +} + +// ClientConfig is used to config an config client +type ClientConfig struct { + BackendType string + + ConfigDB configdb.Config + GCS gcp.GCSConfig +} + +// RegisterFlagsWithPrefix registers flags. +func (cfg *ClientConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.StringVar(&cfg.BackendType, prefix+".store.backend", "configdb", "backend to use for storing and retrieving alerts") + cfg.ConfigDB.RegisterFlagsWithPrefix(prefix, f) + cfg.GCS.RegisterFlagsWithPrefix(prefix+".store.", f) +} + +// NewRuleStore returns a rule store +func NewRuleStore(cfg Config) (ruler.RuleStore, error) { + var ( + store ruler.RuleStore + err error + ) + switch cfg.RuleStoreConfig.BackendType { + case "configdb": + store, err = configdb.New(cfg.RuleStoreConfig.ConfigDB) + case "gcp": + store, err = gcp.NewGCSClient(context.Background(), cfg.RuleStoreConfig.GCS) + default: + return nil, fmt.Errorf("Unrecognized rule store client %v, choose one of: client, gcp", cfg.RuleStoreConfig.BackendType) + } + + return store, err +} + +// NewAlertStore returns a alert store +func NewAlertStore(cfg Config) (alertmanager.AlertStore, error) { + var ( + store alertmanager.AlertStore + err error + ) + switch cfg.AlertStoreConfig.BackendType { + case "configdb": + store, err = configdb.New(cfg.AlertStoreConfig.ConfigDB) + case "gcp": + store, err = gcp.NewGCSClient(context.Background(), cfg.AlertStoreConfig.GCS) + default: + return nil, fmt.Errorf("Unrecognized config storage client %v, choose one of: client, gcp", cfg.AlertStoreConfig.BackendType) + } + + return store, err +} diff --git a/pkg/storage/testutils/testutils.go b/pkg/storage/testutils/testutils.go new file mode 100644 index 00000000000..edb54f7e7ce --- /dev/null +++ b/pkg/storage/testutils/testutils.go @@ -0,0 +1,13 @@ +package testutils + +import ( + "github.com/cortexproject/cortex/pkg/alertmanager" + "github.com/cortexproject/cortex/pkg/ruler" +) + +// Fixture type for per-backend testing. +type Fixture interface { + Name() string + Clients() (alertmanager.AlertStore, ruler.RuleStore, error) + Teardown() error +} From 03e722d350fa90ba8a7bd0f3e47c39fdce78dc73 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Thu, 11 Jul 2019 20:46:21 -0400 Subject: [PATCH 36/60] update poller to return storage Signed-off-by: Jacob Lisi --- pkg/alertmanager/alert_store.go | 8 +++-- pkg/alertmanager/multitenant.go | 4 +-- pkg/cortex/modules.go | 12 +++---- pkg/ruler/rule_store.go | 10 +++++- pkg/ruler/ruler.go | 4 +-- pkg/ruler/ruler_test.go | 5 +++ pkg/ruler/scheduler.go | 4 +-- pkg/storage/clients/configdb/client.go | 40 ++++++------------------ pkg/storage/clients/gcp/config_client.go | 10 ++++++ pkg/storage/factory.go | 8 ++--- 10 files changed, 56 insertions(+), 49 deletions(-) diff --git a/pkg/alertmanager/alert_store.go b/pkg/alertmanager/alert_store.go index 0a674b04b26..e8feec8bcd6 100644 --- a/pkg/alertmanager/alert_store.go +++ b/pkg/alertmanager/alert_store.go @@ -10,10 +10,14 @@ type AlertConfig struct { AlertmanagerConfig string `json:"alertmanager_config"` } -// AlertStore stores config information and template files to configure alertmanager tenants -type AlertStore interface { +// AlertPoller polls for updated alerts +type AlertPoller interface { PollAlertConfigs(ctx context.Context) (map[string]AlertConfig, error) + AlertStore() AlertStore +} +// AlertStore stores config information and template files to configure alertmanager tenants +type AlertStore interface { GetAlertConfig(ctx context.Context, id string) (AlertConfig, error) SetAlertConfig(ctx context.Context, id string, cfg AlertConfig) error DeleteAlertConfig(ctx context.Context, id string) error diff --git a/pkg/alertmanager/multitenant.go b/pkg/alertmanager/multitenant.go index 7a63efba0b5..fcc9f6cb2b3 100644 --- a/pkg/alertmanager/multitenant.go +++ b/pkg/alertmanager/multitenant.go @@ -212,7 +212,7 @@ func (cfg *MultitenantAlertmanagerConfig) RegisterFlags(f *flag.FlagSet) { type MultitenantAlertmanager struct { cfg *MultitenantAlertmanagerConfig - store AlertStore + store AlertPoller // The fallback config is stored as a string and parsed every time it's needed // because we mutate the parsed results and don't want those changes to take @@ -233,7 +233,7 @@ type MultitenantAlertmanager struct { } // NewMultitenantAlertmanager creates a new MultitenantAlertmanager. -func NewMultitenantAlertmanager(cfg *MultitenantAlertmanagerConfig, store AlertStore) (*MultitenantAlertmanager, error) { +func NewMultitenantAlertmanager(cfg *MultitenantAlertmanagerConfig, store AlertPoller) (*MultitenantAlertmanager, error) { err := os.MkdirAll(cfg.DataDir, 0777) if err != nil { return nil, fmt.Errorf("unable to create Alertmanager data directory %q: %s", cfg.DataDir, err) diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 5fb727040be..28bd0ee0ca9 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -320,12 +320,12 @@ func (t *Cortex) initRuler(cfg *Config) (err error) { cfg.Ruler.LifecyclerConfig.ListenPort = &cfg.Server.GRPCListenPort queryable, engine := querier.New(cfg.Querier, t.distributor, t.store) - store, err := config_storage.NewRuleStore(cfg.ConfigStore) + poller, err := config_storage.NewRuleStore(cfg.ConfigStore) if err != nil { return err } - t.ruler, err = ruler.NewRuler(cfg.Ruler, engine, queryable, t.distributor, store) + t.ruler, err = ruler.NewRuler(cfg.Ruler, engine, queryable, t.distributor, poller) if err != nil { return } @@ -334,7 +334,7 @@ func (t *Cortex) initRuler(cfg *Config) (err error) { // serving configs from the configs API. Allows for smoother // migration. See https://github.com/cortexproject/cortex/issues/619 if cfg.ConfigStore.RuleStoreConfig.BackendType != "configdb" { - a := ruler.NewAPI(store) + a := ruler.NewAPI(poller.RuleStore()) a.RegisterRoutes(t.server.HTTP) } @@ -364,12 +364,12 @@ func (t *Cortex) stopConfigs() error { } func (t *Cortex) initAlertmanager(cfg *Config) (err error) { - store, err := config_storage.NewAlertStore(cfg.ConfigStore) + poller, err := config_storage.NewAlertStore(cfg.ConfigStore) if err != nil { return err } - t.alertmanager, err = alertmanager.NewMultitenantAlertmanager(&cfg.Alertmanager, store) + t.alertmanager, err = alertmanager.NewMultitenantAlertmanager(&cfg.Alertmanager, poller) if err != nil { return } @@ -378,7 +378,7 @@ func (t *Cortex) initAlertmanager(cfg *Config) (err error) { // serving configs from the configs API. Allows for smoother // migration. See https://github.com/cortexproject/cortex/issues/619 if cfg.ConfigStore.AlertStoreConfig.BackendType != "configdb" { - a := alertmanager.NewAPI(store) + a := alertmanager.NewAPI(poller.AlertStore()) a.RegisterRoutes(t.server.HTTP) } diff --git a/pkg/ruler/rule_store.go b/pkg/ruler/rule_store.go index a728e0b6858..3822613bee5 100644 --- a/pkg/ruler/rule_store.go +++ b/pkg/ruler/rule_store.go @@ -25,9 +25,17 @@ type RuleStoreConditions struct { Namespace string } +type RulePoller interface { + PollRules(ctx context.Context) (map[string][]RuleGroup, error) + + // RuleStore returns the rule store client used by the poller, this allows a Poller + // to be used for scheduling, and an associated rule store to be used by the API. + RuleStore() RuleStore +} + // RuleStore is used to store and retrieve rules type RuleStore interface { - PollRules(ctx context.Context) (map[string][]RuleGroup, error) + RulePoller ListRuleGroups(ctx context.Context, options RuleStoreConditions) (map[string]RuleNamespace, error) GetRuleGroup(ctx context.Context, userID, namespace, group string) (*rulefmt.RuleGroup, error) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 5a9a7774755..ed0e3b0cd46 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -134,7 +134,7 @@ type Ruler struct { } // NewRuler creates a new ruler from a distributor and chunk store. -func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d *distributor.Distributor, store RuleStore) (*Ruler, error) { +func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d *distributor.Distributor, poller RulePoller) (*Ruler, error) { if cfg.NumWorkers <= 0 { return nil, fmt.Errorf("must have at least 1 worker, got %d", cfg.NumWorkers) } @@ -155,7 +155,7 @@ func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d workerWG: &sync.WaitGroup{}, } - ruler.scheduler = newScheduler(store, cfg.EvaluationInterval, cfg.EvaluationInterval, ruler.newGroup) + ruler.scheduler = newScheduler(poller, cfg.EvaluationInterval, cfg.EvaluationInterval, ruler.newGroup) // If sharding is enabled, create/join a ring to distribute tokens to // the ruler diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index a194af7fa6b..13e383487f1 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -34,6 +34,11 @@ func (m *mockRuleStore) PollRules(ctx context.Context) (map[string][]RuleGroup, return pollPayload, nil } +// RuleStore returns an RuleStore from the client +func (m *mockRuleStore) RuleStore() RuleStore { + return m +} + func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options RuleStoreConditions) (map[string]RuleNamespace, error) { groupPrefix := userID + ":" diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index 973f4fc7906..8543cfb3ac0 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -77,7 +77,7 @@ type userConfig struct { type groupFactory func(context.Context, RuleGroup) (*wrappedGroup, error) type scheduler struct { - store RuleStore + store RulePoller evaluationInterval time.Duration // how often we re-evaluate each rule set q *SchedulingQueue @@ -90,7 +90,7 @@ type scheduler struct { } // newScheduler makes a new scheduler. -func newScheduler(store RuleStore, evaluationInterval, pollInterval time.Duration, groupFn groupFactory) *scheduler { +func newScheduler(store RulePoller, evaluationInterval, pollInterval time.Duration, groupFn groupFactory) *scheduler { return &scheduler{ store: store, evaluationInterval: evaluationInterval, diff --git a/pkg/storage/clients/configdb/client.go b/pkg/storage/clients/configdb/client.go index ed3dc39689f..f8c7d5c4ecd 100644 --- a/pkg/storage/clients/configdb/client.go +++ b/pkg/storage/clients/configdb/client.go @@ -3,7 +3,6 @@ package configdb import ( "context" "encoding/json" - "errors" "fmt" "net/http" "net/url" @@ -16,7 +15,6 @@ import ( "github.com/cortexproject/cortex/pkg/ruler/group" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" - "github.com/prometheus/prometheus/pkg/rulefmt" ) // ConfigsClient allows retrieving recording and alerting rules from the configs server. @@ -166,37 +164,19 @@ func (c *ConfigsClient) PollRules(ctx context.Context) (map[string][]ruler.RuleG return newRules, nil } +// AlertStore returns an AlertStore from the client +func (c *ConfigsClient) AlertStore() alertmanager.AlertStore { + return nil +} + +// RuleStore returns an RuleStore from the client +func (c *ConfigsClient) RuleStore() ruler.RuleStore { + return nil +} + // decomposeGroupSlug breaks the group slug from Parse // into it's group name and file name func decomposeGroupSlug(slug string) (string, string) { components := strings.Split(slug, ";") return components[0], components[1] } - -func (c *ConfigsClient) GetAlertConfig(ctx context.Context, userID string) (alertmanager.AlertConfig, error) { - return alertmanager.AlertConfig{}, errors.New("remote configdb client does not implement GetAlertConfig") -} - -func (c *ConfigsClient) SetAlertConfig(ctx context.Context, userID string, config alertmanager.AlertConfig) error { - return errors.New("remote configdb client does not implement SetAlertConfig") -} - -func (c *ConfigsClient) DeleteAlertConfig(ctx context.Context, userID string) error { - return errors.New("remote configdb client does not implement DeleteAlertConfig") -} - -func (c *ConfigsClient) ListRuleGroups(ctx context.Context, options ruler.RuleStoreConditions) (map[string]ruler.RuleNamespace, error) { - return nil, errors.New("remote configdb client does not implement ListRule") -} - -func (c *ConfigsClient) GetRuleGroup(ctx context.Context, userID, namespace, group string) (*rulefmt.RuleGroup, error) { - return nil, errors.New("remote configdb client does not implement GetRuleGroup") -} - -func (c *ConfigsClient) SetRuleGroup(ctx context.Context, userID, namespace string, group rulefmt.RuleGroup) error { - return errors.New("remote configdb client does not implement SetRuleGroup") -} - -func (c *ConfigsClient) DeleteRuleGroup(ctx context.Context, userID, namespace string, group string) error { - return errors.New("remote configdb client does not implement DeleteRuleGroup") -} diff --git a/pkg/storage/clients/gcp/config_client.go b/pkg/storage/clients/gcp/config_client.go index da1d4d5ad17..c6250e1b652 100644 --- a/pkg/storage/clients/gcp/config_client.go +++ b/pkg/storage/clients/gcp/config_client.go @@ -129,6 +129,11 @@ func (g *GCSClient) PollAlertConfigs(ctx context.Context) (map[string]alertmanag return alertMap, nil } +// AlertStore returns an AlertStore from the client +func (g *GCSClient) AlertStore() alertmanager.AlertStore { + return g +} + // GetAlertConfig returns a specified users alertmanager configuration func (g *GCSClient) GetAlertConfig(ctx context.Context, userID string) (alertmanager.AlertConfig, error) { return g.getAlertConfig(ctx, alertPrefix+userID) @@ -213,6 +218,11 @@ func (g *GCSClient) PollRules(ctx context.Context) (map[string][]ruler.RuleGroup return ruleMap, nil } +// RuleStore returns an RuleStore from the client +func (g *GCSClient) RuleStore() ruler.RuleStore { + return g +} + func (g *GCSClient) checkUser(ctx context.Context, userID string) (bool, error) { objs := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: generateRuleHandle(userID, "", ""), diff --git a/pkg/storage/factory.go b/pkg/storage/factory.go index 0cd49da11cc..850f66056bf 100644 --- a/pkg/storage/factory.go +++ b/pkg/storage/factory.go @@ -39,9 +39,9 @@ func (cfg *ClientConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) } // NewRuleStore returns a rule store -func NewRuleStore(cfg Config) (ruler.RuleStore, error) { +func NewRuleStore(cfg Config) (ruler.RulePoller, error) { var ( - store ruler.RuleStore + store ruler.RulePoller err error ) switch cfg.RuleStoreConfig.BackendType { @@ -57,9 +57,9 @@ func NewRuleStore(cfg Config) (ruler.RuleStore, error) { } // NewAlertStore returns a alert store -func NewAlertStore(cfg Config) (alertmanager.AlertStore, error) { +func NewAlertStore(cfg Config) (alertmanager.AlertPoller, error) { var ( - store alertmanager.AlertStore + store alertmanager.AlertPoller err error ) switch cfg.AlertStoreConfig.BackendType { From 109f65f83f100d054827e68571a5565715312f1f Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Thu, 11 Jul 2019 21:04:30 -0400 Subject: [PATCH 37/60] fix client tests Signed-off-by: Jacob Lisi --- pkg/storage/clients/client_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/storage/clients/client_test.go b/pkg/storage/clients/client_test.go index b4039b7bba9..df3e8568ffa 100644 --- a/pkg/storage/clients/client_test.go +++ b/pkg/storage/clients/client_test.go @@ -5,7 +5,8 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/pkg/storage" + "github.com/cortexproject/cortex/pkg/alertmanager" + "github.com/cortexproject/cortex/pkg/ruler" "github.com/cortexproject/cortex/pkg/storage/clients/gcp" "github.com/cortexproject/cortex/pkg/storage/testutils" "github.com/prometheus/prometheus/pkg/rulefmt" @@ -25,7 +26,7 @@ var ( ) func TestRuleStoreBasic(t *testing.T) { - forAllFixtures(t, func(t *testing.T, _ storage.AlertStore, client storage.RuleStore) { + forAllFixtures(t, func(t *testing.T, _ alertmanager.AlertStore, client ruler.RuleStore) { const batchSize = 5 ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -46,7 +47,7 @@ func TestRuleStoreBasic(t *testing.T) { }) } -func forAllFixtures(t *testing.T, clientTest func(*testing.T, storage.AlertStore, storage.RuleStore)) { +func forAllFixtures(t *testing.T, clientTest func(*testing.T, alertmanager.AlertStore, ruler.RuleStore)) { var fixtures []testutils.Fixture fixtures = append(fixtures, gcp.Fixtures...) From f4cc7336dc5151ebbf944ee051c4e50cbc0e4b91 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 12 Jul 2019 12:13:51 -0400 Subject: [PATCH 38/60] remove unused config client Signed-off-by: Jacob Lisi --- pkg/configs/client/client.go | 165 ------------------------- pkg/configs/client/config.go | 70 ----------- pkg/configs/client/configs_test.go | 54 -------- pkg/storage/clients/configdb/config.go | 8 -- 4 files changed, 297 deletions(-) delete mode 100644 pkg/configs/client/client.go delete mode 100644 pkg/configs/client/config.go delete mode 100644 pkg/configs/client/configs_test.go diff --git a/pkg/configs/client/client.go b/pkg/configs/client/client.go deleted file mode 100644 index a5528b29ecc..00000000000 --- a/pkg/configs/client/client.go +++ /dev/null @@ -1,165 +0,0 @@ -package client - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/url" - "time" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/configs/db" - "github.com/cortexproject/cortex/pkg/util" - "github.com/go-kit/kit/log/level" -) - -// Client is what the ruler and altermanger needs from a config store to process rules. -type Client interface { - // GetRules returns all Cortex configurations from a configs API server - // that have been updated after the given configs.ID was last updated. - GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) - - // GetAlerts fetches all the alerts that have changes since since. - GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) -} - -// New creates a new ConfigClient. -func New(cfg Config) (Client, error) { - // All of this falderal is to allow for a smooth transition away from - // using the configs server and toward directly connecting to the database. - // See https://github.com/cortexproject/cortex/issues/619 - if cfg.ConfigsAPIURL.URL != nil { - return instrumented{ - next: configsClient{ - URL: cfg.ConfigsAPIURL.URL, - Timeout: cfg.ClientTimeout, - }, - }, nil - } - - db, err := db.New(cfg.DBConfig) - if err != nil { - return nil, err - } - return instrumented{ - next: dbStore{ - db: db, - }, - }, nil -} - -// configsClient allows retrieving recording and alerting rules from the configs server. -type configsClient struct { - URL *url.URL - Timeout time.Duration -} - -// GetRules implements ConfigClient. -func (c configsClient) GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { - suffix := "" - if since != 0 { - suffix = fmt.Sprintf("?since=%d", since) - } - endpoint := fmt.Sprintf("%s/private/api/prom/configs/rules%s", c.URL.String(), suffix) - response, err := doRequest(endpoint, c.Timeout, since) - if err != nil { - return nil, err - } - configs := map[string]configs.VersionedRulesConfig{} - for id, view := range response.Configs { - cfg := view.GetVersionedRulesConfig() - if cfg != nil { - configs[id] = *cfg - } - } - return configs, nil -} - -// GetAlerts implements ConfigClient. -func (c configsClient) GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) { - suffix := "" - if since != 0 { - suffix = fmt.Sprintf("?since=%d", since) - } - endpoint := fmt.Sprintf("%s/private/api/prom/configs/alertmanager%s", c.URL.String(), suffix) - return doRequest(endpoint, c.Timeout, since) -} - -func doRequest(endpoint string, timeout time.Duration, since configs.ID) (*ConfigsResponse, error) { - req, err := http.NewRequest("GET", endpoint, nil) - if err != nil { - return nil, err - } - - client := &http.Client{Timeout: timeout} - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("Invalid response from configs server: %v", resp.StatusCode) - } - - var config ConfigsResponse - if err := json.NewDecoder(resp.Body).Decode(&config); err != nil { - level.Error(util.Logger).Log("msg", "configs: couldn't decode JSON body", "err", err) - return nil, err - } - - config.since = since - return &config, nil -} - -type dbStore struct { - db db.DB -} - -// GetRules implements ConfigClient. -func (d dbStore) GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { - if since == 0 { - return d.db.GetAllRulesConfigs(ctx) - } - return d.db.GetRulesConfigs(ctx, since) -} - -// GetAlerts implements ConfigClient. -func (d dbStore) GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) { - var resp map[string]configs.View - var err error - if since == 0 { - resp, err = d.db.GetAllConfigs(ctx) - - } - resp, err = d.db.GetConfigs(ctx, since) - if err != nil { - return nil, err - } - - return &ConfigsResponse{ - since: since, - Configs: resp, - }, nil -} - -// ConfigsResponse is a response from server for GetConfigs. -type ConfigsResponse struct { - // The version since which these configs were changed - since configs.ID - - // Configs maps user ID to their latest configs.View. - Configs map[string]configs.View `json:"configs"` -} - -// GetLatestConfigID returns the last config ID from a set of configs. -func (c ConfigsResponse) GetLatestConfigID() configs.ID { - latest := c.since - for _, config := range c.Configs { - if config.ID > latest { - latest = config.ID - } - } - return latest -} diff --git a/pkg/configs/client/config.go b/pkg/configs/client/config.go deleted file mode 100644 index 3850b967c68..00000000000 --- a/pkg/configs/client/config.go +++ /dev/null @@ -1,70 +0,0 @@ -package client - -import ( - "context" - "flag" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/weaveworks/common/instrument" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/configs/db" - "github.com/cortexproject/cortex/pkg/util/flagext" -) - -// Config says where we can find the ruler configs. -type Config struct { - DBConfig db.Config - - // DEPRECATED - ConfigsAPIURL flagext.URLValue - - // DEPRECATED. HTTP timeout duration for requests made to the Weave Cloud - // configs service. - ClientTimeout time.Duration -} - -// RegisterFlags adds the flags required to config this to the given FlagSet -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - cfg.DBConfig.RegisterFlags(f) - f.Var(&cfg.ConfigsAPIURL, "ruler.configs.url", "DEPRECATED. URL of configs API server.") - f.DurationVar(&cfg.ClientTimeout, "ruler.client-timeout", 5*time.Second, "DEPRECATED. Timeout for requests to Weave Cloud configs service.") - flag.Var(&cfg.ConfigsAPIURL, "alertmanager.configs.url", "URL of configs API server.") - flag.DurationVar(&cfg.ClientTimeout, "alertmanager.configs.client-timeout", 5*time.Second, "Timeout for requests to Weave Cloud configs service.") -} - -var configsRequestDuration = instrument.NewHistogramCollector(prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "cortex", - Name: "configs_request_duration_seconds", - Help: "Time spent requesting configs.", - Buckets: prometheus.DefBuckets, -}, []string{"operation", "status_code"})) - -func init() { - configsRequestDuration.Register() -} - -type instrumented struct { - next Client -} - -func (i instrumented) GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { - var cfgs map[string]configs.VersionedRulesConfig - err := instrument.CollectedRequest(context.Background(), "Configs.GetConfigs", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - cfgs, err = i.next.GetRules(ctx, since) // Warning: this will produce an incorrect result if the configID ever overflows - return err - }) - return cfgs, err -} - -func (i instrumented) GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) { - var cfgs *ConfigsResponse - err := instrument.CollectedRequest(context.Background(), "Configs.GetConfigs", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - cfgs, err = i.next.GetAlerts(ctx, since) // Warning: this will produce an incorrect result if the configID ever overflows - return err - }) - return cfgs, err -} diff --git a/pkg/configs/client/configs_test.go b/pkg/configs/client/configs_test.go deleted file mode 100644 index af7acf25b45..00000000000 --- a/pkg/configs/client/configs_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package client - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/stretchr/testify/assert" -) - -var response = `{ - "configs": { - "2": { - "id": 1, - "config": { - "rules_files": { - "recording.rules": "groups:\n- name: demo-service-alerts\n interval: 15s\n rules:\n - alert: SomethingIsUp\n expr: up == 1\n" - }, - "rule_format_version": "2" - } - } - } -} -` - -func TestDoRequest(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, err := w.Write([]byte(response)) - require.NoError(t, err) - })) - defer server.Close() - - resp, err := doRequest(server.URL, 1*time.Second, 0) - assert.Nil(t, err) - - expected := ConfigsResponse{Configs: map[string]configs.View{ - "2": { - ID: 1, - Config: configs.Config{ - RulesConfig: configs.RulesConfig{ - Files: map[string]string{ - "recording.rules": "groups:\n- name: demo-service-alerts\n interval: 15s\n rules:\n - alert: SomethingIsUp\n expr: up == 1\n", - }, - FormatVersion: configs.RuleFormatV2, - }, - }, - }, - }} - assert.Equal(t, &expected, resp) -} diff --git a/pkg/storage/clients/configdb/config.go b/pkg/storage/clients/configdb/config.go index c344c15f4ca..1dc12e81bff 100644 --- a/pkg/storage/clients/configdb/config.go +++ b/pkg/storage/clients/configdb/config.go @@ -4,25 +4,17 @@ import ( "flag" "time" - "github.com/cortexproject/cortex/pkg/configs/db" "github.com/cortexproject/cortex/pkg/util/flagext" ) // Config says where we can find the ruler configs. type Config struct { - DBConfig db.Config - - // DEPRECATED ConfigsAPIURL flagext.URLValue - - // DEPRECATED. HTTP timeout duration for requests made to the Weave Cloud - // configs service. ClientTimeout time.Duration } // RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { - cfg.DBConfig.RegisterFlags(f) f.Var(&cfg.ConfigsAPIURL, prefix+".configs.url", "DEPRECATED. URL of configs API server.") f.DurationVar(&cfg.ClientTimeout, prefix+".client-timeout", 5*time.Second, "DEPRECATED. Timeout for requests to Weave Cloud configs service.") } From ef8570431c560ed53c881795c40774d4193b2d7c Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 12 Jul 2019 12:29:45 -0400 Subject: [PATCH 39/60] refactor configdb client package Signed-off-by: Jacob Lisi --- pkg/storage/clients/configdb/client.go | 14 +++++++++++++ .../{configs_test.go => client_test.go} | 0 pkg/storage/clients/configdb/config.go | 20 ------------------- 3 files changed, 14 insertions(+), 20 deletions(-) rename pkg/storage/clients/configdb/{configs_test.go => client_test.go} (100%) delete mode 100644 pkg/storage/clients/configdb/config.go diff --git a/pkg/storage/clients/configdb/client.go b/pkg/storage/clients/configdb/client.go index f8c7d5c4ecd..f11c3c6f606 100644 --- a/pkg/storage/clients/configdb/client.go +++ b/pkg/storage/clients/configdb/client.go @@ -3,6 +3,7 @@ package configdb import ( "context" "encoding/json" + "flag" "fmt" "net/http" "net/url" @@ -14,9 +15,22 @@ import ( "github.com/cortexproject/cortex/pkg/ruler" "github.com/cortexproject/cortex/pkg/ruler/group" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/go-kit/kit/log/level" ) +// Config says where we can find the ruler configs. +type Config struct { + ConfigsAPIURL flagext.URLValue + ClientTimeout time.Duration +} + +// RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet +func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.Var(&cfg.ConfigsAPIURL, prefix+".configs.url", "DEPRECATED. URL of configs API server.") + f.DurationVar(&cfg.ClientTimeout, prefix+".client-timeout", 5*time.Second, "DEPRECATED. Timeout for requests to Weave Cloud configs service.") +} + // ConfigsClient allows retrieving recording and alerting rules from the configs server. type ConfigsClient struct { URL *url.URL diff --git a/pkg/storage/clients/configdb/configs_test.go b/pkg/storage/clients/configdb/client_test.go similarity index 100% rename from pkg/storage/clients/configdb/configs_test.go rename to pkg/storage/clients/configdb/client_test.go diff --git a/pkg/storage/clients/configdb/config.go b/pkg/storage/clients/configdb/config.go deleted file mode 100644 index 1dc12e81bff..00000000000 --- a/pkg/storage/clients/configdb/config.go +++ /dev/null @@ -1,20 +0,0 @@ -package configdb - -import ( - "flag" - "time" - - "github.com/cortexproject/cortex/pkg/util/flagext" -) - -// Config says where we can find the ruler configs. -type Config struct { - ConfigsAPIURL flagext.URLValue - ClientTimeout time.Duration -} - -// RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet -func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { - f.Var(&cfg.ConfigsAPIURL, prefix+".configs.url", "DEPRECATED. URL of configs API server.") - f.DurationVar(&cfg.ClientTimeout, prefix+".client-timeout", 5*time.Second, "DEPRECATED. Timeout for requests to Weave Cloud configs service.") -} From d0ebeef28dc5b893aeae421ec9acec170b2e821a Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 12 Jul 2019 19:00:10 -0400 Subject: [PATCH 40/60] add debug log line with rule eval latency Signed-off-by: Jacob Lisi --- pkg/ruler/worker.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/ruler/worker.go b/pkg/ruler/worker.go index e269876698d..331119ac23b 100644 --- a/pkg/ruler/worker.go +++ b/pkg/ruler/worker.go @@ -58,9 +58,10 @@ func (w *worker) Run() { level.Debug(util.Logger).Log("msg", "queue closed and empty; terminating worker") return } - evalLatency.Observe(time.Since(item.scheduled).Seconds()) + latency := time.Since(item.scheduled) + evalLatency.Observe(latency.Seconds()) workerIdleTime.Add(waitElapsed.Seconds()) - level.Debug(util.Logger).Log("msg", "processing item", "item", item) + level.Debug(util.Logger).Log("msg", "processing item", "item", item, "latency", latency.String()) w.ruler.Evaluate(item.userID, item) w.scheduler.workItemDone(*item) level.Debug(util.Logger).Log("msg", "item handed back to queue", "item", item) From ec7aabbe8052005e90a3ca2c9f655c0ba18db797 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 12 Jul 2019 19:43:54 -0400 Subject: [PATCH 41/60] update eval latency bucket sizes Signed-off-by: Jacob Lisi --- pkg/ruler/worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ruler/worker.go b/pkg/ruler/worker.go index 331119ac23b..215c09cc248 100644 --- a/pkg/ruler/worker.go +++ b/pkg/ruler/worker.go @@ -24,7 +24,7 @@ var ( Namespace: "cortex", Name: "group_evaluation_latency_seconds", Help: "How far behind the target time each rule group executed.", - Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 25}, + Buckets: []float64{.025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25, 60}, }) ) From 440fbb5f63e8a3c853f007d9ffc4a615a1a4663e Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Sat, 13 Jul 2019 20:38:14 -0400 Subject: [PATCH 42/60] update rule eval timestamp when iteration is missed Signed-off-by: Jacob Lisi --- pkg/ruler/scheduler.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index 8543cfb3ac0..ee5812eb41c 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -37,6 +37,11 @@ var ( Name: "scheduler_update_failures_total", Help: "Number of failures when updating rule groups", }) + iterationsMissed = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "cortex", + Name: "rule_group_iterations_missed_total", + Help: "The total number of rule group evaluations missed due to slow rule group evaluation.", + }, []string{"id"}) ) type workItem struct { @@ -59,11 +64,6 @@ func (w workItem) Scheduled() time.Time { return w.scheduled } -// Defer returns a work item with updated rules, rescheduled to a later time. -func (w workItem) Defer(interval time.Duration) workItem { - return workItem{w.userID, w.groupName, w.hash, w.group, w.scheduled.Add(interval), w.done} -} - func (w workItem) String() string { return fmt.Sprintf("%s:%s:%d@%s", w.userID, w.groupName, len(w.group.Rules()), w.scheduled.Format(timeLogFormat)) } @@ -258,8 +258,13 @@ func (s *scheduler) workItemDone(i workItem) { level.Debug(util.Logger).Log("msg", "scheduler: work item dropped", "item", i) return default: - next := i.Defer(s.evaluationInterval) - level.Debug(util.Logger).Log("msg", "scheduler: work item rescheduled", "item", i, "time", next.scheduled.Format(timeLogFormat)) - s.addWorkItem(next) + missed := (time.Since(i.scheduled) / s.evaluationInterval) - 1 + if missed > 0 { + level.Warn(util.Logger).Log("msg", "scheduler: work item missed evaluation", "item", i) + iterationsMissed.WithLabelValues(i.userID).Add(float64(missed)) + } + i.scheduled = i.scheduled.Add((missed + 1) * s.evaluationInterval) + level.Debug(util.Logger).Log("msg", "scheduler: work item rescheduled", "item", i, "time", i.scheduled.Format(timeLogFormat)) + s.addWorkItem(i) } } From ef0ec504eaa270517510988d0766ac87ddf68cb2 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Sun, 14 Jul 2019 17:37:04 -0400 Subject: [PATCH 43/60] reorganize scheduling instrumentation Signed-off-by: Jacob Lisi --- pkg/ruler/scheduler.go | 16 ++++++++++++++++ pkg/ruler/worker.go | 9 --------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index ee5812eb41c..cc33a4d841e 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -42,6 +42,12 @@ var ( Name: "rule_group_iterations_missed_total", Help: "The total number of rule group evaluations missed due to slow rule group evaluation.", }, []string{"id"}) + evalLatency = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "group_evaluation_latency_seconds", + Help: "How far behind the target time each rule group executed.", + Buckets: []float64{.025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25, 60}, + }) ) type workItem struct { @@ -247,6 +253,12 @@ func (s *scheduler) nextWorkItem() *workItem { } item := op.(workItem) level.Debug(util.Logger).Log("msg", "scheduler: work item granted", "item", item) + + // Record the latency of the items evaluation here + latency := time.Since(item.scheduled) + evalLatency.Observe(latency.Seconds()) + level.Debug(util.Logger).Log("msg", "sheduler: returning item", "item", item, "latency", latency.String()) + return &item } @@ -258,11 +270,15 @@ func (s *scheduler) workItemDone(i workItem) { level.Debug(util.Logger).Log("msg", "scheduler: work item dropped", "item", i) return default: + // If the evaluation of the item took longer than it's evaluation interval, skip to the next valid interval + // and record any evaluation misses. This must be differentiated from lateness due to scheduling which is + // caused by the overall workload, not the result of latency within a single rule group. missed := (time.Since(i.scheduled) / s.evaluationInterval) - 1 if missed > 0 { level.Warn(util.Logger).Log("msg", "scheduler: work item missed evaluation", "item", i) iterationsMissed.WithLabelValues(i.userID).Add(float64(missed)) } + i.scheduled = i.scheduled.Add((missed + 1) * s.evaluationInterval) level.Debug(util.Logger).Log("msg", "scheduler: work item rescheduled", "item", i, "time", i.scheduled.Format(timeLogFormat)) s.addWorkItem(i) diff --git a/pkg/ruler/worker.go b/pkg/ruler/worker.go index 215c09cc248..bbff15a9d84 100644 --- a/pkg/ruler/worker.go +++ b/pkg/ruler/worker.go @@ -20,12 +20,6 @@ var ( Name: "worker_idle_seconds_total", Help: "How long workers have spent waiting for work.", }) - evalLatency = promauto.NewHistogram(prometheus.HistogramOpts{ - Namespace: "cortex", - Name: "group_evaluation_latency_seconds", - Help: "How far behind the target time each rule group executed.", - Buckets: []float64{.025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25, 60}, - }) ) // Worker does a thing until it's told to stop. @@ -58,10 +52,7 @@ func (w *worker) Run() { level.Debug(util.Logger).Log("msg", "queue closed and empty; terminating worker") return } - latency := time.Since(item.scheduled) - evalLatency.Observe(latency.Seconds()) workerIdleTime.Add(waitElapsed.Seconds()) - level.Debug(util.Logger).Log("msg", "processing item", "item", item, "latency", latency.String()) w.ruler.Evaluate(item.userID, item) w.scheduler.workItemDone(*item) level.Debug(util.Logger).Log("msg", "item handed back to queue", "item", item) From e78e356785748d03fa5a1e2545f620ece36465cc Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Sun, 14 Jul 2019 17:44:17 -0400 Subject: [PATCH 44/60] increase size of eval duration buckets Signed-off-by: Jacob Lisi --- pkg/ruler/ruler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index ed0e3b0cd46..b62df403492 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -35,7 +35,7 @@ var ( Namespace: "cortex", Name: "group_evaluation_duration_seconds", Help: "The duration for a rule group to execute.", - Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 25}, + Buckets: []float64{.5, 1, 2.5, 5, 10, 25, 60, 120}, }) rulesProcessed = promauto.NewCounter(prometheus.CounterOpts{ Namespace: "cortex", From 3c94e10bceafafaa16dde66b0957b9e9ee534f2c Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Mon, 15 Jul 2019 15:40:02 -0400 Subject: [PATCH 45/60] readd total configs metric Signed-off-by: Jacob Lisi --- pkg/ruler/scheduler.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index cc33a4d841e..3b33d9b7894 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -27,6 +27,11 @@ const ( ) var ( + totalConfigs = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "cortex", + Name: "scheduler_configs_total", + Help: "How many user configs the scheduler knows about.", + }) totalRuleGroups = promauto.NewGauge(prometheus.GaugeOpts{ Namespace: "cortex", Name: "scheduler_groups_total", @@ -37,17 +42,17 @@ var ( Name: "scheduler_update_failures_total", Help: "Number of failures when updating rule groups", }) - iterationsMissed = promauto.NewCounterVec(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "rule_group_iterations_missed_total", - Help: "The total number of rule group evaluations missed due to slow rule group evaluation.", - }, []string{"id"}) evalLatency = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: "cortex", Name: "group_evaluation_latency_seconds", Help: "How far behind the target time each rule group executed.", Buckets: []float64{.025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25, 60}, }) + iterationsMissed = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "cortex", + Name: "rule_group_iterations_missed_total", + Help: "The total number of rule group evaluations missed due to slow rule group evaluation.", + }, []string{"user"}) ) type workItem struct { @@ -150,6 +155,7 @@ func (s *scheduler) updateConfigs(ctx context.Context) error { s.addUserConfig(ctx, user, cfg) } + totalConfigs.Set(float64(len(s.cfgs))) return nil } From eaa6bf1605cfb09a4bf71b1e71568047fb951401 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Mon, 15 Jul 2019 16:19:45 -0400 Subject: [PATCH 46/60] pass user id as label to the rule group metrics Signed-off-by: Jacob Lisi --- pkg/ruler/ruler.go | 44 ++++++++++++++++++++---------------------- pkg/ruler/scheduler.go | 5 ----- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index b62df403492..005f0e37be2 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -37,27 +37,15 @@ var ( Help: "The duration for a rule group to execute.", Buckets: []float64{.5, 1, 2.5, 5, 10, 25, 60, 120}, }) - rulesProcessed = promauto.NewCounter(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "rules_processed_total", - Help: "How many rules have been processed.", - }) ringCheckErrors = promauto.NewCounter(prometheus.CounterOpts{ Namespace: "cortex", Name: "ruler_ring_check_errors_total", Help: "Number of errors that have occurred when checking the ring for ownership", }) - evalFailures = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "ruler_evalauation_failures_total", - Help: "Total number of rule groups failed to be evaluated.", - }, []string{"id"}) - ruleMetrics *rules.Metrics ) func init() { evalDuration.Register() - ruleMetrics = rules.NewGroupMetrics(prometheus.DefaultRegisterer) } // Config is the configuration for the recording rules server. @@ -131,6 +119,10 @@ type Ruler struct { // Per-user notifiers with separate queues. notifiersMtx sync.Mutex notifiers map[string]*rulerNotifier + + // Per-user rules metrics + userMetricsMtx sync.Mutex + userMetrics map[string]*rules.Metrics } // NewRuler creates a new ruler from a distributor and chunk store. @@ -153,6 +145,7 @@ func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d notifierCfg: ncfg, notifiers: map[string]*rulerNotifier{}, workerWG: &sync.WaitGroup{}, + userMetrics: map[string]*rules.Metrics{}, } ruler.scheduler = newScheduler(poller, cfg.EvaluationInterval, cfg.EvaluationInterval, ruler.newGroup) @@ -213,16 +206,29 @@ func (r *Ruler) Stop() { } func (r *Ruler) newGroup(ctx context.Context, g RuleGroup) (*wrappedGroup, error) { + user := g.User() appendable := &appendableAppender{pusher: r.pusher} - notifier, err := r.getOrCreateNotifier(g.User()) + notifier, err := r.getOrCreateNotifier(user) if err != nil { return nil, err } + rls, err := g.Rules(ctx) if err != nil { return nil, err } + // Get the rule group metrics for set user or create it if it does not exist + r.userMetricsMtx.Lock() + metrics, exists := r.userMetrics[user] + if !exists { + // Wrap the default register with the users ID and pass + reg := prometheus.WrapRegistererWith(prometheus.Labels{"user": user}, prometheus.DefaultRegisterer) + metrics = rules.NewGroupMetrics(reg) + r.userMetrics[user] = metrics + } + r.userMetricsMtx.Unlock() + opts := &rules.ManagerOptions{ Appendable: appendable, QueryFunc: rules.EngineQueryFunc(r.engine, r.queryable), @@ -230,7 +236,7 @@ func (r *Ruler) newGroup(ctx context.Context, g RuleGroup) (*wrappedGroup, error ExternalURL: r.alertURL, NotifyFunc: sendAlerts(notifier, r.alertURL.String()), Logger: util.Logger, - Metrics: ruleMetrics, + Metrics: metrics, } return newGroup(g.Name(), rls, appendable, opts), nil } @@ -310,7 +316,7 @@ func (r *Ruler) Evaluate(userID string, item *workItem) { return } level.Debug(logger).Log("msg", "evaluating rules...", "num_rules", len(item.group.Rules())) - ctx, cancelTimeout := context.WithTimeout(ctx, r.cfg.GroupTimeout) + instrument.CollectedRequest(ctx, "Evaluate", evalDuration, nil, func(ctx native_ctx.Context) error { if span := opentracing.SpanFromContext(ctx); span != nil { span.SetTag("instance", userID) @@ -319,14 +325,6 @@ func (r *Ruler) Evaluate(userID string, item *workItem) { item.group.Eval(ctx, time.Now()) return nil }) - if err := ctx.Err(); err == nil { - cancelTimeout() // release resources - } else { - evalFailures.WithLabelValues(userID).Inc() - level.Warn(logger).Log("msg", "context error", "error", err) - } - - rulesProcessed.Add(float64(len(item.group.Rules()))) } func (r *Ruler) ownsRule(hash uint32) bool { diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index 3b33d9b7894..dd54e40b905 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -37,11 +37,6 @@ var ( Name: "scheduler_groups_total", Help: "How many rule groups the scheduler is currently evaluating", }) - scheduleFailures = promauto.NewCounter(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "scheduler_update_failures_total", - Help: "Number of failures when updating rule groups", - }) evalLatency = promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: "cortex", Name: "group_evaluation_latency_seconds", From f7f28bbd48b794ac85883b6ee1cd4d110bc7f62f Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Tue, 16 Jul 2019 14:44:21 -0400 Subject: [PATCH 47/60] pass user id to prometheus rule group metrics Signed-off-by: Jacob Lisi --- pkg/ruler/ruler.go | 21 --------------------- pkg/ruler/worker.go | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 005f0e37be2..c738188ebcc 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -10,7 +10,6 @@ import ( "time" "github.com/go-kit/kit/log/level" - opentracing "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/prometheus/config" @@ -307,26 +306,6 @@ func (r *Ruler) getOrCreateNotifier(userID string) (*notifier.Manager, error) { return n.notifier, nil } -// Evaluate a list of rules in the given context. -func (r *Ruler) Evaluate(userID string, item *workItem) { - ctx := user.InjectOrgID(context.Background(), userID) - logger := util.WithContext(ctx, util.Logger) - if r.cfg.EnableSharding && !r.ownsRule(item.hash) { - level.Debug(util.Logger).Log("msg", "ruler: skipping evaluation, not owned", "user_id", item.userID, "group", item.groupName) - return - } - level.Debug(logger).Log("msg", "evaluating rules...", "num_rules", len(item.group.Rules())) - - instrument.CollectedRequest(ctx, "Evaluate", evalDuration, nil, func(ctx native_ctx.Context) error { - if span := opentracing.SpanFromContext(ctx); span != nil { - span.SetTag("instance", userID) - span.SetTag("groupName", item.groupName) - } - item.group.Eval(ctx, time.Now()) - return nil - }) -} - func (r *Ruler) ownsRule(hash uint32) bool { rlrs, err := r.ring.Get(hash, ring.Read) // If an error occurs evaluate a rule as if it is owned diff --git a/pkg/ruler/worker.go b/pkg/ruler/worker.go index bbff15a9d84..fed6b27188d 100644 --- a/pkg/ruler/worker.go +++ b/pkg/ruler/worker.go @@ -1,12 +1,17 @@ package ruler import ( + "context" + native_ctx "context" "time" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" + opentracing "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/weaveworks/common/instrument" + "github.com/weaveworks/common/user" ) var ( @@ -42,19 +47,45 @@ func newWorker(ruler *Ruler) worker { func (w *worker) Run() { for { + // Grab next scheduled item from the queue + level.Debug(util.Logger).Log("msg", "waiting for next work item") waitStart := time.Now() + blockedWorkers.Inc() - level.Debug(util.Logger).Log("msg", "waiting for next work item") item := w.scheduler.nextWorkItem() blockedWorkers.Dec() + waitElapsed := time.Now().Sub(waitStart) + workerIdleTime.Add(waitElapsed.Seconds()) + + // If no item is returned, worker is safe to terminate if item == nil { level.Debug(util.Logger).Log("msg", "queue closed and empty; terminating worker") return } - workerIdleTime.Add(waitElapsed.Seconds()) - w.ruler.Evaluate(item.userID, item) + + w.Evaluate(item.userID, item) w.scheduler.workItemDone(*item) level.Debug(util.Logger).Log("msg", "item handed back to queue", "item", item) } } + +// Evaluate a list of rules in the given context. +func (w *worker) Evaluate(userID string, item *workItem) { + ctx := user.InjectOrgID(context.Background(), userID) + logger := util.WithContext(ctx, util.Logger) + if w.ruler.cfg.EnableSharding && !w.ruler.ownsRule(item.hash) { + level.Debug(util.Logger).Log("msg", "ruler: skipping evaluation, not owned", "user_id", item.userID, "group", item.groupName) + return + } + level.Debug(logger).Log("msg", "evaluating rules...", "num_rules", len(item.group.Rules())) + + instrument.CollectedRequest(ctx, "Evaluate", evalDuration, nil, func(ctx native_ctx.Context) error { + if span := opentracing.SpanFromContext(ctx); span != nil { + span.SetTag("instance", userID) + span.SetTag("groupName", item.groupName) + } + item.group.Eval(ctx, time.Now()) + return nil + }) +} From ad0dd20e14b7bd8789a01fa8c059cd43c6ec6355 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Thu, 18 Jul 2019 13:30:59 -0400 Subject: [PATCH 48/60] refactor to use kv store to check for config updates Signed-off-by: Jacob Lisi --- Makefile | 3 +- pkg/alertmanager/api.go | 40 +- pkg/alertmanager/kv_poller.go | 101 +++ pkg/alertmanager/multitenant.go | 30 +- pkg/alertmanager/storage.go | 55 ++ pkg/alertmanager/{ => storage}/alert_store.go | 7 +- pkg/cortex/cortex.go | 3 - pkg/cortex/modules.go | 31 +- pkg/ruler/api.go | 75 ++- pkg/ruler/group/compat.go | 102 --- pkg/ruler/group/rulegroup.go | 41 -- pkg/ruler/group/rulegroup.proto | 32 - pkg/ruler/kv_poller.go | 107 ++++ pkg/ruler/lifecycle.go | 4 - pkg/ruler/lifecycle_test.go | 2 - pkg/ruler/mock_store.go | 100 +++ pkg/ruler/ruler.go | 21 +- pkg/ruler/ruler_test.go | 103 +-- pkg/ruler/scheduler.go | 43 +- pkg/ruler/scheduler_test.go | 23 +- pkg/ruler/storage.go | 60 ++ pkg/ruler/store/compat.go | 98 +++ pkg/ruler/store/group.go | 110 ++++ pkg/ruler/{rule_store.go => store/store.go} | 43 +- .../rulegroup.pb.go => store/store.pb.go} | 599 +++++++++++++----- pkg/ruler/store/store.proto | 38 ++ pkg/ruler/worker.go | 4 +- pkg/storage/clients/aws/s3_config_client.go | 121 ++-- pkg/storage/clients/client_test.go | 10 +- pkg/storage/clients/configdb/client.go | 49 +- pkg/storage/clients/gcp/config_client.go | 254 ++------ pkg/storage/clients/gcp/fixtures.go | 6 +- pkg/storage/factory.go | 75 --- pkg/storage/testutils/testutils.go | 6 +- pkg/util/usertracker/tracker.go | 115 ++++ pkg/util/usertracker/tracker.pb.go | 446 +++++++++++++ pkg/util/usertracker/tracker.proto | 16 + 37 files changed, 2048 insertions(+), 925 deletions(-) create mode 100644 pkg/alertmanager/kv_poller.go create mode 100644 pkg/alertmanager/storage.go rename pkg/alertmanager/{ => storage}/alert_store.go (80%) delete mode 100644 pkg/ruler/group/compat.go delete mode 100644 pkg/ruler/group/rulegroup.go delete mode 100644 pkg/ruler/group/rulegroup.proto create mode 100644 pkg/ruler/kv_poller.go create mode 100644 pkg/ruler/mock_store.go create mode 100644 pkg/ruler/storage.go create mode 100644 pkg/ruler/store/compat.go create mode 100644 pkg/ruler/store/group.go rename pkg/ruler/{rule_store.go => store/store.go} (81%) rename pkg/ruler/{group/rulegroup.pb.go => store/store.pb.go} (59%) create mode 100644 pkg/ruler/store/store.proto delete mode 100644 pkg/storage/factory.go create mode 100644 pkg/util/usertracker/tracker.go create mode 100644 pkg/util/usertracker/tracker.pb.go create mode 100644 pkg/util/usertracker/tracker.proto diff --git a/Makefile b/Makefile index 94966908111..e3125c9c057 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,8 @@ pkg/ring/ring.pb.go: pkg/ring/ring.proto pkg/querier/frontend/frontend.pb.go: pkg/querier/frontend/frontend.proto pkg/chunk/storage/caching_index_client.pb.go: pkg/chunk/storage/caching_index_client.proto pkg/distributor/ha_tracker.pb.go: pkg/distributor/ha_tracker.proto -pkg/ruler/group/group.pb.go: pkg/ruler/group/group.proto +pkg/ruler/store/store.pb.go: pkg/ruler/store/store.proto +pkg/util/usertracker/usertracker.pb.go: pkg/util/usertracker/usertracker.proto all: $(UPTODATE_FILES) test: protos mod-check: protos diff --git a/pkg/alertmanager/api.go b/pkg/alertmanager/api.go index 1489d8d7eee..c42c80b2204 100644 --- a/pkg/alertmanager/api.go +++ b/pkg/alertmanager/api.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "net/http" + "github.com/cortexproject/cortex/pkg/alertmanager/storage" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "github.com/gorilla/mux" @@ -11,31 +12,25 @@ import ( "gopkg.in/yaml.v2" ) -// API is used to provided endpoints to directly interact with the ruler -type API struct { - store AlertStore -} - -// NewAPI returns a ruler API -func NewAPI(store AlertStore) *API { - return &API{store} -} - // RegisterRoutes registers the configs API HTTP routes with the provided Router. -func (a *API) RegisterRoutes(r *mux.Router) { +func (am *MultitenantAlertmanager) RegisterRoutes(r *mux.Router) { + // if no storeis set return without regisering routes + if am.store == nil { + return + } for _, route := range []struct { name, method, path string handler http.HandlerFunc }{ - {"get_config", "GET", "/api/prom/alertmanager", a.getConfig}, - {"set_config", "POST", "/api/prom/alertmanager", a.setConfig}, - {"delete_config", "DELETE", "/api/prom/alertmanager", a.deleteConfig}, + {"get_config", "GET", "/api/prom/alertmanager", am.getUserConfig}, + {"set_config", "POST", "/api/prom/alertmanager", am.setUserConfig}, + {"delete_config", "DELETE", "/api/prom/alertmanager", am.deleteUserConfig}, } { r.Handle(route.path, route.handler).Methods(route.method).Name(route.name) } } -func (a *API) getConfig(w http.ResponseWriter, r *http.Request) { +func (am *MultitenantAlertmanager) getUserConfig(w http.ResponseWriter, r *http.Request) { userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) if err != nil { http.Error(w, err.Error(), http.StatusUnauthorized) @@ -55,7 +50,7 @@ func (a *API) getConfig(w http.ResponseWriter, r *http.Request) { return } - cfg, err := a.store.GetAlertConfig(r.Context(), userID) + cfg, err := am.store.GetAlertConfig(r.Context(), userID) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -76,7 +71,7 @@ func (a *API) getConfig(w http.ResponseWriter, r *http.Request) { } } -func (a *API) setConfig(w http.ResponseWriter, r *http.Request) { +func (am *MultitenantAlertmanager) setUserConfig(w http.ResponseWriter, r *http.Request) { userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) if err != nil { http.Error(w, err.Error(), http.StatusUnauthorized) @@ -84,11 +79,6 @@ func (a *API) setConfig(w http.ResponseWriter, r *http.Request) { } logger := util.WithContext(r.Context(), util.Logger) - if err != nil { - level.Error(logger).Log("err", err.Error()) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } if userID == "" { level.Error(logger).Log("err", err.Error()) @@ -103,7 +93,7 @@ func (a *API) setConfig(w http.ResponseWriter, r *http.Request) { return } - cfg := AlertConfig{} + cfg := storage.AlertConfig{} err = yaml.Unmarshal(payload, &cfg) if err != nil { level.Error(logger).Log("err", err.Error()) @@ -111,7 +101,7 @@ func (a *API) setConfig(w http.ResponseWriter, r *http.Request) { return } - err = a.store.SetAlertConfig(r.Context(), userID, cfg) + err = am.store.SetAlertConfig(r.Context(), userID, cfg) if err != nil { level.Error(logger).Log("err", err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) @@ -121,6 +111,6 @@ func (a *API) setConfig(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } -func (a *API) deleteConfig(w http.ResponseWriter, r *http.Request) { +func (am *MultitenantAlertmanager) deleteUserConfig(w http.ResponseWriter, r *http.Request) { } diff --git a/pkg/alertmanager/kv_poller.go b/pkg/alertmanager/kv_poller.go new file mode 100644 index 00000000000..15afab33166 --- /dev/null +++ b/pkg/alertmanager/kv_poller.go @@ -0,0 +1,101 @@ +package alertmanager + +import ( + "context" + + "github.com/cortexproject/cortex/pkg/alertmanager/storage" + "github.com/cortexproject/cortex/pkg/util/usertracker" +) + +// trackedAlertPoller checks for updated user configs and +// retreives the updated configuration from the backend +type trackedAlertPoller struct { + tracker *usertracker.Tracker + store storage.AlertStore + + initialized bool +} + +func newTrackedAlertPoller(tracker *usertracker.Tracker, store storage.AlertStore) (*trackedAlertPoller, error) { + return &trackedAlertPoller{ + tracker: tracker, + store: store, + + initialized: false, + }, nil +} + +func (p *trackedAlertPoller) trackedAlertStore() *trackedAlertStore { + return &trackedAlertStore{ + tracker: p.tracker, + store: p.store, + } +} + +// PollAlerts returns the alerts changed since the last poll +// All alert configurations are returned on the first poll +func (p *trackedAlertPoller) PollAlerts(ctx context.Context) (map[string]storage.AlertConfig, error) { + updatedConfigs := map[string]storage.AlertConfig{} + + // First poll will return all rule groups + if !p.initialized { + p.initialized = true + return p.store.ListAlertConfigs(ctx) + } + + // Get the changed users from the user update tracker + users := p.tracker.GetUpdatedUsers(ctx) + + // Retreive user configuration from the rule store + // TODO: Add Retry logic for failed requests + // TODO: store users that were failed to be updated and reattempt to retrieve on the next poll + for _, u := range users { + cfg, err := p.store.GetAlertConfig(ctx, u) + if err != nil { + return nil, err + } + + updatedConfigs[u] = cfg + } + + return updatedConfigs, nil +} + +func (p *trackedAlertPoller) Stop() { + p.tracker.Stop() +} + +type trackedAlertStore struct { + tracker *usertracker.Tracker + store storage.AlertStore +} + +// ListAlertConfigs passes through to the embedded alert store +func (w *trackedAlertStore) ListAlertConfigs(ctx context.Context) (map[string]storage.AlertConfig, error) { + return w.store.ListAlertConfigs(ctx) +} + +// GetAlertConfig passes through to the embedded alert store +func (w *trackedAlertStore) GetAlertConfig(ctx context.Context, id string) (storage.AlertConfig, error) { + return w.store.GetAlertConfig(ctx, id) +} + +// SetAlertConfig passes through to the embedded alert store, and tracks a user change +func (w *trackedAlertStore) SetAlertConfig(ctx context.Context, id string, cfg storage.AlertConfig) error { + err := w.store.SetAlertConfig(ctx, id, cfg) + if err != nil { + return err + } + + return w.tracker.UpdateUser(ctx, id) +} + +// DeleteAlertConfig passes through to the embedded alert store, and tracks a user change +func (w *trackedAlertStore) DeleteAlertConfig(ctx context.Context, id string) error { + err := w.store.DeleteAlertConfig(ctx, id) + if err != nil { + return err + } + + return w.tracker.UpdateUser(ctx, id) +} diff --git a/pkg/alertmanager/multitenant.go b/pkg/alertmanager/multitenant.go index fcc9f6cb2b3..b2bbf92dfe7 100644 --- a/pkg/alertmanager/multitenant.go +++ b/pkg/alertmanager/multitenant.go @@ -22,6 +22,7 @@ import ( "github.com/weaveworks/common/user" "github.com/weaveworks/mesh" + "github.com/cortexproject/cortex/pkg/alertmanager/storage" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" ) @@ -183,6 +184,8 @@ type MultitenantAlertmanagerConfig struct { FallbackConfigFile string AutoWebhookRoot string AutoSlackRoot string + + AlertStore AlertStoreConfig } // RegisterFlags adds the flags required to config this to the given FlagSet. @@ -212,7 +215,8 @@ func (cfg *MultitenantAlertmanagerConfig) RegisterFlags(f *flag.FlagSet) { type MultitenantAlertmanager struct { cfg *MultitenantAlertmanagerConfig - store AlertPoller + store storage.AlertStore + poller storage.AlertPoller // The fallback config is stored as a string and parsed every time it's needed // because we mutate the parsed results and don't want those changes to take @@ -220,7 +224,7 @@ type MultitenantAlertmanager struct { fallbackConfig string // All the organization configurations that we have. Only used for instrumentation. - cfgs map[string]AlertConfig + cfgs map[string]storage.AlertConfig alertmanagersMtx sync.Mutex alertmanagers map[string]*Alertmanager @@ -233,7 +237,7 @@ type MultitenantAlertmanager struct { } // NewMultitenantAlertmanager creates a new MultitenantAlertmanager. -func NewMultitenantAlertmanager(cfg *MultitenantAlertmanagerConfig, store AlertPoller) (*MultitenantAlertmanager, error) { +func NewMultitenantAlertmanager(cfg *MultitenantAlertmanagerConfig) (*MultitenantAlertmanager, error) { err := os.MkdirAll(cfg.DataDir, 0777) if err != nil { return nil, fmt.Errorf("unable to create Alertmanager data directory %q: %s", cfg.DataDir, err) @@ -255,8 +259,15 @@ func NewMultitenantAlertmanager(cfg *MultitenantAlertmanagerConfig, store AlertP } gf := newGossipFactory(mrouter) + + poller, store, err := NewAlertStore(cfg.AlertStore) + if err != nil { + return nil, fmt.Errorf("unable to create Alertmanager storage, %s", err) + } + am := &MultitenantAlertmanager{ cfg: cfg, + poller: poller, store: store, fallbackConfig: string(fallbackConfig), alertmanagers: map[string]*Alertmanager{}, @@ -303,6 +314,7 @@ func (am *MultitenantAlertmanager) Run() { // Stop stops the MultitenantAlertmanager. func (am *MultitenantAlertmanager) Stop() { + am.poller.Stop() am.srvDiscovery.Stop() close(am.stop) <-am.done @@ -315,7 +327,7 @@ func (am *MultitenantAlertmanager) Stop() { // Load the full set of configurations from the server, retrying with backoff // until we can get them. -func (am *MultitenantAlertmanager) loadAllConfigs() map[string]AlertConfig { +func (am *MultitenantAlertmanager) loadAllConfigs() map[string]storage.AlertConfig { backoff := util.NewBackoff(context.Background(), backoffConfig) for { cfgs, err := am.poll() @@ -338,8 +350,8 @@ func (am *MultitenantAlertmanager) updateConfigs(now time.Time) error { } // poll the configuration server. Not re-entrant. -func (am *MultitenantAlertmanager) poll() (map[string]AlertConfig, error) { - cfgs, err := am.store.PollAlertConfigs(context.Background()) +func (am *MultitenantAlertmanager) poll() (map[string]storage.AlertConfig, error) { + cfgs, err := am.poller.PollAlerts(context.Background()) if err != nil { level.Warn(util.Logger).Log("msg", "MultitenantAlertmanager: configs server poll failed", "err", err) return nil, err @@ -347,7 +359,7 @@ func (am *MultitenantAlertmanager) poll() (map[string]AlertConfig, error) { return cfgs, nil } -func (am *MultitenantAlertmanager) addNewConfigs(cfgs map[string]AlertConfig) { +func (am *MultitenantAlertmanager) addNewConfigs(cfgs map[string]storage.AlertConfig) { // TODO: instrument how many configs we have, both valid & invalid. level.Debug(util.Logger).Log("msg", "adding configurations", "num_configs", len(cfgs)) for userID, config := range cfgs { @@ -411,7 +423,7 @@ func (am *MultitenantAlertmanager) createTemplatesFile(userID, fn, content strin // setConfig applies the given configuration to the alertmanager for `userID`, // creating an alertmanager if it doesn't already exist. -func (am *MultitenantAlertmanager) setConfig(userID string, config AlertConfig) error { +func (am *MultitenantAlertmanager) setConfig(userID string, config storage.AlertConfig) error { _, hasExisting := am.alertmanagers[userID] var amConfig *amconfig.Config var err error @@ -474,7 +486,7 @@ func (am *MultitenantAlertmanager) setConfig(userID string, config AlertConfig) } // alertmanagerConfigFromConfig returns the Alertmanager config from the Cortex configuration. -func alertmanagerConfigFromConfig(c AlertConfig) (*amconfig.Config, error) { +func alertmanagerConfigFromConfig(c storage.AlertConfig) (*amconfig.Config, error) { cfg, err := amconfig.Load(c.AlertmanagerConfig) if err != nil { return nil, fmt.Errorf("error parsing Alertmanager config: %s", err) diff --git a/pkg/alertmanager/storage.go b/pkg/alertmanager/storage.go new file mode 100644 index 00000000000..7c3800c11c3 --- /dev/null +++ b/pkg/alertmanager/storage.go @@ -0,0 +1,55 @@ +package alertmanager + +import ( + "context" + "flag" + "fmt" + + "github.com/cortexproject/cortex/pkg/alertmanager/storage" + "github.com/cortexproject/cortex/pkg/storage/clients/configdb" + "github.com/cortexproject/cortex/pkg/storage/clients/gcp" + "github.com/cortexproject/cortex/pkg/util/usertracker" +) + +type AlertStoreConfig struct { + Type string `yaml:"type"` + + ConfigDB configdb.Config + + GCS gcp.GCSConfig + Tracker usertracker.Config +} + +// RegisterFlags registers flags. +func (cfg *AlertStoreConfig) RegisterFlags(f *flag.FlagSet) { + cfg.ConfigDB.RegisterFlagsWithPrefix("alertmanager", f) + cfg.GCS.RegisterFlagsWithPrefix("alertmanager.store.", f) + cfg.Tracker.RegisterFlagsWithPrefix("alertmanager.", f) + f.StringVar(&cfg.Type, "alertmanager.storage.type", "configdb", "Method to use for backend rule storage (configdb, gcs)") +} + +// NewAlertStore returns a new rule storage backend poller and store +func NewAlertStore(cfg AlertStoreConfig) (storage.AlertPoller, storage.AlertStore, error) { + var ( + alertStore storage.AlertStore + err error + ) + switch cfg.Type { + case "configdb": + poller, err := configdb.New(cfg.ConfigDB) + return poller, nil, err + case "gcs": + alertStore, err = gcp.NewGCSClient(context.Background(), cfg.GCS) + default: + return nil, nil, fmt.Errorf("Unrecognized rule storage mode %v, choose one of: configdb, gcs", cfg.Type) + } + + tracker, err := usertracker.NewTracker(cfg.Tracker) + if err != nil { + return nil, nil, err + } + + p, err := newTrackedAlertPoller(tracker, alertStore) + + return p, p.trackedAlertStore(), err +} diff --git a/pkg/alertmanager/alert_store.go b/pkg/alertmanager/storage/alert_store.go similarity index 80% rename from pkg/alertmanager/alert_store.go rename to pkg/alertmanager/storage/alert_store.go index e8feec8bcd6..a8246daaf89 100644 --- a/pkg/alertmanager/alert_store.go +++ b/pkg/alertmanager/storage/alert_store.go @@ -1,4 +1,4 @@ -package alertmanager +package storage import ( "context" @@ -12,12 +12,13 @@ type AlertConfig struct { // AlertPoller polls for updated alerts type AlertPoller interface { - PollAlertConfigs(ctx context.Context) (map[string]AlertConfig, error) - AlertStore() AlertStore + PollAlerts(ctx context.Context) (map[string]AlertConfig, error) + Stop() } // AlertStore stores config information and template files to configure alertmanager tenants type AlertStore interface { + ListAlertConfigs(ctx context.Context) (map[string]AlertConfig, error) GetAlertConfig(ctx context.Context, id string) (AlertConfig, error) SetAlertConfig(ctx context.Context, id string, cfg AlertConfig) error DeleteAlertConfig(ctx context.Context, id string) error diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index 30271445a4c..ac0fae0c59e 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -26,7 +26,6 @@ import ( "github.com/cortexproject/cortex/pkg/querier/frontend" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ruler" - config_storage "github.com/cortexproject/cortex/pkg/storage" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -70,7 +69,6 @@ type Config struct { Encoding encoding.Config `yaml:"-"` // No yaml for this, it only works with flags. ConfigDB db.Config `yaml:"configdb,omitempty"` - ConfigStore config_storage.Config `yaml:"config_store,omitempty"` Ruler ruler.Config `yaml:"ruler,omitempty"` Alertmanager alertmanager.MultitenantAlertmanagerConfig `yaml:"alertmanager,omitempty"` } @@ -101,7 +99,6 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { c.Ruler.RegisterFlags(f) c.ConfigDB.RegisterFlags(f) - c.ConfigStore.RegisterFlags(f) c.Alertmanager.RegisterFlags(f) // These don't seem to have a home. diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 28bd0ee0ca9..9ef69fb37ce 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -29,7 +29,6 @@ import ( "github.com/cortexproject/cortex/pkg/querier/frontend" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ruler" - config_storage "github.com/cortexproject/cortex/pkg/storage" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -320,23 +319,12 @@ func (t *Cortex) initRuler(cfg *Config) (err error) { cfg.Ruler.LifecyclerConfig.ListenPort = &cfg.Server.GRPCListenPort queryable, engine := querier.New(cfg.Querier, t.distributor, t.store) - poller, err := config_storage.NewRuleStore(cfg.ConfigStore) - if err != nil { - return err - } - - t.ruler, err = ruler.NewRuler(cfg.Ruler, engine, queryable, t.distributor, poller) + t.ruler, err = ruler.NewRuler(cfg.Ruler, engine, queryable, t.distributor) if err != nil { return } - // Only serve the API for setting & getting rules configs if we're not - // serving configs from the configs API. Allows for smoother - // migration. See https://github.com/cortexproject/cortex/issues/619 - if cfg.ConfigStore.RuleStoreConfig.BackendType != "configdb" { - a := ruler.NewAPI(poller.RuleStore()) - a.RegisterRoutes(t.server.HTTP) - } + t.ruler.RegisterRoutes(t.server.HTTP) t.server.HTTP.Handle("/ruler_ring", t.ruler) return @@ -364,23 +352,12 @@ func (t *Cortex) stopConfigs() error { } func (t *Cortex) initAlertmanager(cfg *Config) (err error) { - poller, err := config_storage.NewAlertStore(cfg.ConfigStore) - if err != nil { - return err - } - - t.alertmanager, err = alertmanager.NewMultitenantAlertmanager(&cfg.Alertmanager, poller) + t.alertmanager, err = alertmanager.NewMultitenantAlertmanager(&cfg.Alertmanager) if err != nil { return } - // Only serve the API for setting & getting alert configs if we're not - // serving configs from the configs API. Allows for smoother - // migration. See https://github.com/cortexproject/cortex/issues/619 - if cfg.ConfigStore.AlertStoreConfig.BackendType != "configdb" { - a := alertmanager.NewAPI(poller.AlertStore()) - a.RegisterRoutes(t.server.HTTP) - } + t.alertmanager.RegisterRoutes(t.server.HTTP) go t.alertmanager.Run() diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index d542e2c3a4a..669bb0f9728 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -7,6 +7,7 @@ import ( "github.com/prometheus/prometheus/pkg/rulefmt" + "github.com/cortexproject/cortex/pkg/ruler/store" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "github.com/gorilla/mux" @@ -20,40 +21,35 @@ var ( ErrNoRuleGroups = errors.New("no rule groups found") ) -// API is used to provided endpoints to directly interact with the ruler -type API struct { - store RuleStore -} - -// NewAPI returns a ruler API -func NewAPI(store RuleStore) *API { - return &API{store} -} - // RegisterRoutes registers the configs API HTTP routes with the provided Router. -func (a *API) RegisterRoutes(r *mux.Router) { +func (r *Ruler) RegisterRoutes(router *mux.Router) { + // If no store is set do not register routes in the api. This will only be the case if the configdb + // is used to store rules + if r.store == nil { + return + } for _, route := range []struct { name, method, path string handler http.HandlerFunc }{ - {"list_rules", "GET", "/api/prom/rules", a.listRules}, - {"list_rules_namespace", "GET", "/api/prom/rules/{namespace}", a.listRules}, - {"get_rulegroup", "GET", "/api/prom/rules/{namespace}/{groupName}", a.getRuleGroup}, - {"set_rulegroup", "POST", "/api/prom/rules/{namespace}", a.createRuleGroup}, - {"delete_rulegroup", "DELETE", "/api/prom/rules/{namespace}/{groupName}", a.deleteRuleGroup}, + {"list_rules", "GET", "/api/prom/rules", r.listRules}, + {"list_rules_namespace", "GET", "/api/prom/rules/{namespace}", r.listRules}, + {"get_rulegroup", "GET", "/api/prom/rules/{namespace}/{groupName}", r.getRuleGroup}, + {"set_rulegroup", "POST", "/api/prom/rules/{namespace}", r.createRuleGroup}, + {"delete_rulegroup", "DELETE", "/api/prom/rules/{namespace}/{groupName}", r.deleteRuleGroup}, } { - r.Handle(route.path, route.handler).Methods(route.method).Name(route.name) + router.Handle(route.path, route.handler).Methods(route.method).Name(route.name) } } -func (a *API) listRules(w http.ResponseWriter, r *http.Request) { - userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) +func (r *Ruler) listRules(w http.ResponseWriter, req *http.Request) { + userID, _, err := user.ExtractOrgIDFromHTTPRequest(req) if err != nil { http.Error(w, err.Error(), http.StatusUnauthorized) return } - logger := util.WithContext(r.Context(), util.Logger) + logger := util.WithContext(req.Context(), util.Logger) if err != nil { level.Error(logger).Log("err", err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) @@ -66,11 +62,11 @@ func (a *API) listRules(w http.ResponseWriter, r *http.Request) { return } - options := RuleStoreConditions{ + options := store.RuleStoreConditions{ UserID: userID, } - vars := mux.Vars(r) + vars := mux.Vars(req) namespace := vars["namespace"] if namespace != "" { @@ -79,8 +75,7 @@ func (a *API) listRules(w http.ResponseWriter, r *http.Request) { } level.Debug(logger).Log("msg", "retrieving rule groups from rule store", "userID", userID) - rgs, err := a.store.ListRuleGroups(r.Context(), options) - + rgs, err := r.store.ListRuleGroups(req.Context(), options) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -94,7 +89,9 @@ func (a *API) listRules(w http.ResponseWriter, r *http.Request) { return } - d, err := yaml.Marshal(&rgs) + formatted := rgs.Formatted(userID) + + d, err := yaml.Marshal(&formatted) if err != nil { level.Error(logger).Log("msg", "error marshalling yaml rule groups", "err", err) http.Error(w, err.Error(), http.StatusInternalServerError) @@ -109,14 +106,14 @@ func (a *API) listRules(w http.ResponseWriter, r *http.Request) { } } -func (a *API) getRuleGroup(w http.ResponseWriter, r *http.Request) { - userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) +func (r *Ruler) getRuleGroup(w http.ResponseWriter, req *http.Request) { + userID, _, err := user.ExtractOrgIDFromHTTPRequest(req) if err != nil { http.Error(w, err.Error(), http.StatusUnauthorized) return } - logger := util.WithContext(r.Context(), util.Logger) + logger := util.WithContext(req.Context(), util.Logger) if err != nil { level.Error(logger).Log("err", err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) @@ -129,7 +126,7 @@ func (a *API) getRuleGroup(w http.ResponseWriter, r *http.Request) { return } - vars := mux.Vars(r) + vars := mux.Vars(req) ns, exists := vars["namespace"] if !exists { http.Error(w, ErrNoNamespace.Error(), http.StatusUnauthorized) @@ -142,9 +139,9 @@ func (a *API) getRuleGroup(w http.ResponseWriter, r *http.Request) { return } - rg, err := a.store.GetRuleGroup(r.Context(), userID, ns, gn) + rg, err := r.store.GetRuleGroup(req.Context(), userID, ns, gn) if err != nil { - if err == ErrGroupNotFound { + if err == store.ErrGroupNotFound { http.Error(w, err.Error(), http.StatusNotFound) } http.Error(w, err.Error(), http.StatusBadRequest) @@ -166,14 +163,14 @@ func (a *API) getRuleGroup(w http.ResponseWriter, r *http.Request) { } } -func (a *API) createRuleGroup(w http.ResponseWriter, r *http.Request) { - userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) +func (r *Ruler) createRuleGroup(w http.ResponseWriter, req *http.Request) { + userID, _, err := user.ExtractOrgIDFromHTTPRequest(req) if err != nil { http.Error(w, err.Error(), http.StatusUnauthorized) return } - logger := util.WithContext(r.Context(), util.Logger) + logger := util.WithContext(req.Context(), util.Logger) if err != nil { level.Error(logger).Log("err", err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) @@ -186,7 +183,7 @@ func (a *API) createRuleGroup(w http.ResponseWriter, r *http.Request) { return } - vars := mux.Vars(r) + vars := mux.Vars(req) namespace := vars["namespace"] if namespace == "" { @@ -195,7 +192,7 @@ func (a *API) createRuleGroup(w http.ResponseWriter, r *http.Request) { return } - payload, err := ioutil.ReadAll(r.Body) + payload, err := ioutil.ReadAll(req.Body) if err != nil { level.Error(logger).Log("err", err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) @@ -212,14 +209,14 @@ func (a *API) createRuleGroup(w http.ResponseWriter, r *http.Request) { return } - errs := ValidateRuleGroup(rg) + errs := store.ValidateRuleGroup(rg) if len(errs) > 0 { level.Error(logger).Log("err", err.Error()) http.Error(w, errs[0].Error(), http.StatusBadRequest) return } - err = a.store.SetRuleGroup(r.Context(), userID, namespace, rg) + err = r.store.SetRuleGroup(req.Context(), userID, namespace, rg) if err != nil { level.Error(logger).Log("err", err.Error()) http.Error(w, err.Error(), http.StatusInternalServerError) @@ -230,6 +227,6 @@ func (a *API) createRuleGroup(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusAccepted) } -func (a *API) deleteRuleGroup(w http.ResponseWriter, r *http.Request) { +func (r *Ruler) deleteRuleGroup(w http.ResponseWriter, req *http.Request) { } diff --git a/pkg/ruler/group/compat.go b/pkg/ruler/group/compat.go deleted file mode 100644 index f891cec0f66..00000000000 --- a/pkg/ruler/group/compat.go +++ /dev/null @@ -1,102 +0,0 @@ -package group - -import ( - time "time" - - "github.com/cortexproject/cortex/pkg/ingester/client" - "github.com/cortexproject/cortex/pkg/util" - - "github.com/go-kit/kit/log" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/pkg/rulefmt" - "github.com/prometheus/prometheus/promql" - "github.com/prometheus/prometheus/rules" -) - -// ToProto transforms a formatted prometheus rulegroup to a rule group protobuf -func ToProto(namespace string, rl rulefmt.RuleGroup) RuleGroup { - dur := time.Duration(rl.Interval) - rg := RuleGroup{ - Name: rl.Name, - Namespace: namespace, - Interval: &dur, - } - - rules := make([]*Rule, len(rl.Rules)) - for i := range rl.Rules { - f := time.Duration(rl.Rules[i].For) - - rules[i] = &Rule{ - Expr: rl.Rules[i].Expr, - Record: rl.Rules[i].Record, - Alert: rl.Rules[i].Alert, - - For: &f, - Labels: client.FromLabelsToLabelAdapaters(labels.FromMap(rl.Rules[i].Labels)), - Annotations: client.FromLabelsToLabelAdapaters(labels.FromMap(rl.Rules[i].Labels)), - } - } - - rg.Rules = rules - - return rg -} - -// FromProto generates a rulefmt RuleGroup -func FromProto(rg *RuleGroup) *rulefmt.RuleGroup { - formattedRuleGroup := rulefmt.RuleGroup{ - Name: rg.GetName(), - Interval: model.Duration(*rg.Interval), - Rules: make([]rulefmt.Rule, len(rg.GetRules())), - } - - for i, rl := range rg.GetRules() { - formattedRuleGroup.Rules[i] = rulefmt.Rule{ - Record: rl.GetRecord(), - Alert: rl.GetAlert(), - Expr: rl.GetExpr(), - For: model.Duration(*rl.GetFor()), - Labels: client.FromLabelAdaptersToLabels(rl.Labels).Map(), - Annotations: client.FromLabelAdaptersToLabels(rl.Annotations).Map(), - } - } - - return &formattedRuleGroup -} - -// GenerateRuleGroup returns a functional rulegroup from a proto -func GenerateRuleGroup(userID string, rg *RuleGroup) (*Group, error) { - rls := make([]rules.Rule, 0, len(rg.Rules)) - for _, rl := range rg.Rules { - expr, err := promql.ParseExpr(rl.GetExpr()) - if err != nil { - return nil, err - } - - if rl.Alert != "" { - rls = append(rls, rules.NewAlertingRule( - rl.Alert, - expr, - *rl.GetFor(), - client.FromLabelAdaptersToLabels(rl.Labels), - client.FromLabelAdaptersToLabels(rl.Annotations), - true, - log.With(util.Logger, "alert", rl.Alert), - )) - continue - } - rls = append(rls, rules.NewRecordingRule( - rl.Record, - expr, - client.FromLabelAdaptersToLabels(rl.Labels), - )) - } - - return NewRuleGroup( - rg.GetName(), - rg.GetNamespace(), - userID, - rls, - ), nil -} diff --git a/pkg/ruler/group/rulegroup.go b/pkg/ruler/group/rulegroup.go deleted file mode 100644 index 6f17e197c50..00000000000 --- a/pkg/ruler/group/rulegroup.go +++ /dev/null @@ -1,41 +0,0 @@ -package group - -import ( - "context" - - "github.com/prometheus/prometheus/rules" -) - -// TODO: Add a lazy rule group that only loads rules when they are needed -// TODO: The cortex project should implement a separate Group struct from -// the prometheus project. This will allow for more precise instrumentation - -type Group struct { - name string - namespace string - user string - - rules []rules.Rule -} - -func (rg *Group) Rules(ctx context.Context) ([]rules.Rule, error) { - return rg.rules, nil -} - -func (rg *Group) Name() string { - return rg.namespace + "/" + rg.name -} - -func (rg *Group) User() string { - return rg.user -} - -// NewRuleGroup returns a rulegroup -func NewRuleGroup(name, namespace, user string, rules []rules.Rule) *Group { - return &Group{ - name: name, - namespace: namespace, - user: user, - rules: rules, - } -} diff --git a/pkg/ruler/group/rulegroup.proto b/pkg/ruler/group/rulegroup.proto deleted file mode 100644 index ea6a6eede7c..00000000000 --- a/pkg/ruler/group/rulegroup.proto +++ /dev/null @@ -1,32 +0,0 @@ -syntax = "proto3"; - -package group; - -option go_package = "group"; - -import "github.com/gogo/protobuf/gogoproto/gogo.proto"; -import "google/protobuf/duration.proto"; -import "github.com/cortexproject/cortex/pkg/ingester/client/cortex.proto"; - -option (gogoproto.marshaler_all) = true; -option (gogoproto.unmarshaler_all) = true; - - -message RuleGroup { - string name = 1; - string namespace = 2; - google.protobuf.Duration interval = 3 [(gogoproto.stdduration) = true]; - - repeated Rule rules = 4; - - bool deleted = 5; -} - -message Rule { - string expr = 1; - string record = 2; - string alert = 3; - google.protobuf.Duration for = 4 [(gogoproto.stdduration) = true]; - repeated cortex.LabelPair labels = 5 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter"];; - repeated cortex.LabelPair annotations = 6 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter"];; -} \ No newline at end of file diff --git a/pkg/ruler/kv_poller.go b/pkg/ruler/kv_poller.go new file mode 100644 index 00000000000..bd4ebe2601e --- /dev/null +++ b/pkg/ruler/kv_poller.go @@ -0,0 +1,107 @@ +package ruler + +import ( + "context" + + "github.com/cortexproject/cortex/pkg/ruler/store" + "github.com/cortexproject/cortex/pkg/util/usertracker" + "github.com/prometheus/prometheus/pkg/rulefmt" +) + +type trackedPoller struct { + tracker *usertracker.Tracker + store store.RuleStore + + initialized bool +} + +func newTrackedPoller(tracker *usertracker.Tracker, store store.RuleStore) (*trackedPoller, error) { + return &trackedPoller{ + tracker: tracker, + store: store, + + initialized: false, + }, nil +} + +func (p *trackedPoller) trackedRuleStore() *trackedRuleStore { + return &trackedRuleStore{ + tracker: p.tracker, + store: p.store, + } +} + +func (p *trackedPoller) PollRules(ctx context.Context) (map[string][]store.RuleGroup, error) { + updatedRules := map[string][]store.RuleGroup{} + + // First poll will return all rule groups + if !p.initialized { + rgs, err := p.store.ListRuleGroups(ctx, store.RuleStoreConditions{}) + if err != nil { + return nil, nil + } + for _, rg := range rgs { + if _, exists := updatedRules[rg.User()]; !exists { + updatedRules[rg.User()] = []store.RuleGroup{rg} + } else { + updatedRules[rg.User()] = append(updatedRules[rg.User()], rg) + } + } + p.initialized = true + } else { + users := p.tracker.GetUpdatedUsers(ctx) + for _, u := range users { + rgs, err := p.store.ListRuleGroups(ctx, store.RuleStoreConditions{ + UserID: u, + }) + if err != nil { + return nil, nil + } + + updatedRules[u] = rgs + } + } + + return updatedRules, nil +} + +func (p *trackedPoller) Stop() { + p.tracker.Stop() +} + +type trackedRuleStore struct { + tracker *usertracker.Tracker + store store.RuleStore +} + +// ListRuleGroups returns set of all rule groups matching the provided conditions +func (w *trackedRuleStore) ListRuleGroups(ctx context.Context, options store.RuleStoreConditions) (store.RuleGroupList, error) { + return w.store.ListRuleGroups(ctx, options) +} + +// GetRuleGroup retrieves the specified rule group from the backend store +func (w *trackedRuleStore) GetRuleGroup(ctx context.Context, userID, namespace, group string) (store.RuleGroup, error) { + return w.store.GetRuleGroup(ctx, userID, namespace, group) +} + +// SetRuleGroup updates a rule group in the backend persistent store, then it pushes a change update to the +// userID key entry in the KV store +func (w *trackedRuleStore) SetRuleGroup(ctx context.Context, userID, namespace string, group rulefmt.RuleGroup) error { + err := w.store.SetRuleGroup(ctx, userID, namespace, group) + if err != nil { + return err + } + + return w.tracker.UpdateUser(ctx, userID) +} + +// DeleteRuleGroup deletes a rule group in the backend persistent store, then it pushes a change update to the +// userID key entry in the KV store +func (w *trackedRuleStore) DeleteRuleGroup(ctx context.Context, userID, namespace string, group string) error { + err := w.store.DeleteRuleGroup(ctx, userID, namespace, group) + if err != nil { + return err + } + + return w.tracker.UpdateUser(ctx, userID) +} diff --git a/pkg/ruler/lifecycle.go b/pkg/ruler/lifecycle.go index 0fc7020a566..49dfb0034d8 100644 --- a/pkg/ruler/lifecycle.go +++ b/pkg/ruler/lifecycle.go @@ -8,10 +8,6 @@ import ( "github.com/go-kit/kit/log/level" ) -const ( - pendingSearchIterations = 10 -) - // TransferOut is a noop for the ruler func (r *Ruler) TransferOut(ctx context.Context) error { return nil diff --git a/pkg/ruler/lifecycle_test.go b/pkg/ruler/lifecycle_test.go index d8240552211..272cf43e81a 100644 --- a/pkg/ruler/lifecycle_test.go +++ b/pkg/ruler/lifecycle_test.go @@ -8,8 +8,6 @@ import ( "github.com/cortexproject/cortex/pkg/util/test" ) -const userID = "1" - // TestRulerShutdown tests shutting down ruler unregisters correctly func TestRulerShutdown(t *testing.T) { config := defaultRulerConfig() diff --git a/pkg/ruler/mock_store.go b/pkg/ruler/mock_store.go new file mode 100644 index 00000000000..ec4e1d422a4 --- /dev/null +++ b/pkg/ruler/mock_store.go @@ -0,0 +1,100 @@ +package ruler + +import ( + "context" + "strings" + + "github.com/cortexproject/cortex/pkg/ruler/store" + "github.com/prometheus/prometheus/pkg/rulefmt" +) + +type mockRuleStore struct { + rules map[string]store.RuleGroup + + pollPayload map[string][]store.RuleGroup +} + +func (m *mockRuleStore) PollRules(ctx context.Context) (map[string][]store.RuleGroup, error) { + pollPayload := m.pollPayload + m.pollPayload = map[string][]store.RuleGroup{} + return pollPayload, nil +} + +func (m *mockRuleStore) Stop() {} + +// RuleStore returns an RuleStore from the client +func (m *mockRuleStore) RuleStore() store.RuleStore { + return m +} + +func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options store.RuleStoreConditions) (store.RuleGroupList, error) { + groupPrefix := options.UserID + ":" + + namespaces := []string{} + nss := store.RuleGroupList{} + for n := range m.rules { + if strings.HasPrefix(n, groupPrefix) { + components := strings.Split(n, ":") + if len(components) != 3 { + continue + } + namespaces = append(namespaces, components[1]) + } + } + + if len(namespaces) == 0 { + return nss, store.ErrUserNotFound + } + + for _, n := range namespaces { + ns, err := m.getRuleNamespace(ctx, options.UserID, n) + if err != nil { + continue + } + + nss = append(nss, ns...) + } + + return nss, nil +} + +func (m *mockRuleStore) getRuleNamespace(ctx context.Context, userID string, namespace string) (store.RuleGroupList, error) { + groupPrefix := userID + ":" + namespace + ":" + + ns := store.RuleGroupList{} + for n, g := range m.rules { + if strings.HasPrefix(n, groupPrefix) { + ns = append(ns, g) + } + } + + if len(ns) == 0 { + return ns, store.ErrGroupNamespaceNotFound + } + + return ns, nil +} + +func (m *mockRuleStore) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (store.RuleGroup, error) { + groupID := userID + ":" + namespace + ":" + group + g, ok := m.rules[groupID] + + if !ok { + return nil, store.ErrGroupNotFound + } + + return g, nil + +} + +func (m *mockRuleStore) SetRuleGroup(ctx context.Context, userID string, namespace string, group rulefmt.RuleGroup) error { + groupID := userID + ":" + namespace + ":" + group.Name + m.rules[groupID] = store.NewRuleGroup(group.Name, namespace, userID, group.Rules) + return nil +} + +func (m *mockRuleStore) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { + groupID := userID + ":" + namespace + ":" + group + delete(m.rules, groupID) + return nil +} diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index c738188ebcc..a6651fff7bb 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -16,13 +16,14 @@ import ( "github.com/prometheus/prometheus/notifier" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/rules" - "github.com/prometheus/prometheus/storage" + promStorage "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/strutil" "golang.org/x/net/context" "golang.org/x/net/context/ctxhttp" "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/ring" + "github.com/cortexproject/cortex/pkg/ruler/store" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/weaveworks/common/instrument" @@ -76,11 +77,14 @@ type Config struct { SearchPendingFor time.Duration LifecyclerConfig ring.LifecyclerConfig FlushCheckPeriod time.Duration + + StoreConfig RuleStoreConfig } // RegisterFlags adds the flags required to config this to the given FlagSet func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.LifecyclerConfig.RegisterFlagsWithPrefix("ruler.", f) + cfg.StoreConfig.RegisterFlags(f) cfg.ExternalURL.URL, _ = url.Parse("") // Must be non-nil f.Var(&cfg.ExternalURL, "ruler.external.url", "URL of alerts return path.") @@ -104,7 +108,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { type Ruler struct { cfg Config engine *promql.Engine - queryable storage.Queryable + queryable promStorage.Queryable pusher Pusher alertURL *url.URL notifierCfg *config.Config @@ -115,6 +119,8 @@ type Ruler struct { lifecycler *ring.Lifecycler ring *ring.Ring + store store.RuleStore + // Per-user notifiers with separate queues. notifiersMtx sync.Mutex notifiers map[string]*rulerNotifier @@ -125,7 +131,7 @@ type Ruler struct { } // NewRuler creates a new ruler from a distributor and chunk store. -func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d *distributor.Distributor, poller RulePoller) (*Ruler, error) { +func NewRuler(cfg Config, engine *promql.Engine, queryable promStorage.Queryable, d *distributor.Distributor) (*Ruler, error) { if cfg.NumWorkers <= 0 { return nil, fmt.Errorf("must have at least 1 worker, got %d", cfg.NumWorkers) } @@ -135,6 +141,8 @@ func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d return nil, err } + rulePoller, ruleStore, err := NewRuleStorage(cfg.StoreConfig) + ruler := &Ruler{ cfg: cfg, engine: engine, @@ -145,9 +153,10 @@ func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d notifiers: map[string]*rulerNotifier{}, workerWG: &sync.WaitGroup{}, userMetrics: map[string]*rules.Metrics{}, + store: ruleStore, } - ruler.scheduler = newScheduler(poller, cfg.EvaluationInterval, cfg.EvaluationInterval, ruler.newGroup) + ruler.scheduler = newScheduler(rulePoller, cfg.EvaluationInterval, cfg.EvaluationInterval, ruler.newGroup) // If sharding is enabled, create/join a ring to distribute tokens to // the ruler @@ -204,7 +213,7 @@ func (r *Ruler) Stop() { } } -func (r *Ruler) newGroup(ctx context.Context, g RuleGroup) (*wrappedGroup, error) { +func (r *Ruler) newGroup(ctx context.Context, g store.RuleGroup) (*wrappedGroup, error) { user := g.User() appendable := &appendableAppender{pusher: r.pusher} notifier, err := r.getOrCreateNotifier(user) @@ -237,7 +246,7 @@ func (r *Ruler) newGroup(ctx context.Context, g RuleGroup) (*wrappedGroup, error Logger: util.Logger, Metrics: metrics, } - return newGroup(g.Name(), rls, appendable, opts), nil + return newGroup(g.ID(), rls, appendable, opts), nil } // sendAlerts implements a rules.NotifyFunc for a Notifier. diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index 13e383487f1..e970aed185a 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -1,10 +1,8 @@ package ruler import ( - "context" "net/http" "net/http/httptest" - "strings" "sync" "testing" "time" @@ -16,107 +14,19 @@ import ( "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/prometheus/prometheus/notifier" "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/pkg/rulefmt" "github.com/prometheus/prometheus/promql" "github.com/stretchr/testify/assert" "github.com/weaveworks/common/user" ) -type mockRuleStore struct { - rules map[string]rulefmt.RuleGroup - - pollPayload map[string][]RuleGroup -} - -func (m *mockRuleStore) PollRules(ctx context.Context) (map[string][]RuleGroup, error) { - pollPayload := m.pollPayload - m.pollPayload = map[string][]RuleGroup{} - return pollPayload, nil -} - -// RuleStore returns an RuleStore from the client -func (m *mockRuleStore) RuleStore() RuleStore { - return m -} - -func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options RuleStoreConditions) (map[string]RuleNamespace, error) { - groupPrefix := userID + ":" - - namespaces := []string{} - nss := map[string]RuleNamespace{} - for n := range m.rules { - if strings.HasPrefix(n, groupPrefix) { - components := strings.Split(n, ":") - if len(components) != 3 { - continue - } - namespaces = append(namespaces, components[1]) - } - } - - if len(namespaces) == 0 { - return nss, ErrUserNotFound - } - - for _, n := range namespaces { - ns, err := m.getRuleNamespace(ctx, userID, n) - if err != nil { - continue - } - - nss[n] = ns - } - - return nss, nil -} - -func (m *mockRuleStore) getRuleNamespace(ctx context.Context, userID string, namespace string) (RuleNamespace, error) { - groupPrefix := userID + ":" + namespace + ":" - - ns := RuleNamespace{ - Groups: []rulefmt.RuleGroup{}, - } - for n, g := range m.rules { - if strings.HasPrefix(n, groupPrefix) { - ns.Groups = append(ns.Groups, g) - } - } - - if len(ns.Groups) == 0 { - return ns, ErrGroupNamespaceNotFound - } - - return ns, nil -} - -func (m *mockRuleStore) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (*rulefmt.RuleGroup, error) { - groupID := userID + ":" + namespace + ":" + group - g, ok := m.rules[groupID] - - if !ok { - return nil, ErrGroupNotFound - } - - return &g, nil - -} - -func (m *mockRuleStore) SetRuleGroup(ctx context.Context, userID string, namespace string, group rulefmt.RuleGroup) error { - groupID := userID + ":" + namespace + ":" + group.Name - m.rules[groupID] = group - return nil -} - -func (m *mockRuleStore) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { - groupID := userID + ":" + namespace + ":" + group - delete(m.rules, groupID) - return nil -} - func defaultRulerConfig() Config { codec := codec.Proto{Factory: ring.ProtoDescFactory} consul := consul.NewInMemoryClient(codec) - cfg := Config{} + cfg := Config{ + StoreConfig: RuleStoreConfig{ + mock: &mockRuleStore{}, + }, + } flagext.DefaultValues(&cfg) flagext.DefaultValues(&cfg.LifecyclerConfig) cfg.LifecyclerConfig.RingConfig.ReplicationFactor = 1 @@ -139,7 +49,8 @@ func newTestRuler(t *testing.T, cfg Config) *Ruler { Timeout: 2 * time.Minute, }) queryable := querier.NewQueryable(nil, nil, nil, 0) - ruler, err := NewRuler(cfg, engine, queryable, nil, &mockRuleStore{}) + ruler, err := NewRuler(cfg, engine, queryable, nil) + ruler.store = &mockRuleStore{} if err != nil { t.Fatal(err) } diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index dd54e40b905..6a34d79a0cd 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/cortexproject/cortex/pkg/ruler/store" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "github.com/jonboulle/clockwork" @@ -16,10 +17,13 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" ) -var backoffConfig = util.BackoffConfig{ - // Backoff for loading initial configuration set. - MinBackoff: 100 * time.Millisecond, - MaxBackoff: 2 * time.Second, +type Scheduler interface { + Next() *WorkItem + Done(WorkItem) +} + +type WorkItem interface { + Evaluate(context.Context) } const ( @@ -52,7 +56,7 @@ var ( type workItem struct { userID string - groupName string + groupID string hash uint32 group *wrappedGroup scheduled time.Time @@ -62,7 +66,7 @@ type workItem struct { // Key implements ScheduledItem func (w workItem) Key() string { - return w.userID + ":" + w.groupName + return w.userID + ":" + w.groupID } // Scheduled implements ScheduledItem @@ -71,19 +75,19 @@ func (w workItem) Scheduled() time.Time { } func (w workItem) String() string { - return fmt.Sprintf("%s:%s:%d@%s", w.userID, w.groupName, len(w.group.Rules()), w.scheduled.Format(timeLogFormat)) + return fmt.Sprintf("%s:%s:%d@%s", w.userID, w.groupID, len(w.group.Rules()), w.scheduled.Format(timeLogFormat)) } type userConfig struct { done chan struct{} id string - rules []RuleGroup + rules []store.RuleGroup } -type groupFactory func(context.Context, RuleGroup) (*wrappedGroup, error) +type groupFactory func(context.Context, store.RuleGroup) (*wrappedGroup, error) type scheduler struct { - store RulePoller + poller store.RulePoller evaluationInterval time.Duration // how often we re-evaluate each rule set q *SchedulingQueue @@ -96,9 +100,9 @@ type scheduler struct { } // newScheduler makes a new scheduler. -func newScheduler(store RulePoller, evaluationInterval, pollInterval time.Duration, groupFn groupFactory) *scheduler { +func newScheduler(poller store.RulePoller, evaluationInterval, pollInterval time.Duration, groupFn groupFactory) *scheduler { return &scheduler{ - store: store, + poller: poller, evaluationInterval: evaluationInterval, pollInterval: pollInterval, q: NewSchedulingQueue(clockwork.NewRealClock()), @@ -135,13 +139,14 @@ func (s *scheduler) Run() { } func (s *scheduler) Stop() { + s.poller.Stop() close(s.done) s.q.Close() level.Debug(util.Logger).Log("msg", "scheduler stopped") } func (s *scheduler) updateConfigs(ctx context.Context) error { - cfgs, err := s.store.PollRules(ctx) + cfgs, err := s.poller.PollRules(ctx) if err != nil { return err } @@ -154,7 +159,7 @@ func (s *scheduler) updateConfigs(ctx context.Context) error { return nil } -func (s *scheduler) addUserConfig(ctx context.Context, userID string, rgs []RuleGroup) { +func (s *scheduler) addUserConfig(ctx context.Context, userID string, rgs []store.RuleGroup) { level.Info(util.Logger).Log("msg", "scheduler: updating rules for user", "user_id", userID, "num_groups", len(rgs)) // create a new userchan for rulegroups of this user @@ -165,17 +170,17 @@ func (s *scheduler) addUserConfig(ctx context.Context, userID string, rgs []Rule evalTime := s.determineEvalTime(userID) for _, rg := range rgs { - level.Debug(util.Logger).Log("msg", "scheduler: updating rules for user and group", "user_id", userID, "group", rg.Name()) + level.Debug(util.Logger).Log("msg", "scheduler: updating rules for user and group", "user_id", userID, "group", rg.ID()) grp, err := s.groupFn(ctx, rg) if err != nil { - level.Error(util.Logger).Log("msg", "scheduler: failed to create group for user", "user_id", userID, "group", rg.Name(), "err", err) + level.Error(util.Logger).Log("msg", "scheduler: failed to create group for user", "user_id", userID, "group", rg.ID(), "err", err) return } ringHasher.Reset() - ringHasher.Write([]byte(rg.Name())) + ringHasher.Write([]byte(rg.ID())) hash := ringHasher.Sum32() - workItems = append(workItems, workItem{userID, rg.Name(), hash, grp, evalTime, userChan}) + workItems = append(workItems, workItem{userID, rg.ID(), hash, grp, evalTime, userChan}) } s.updateUserConfig(ctx, userConfig{ @@ -233,7 +238,7 @@ func (s *scheduler) addWorkItem(i workItem) { level.Debug(util.Logger).Log("msg", "scheduler: work item not added, scheduler stoped", "item", i) return default: - // The queue is keyed by userID+groupName, so items for existing userID+groupName will be replaced. + // The queue is keyed by userID+groupID, so items for existing userID+groupID will be replaced. s.q.Enqueue(i) level.Debug(util.Logger).Log("msg", "scheduler: work item added", "item", i) } diff --git a/pkg/ruler/scheduler_test.go b/pkg/ruler/scheduler_test.go index d7b6cdca4cc..2628c27f1cf 100644 --- a/pkg/ruler/scheduler_test.go +++ b/pkg/ruler/scheduler_test.go @@ -6,14 +6,13 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/pkg/ruler/group" - "github.com/prometheus/prometheus/rules" + "github.com/cortexproject/cortex/pkg/ruler/store" + "github.com/prometheus/prometheus/pkg/rulefmt" "github.com/stretchr/testify/assert" ) type fakeHasher struct { - something uint32 - data *[]byte + data *[]byte } func (h *fakeHasher) Write(data []byte) (int, error) { @@ -76,12 +75,12 @@ func TestSchedulerRulesOverlap(t *testing.T) { groupTwo := "test2" next := time.Now() - ruleSetsOne := []RuleGroup{ - group.NewRuleGroup(groupOne, "default", userID, []rules.Rule{nil}), + ruleSetsOne := []store.RuleGroup{ + store.NewRuleGroup(groupOne, "default", userID, []rulefmt.Rule{}), } - ruleSetsTwo := []RuleGroup{ - group.NewRuleGroup(groupTwo, "default", userID, []rules.Rule{nil}), + ruleSetsTwo := []store.RuleGroup{ + store.NewRuleGroup(groupTwo, "default", userID, []rulefmt.Rule{}), } userChanOne := make(chan struct{}) userChanTwo := make(chan struct{}) @@ -90,13 +89,13 @@ func TestSchedulerRulesOverlap(t *testing.T) { cfgTwo := userConfig{rules: ruleSetsTwo, done: userChanTwo} s.updateUserConfig(context.Background(), cfgOne) - w0 := workItem{userID: userID, groupName: groupOne, scheduled: next, done: userChanOne} + w0 := workItem{userID: userID, groupID: groupOne, scheduled: next, done: userChanOne} s.workItemDone(w0) item := s.q.Dequeue().(workItem) - assert.Equal(t, item.groupName, groupOne) + assert.Equal(t, item.groupID, groupOne) // create a new workitem for the updated ruleset - w1 := workItem{userID: userID, groupName: groupTwo, scheduled: next, done: userChanTwo} + w1 := workItem{userID: userID, groupID: groupTwo, scheduled: next, done: userChanTwo} // Apply the new config, scheduling the previous config to be dropped s.updateUserConfig(context.Background(), cfgTwo) @@ -108,7 +107,7 @@ func TestSchedulerRulesOverlap(t *testing.T) { // Ensure the old config was dropped due to the done channel being closed // when the new user config was updated item = s.q.Dequeue().(workItem) - assert.Equal(t, item.groupName, groupTwo) + assert.Equal(t, item.groupID, groupTwo) s.q.Close() assert.Equal(t, nil, s.q.Dequeue()) diff --git a/pkg/ruler/storage.go b/pkg/ruler/storage.go new file mode 100644 index 00000000000..a5268a70efe --- /dev/null +++ b/pkg/ruler/storage.go @@ -0,0 +1,60 @@ +package ruler + +import ( + "context" + "flag" + "fmt" + + "github.com/cortexproject/cortex/pkg/ruler/store" + "github.com/cortexproject/cortex/pkg/storage/clients/configdb" + "github.com/cortexproject/cortex/pkg/storage/clients/gcp" + "github.com/cortexproject/cortex/pkg/util/usertracker" +) + +// RuleStoreConfig conigures a rule store +type RuleStoreConfig struct { + Type string `yaml:"type"` + ConfigDB configdb.Config + GCS gcp.GCSConfig + Tracker usertracker.Config + + mock *mockRuleStore +} + +// RegisterFlags registers flags. +func (cfg *RuleStoreConfig) RegisterFlags(f *flag.FlagSet) { + cfg.ConfigDB.RegisterFlagsWithPrefix("ruler", f) + cfg.GCS.RegisterFlagsWithPrefix("ruler.store.", f) + cfg.Tracker.RegisterFlagsWithPrefix("ruler.", f) + f.StringVar(&cfg.Type, "ruler.storage.type", "configdb", "Method to use for backend rule storage (configdb, gcs)") +} + +// NewRuleStorage returns a new rule storage backend poller and store +func NewRuleStorage(cfg RuleStoreConfig) (store.RulePoller, store.RuleStore, error) { + if cfg.mock != nil { + return cfg.mock, cfg.mock, nil + } + + var ( + ruleStore store.RuleStore + err error + ) + switch cfg.Type { + case "configdb": + poller, err := configdb.New(cfg.ConfigDB) + return poller, nil, err + case "gcs": + ruleStore, err = gcp.NewGCSClient(context.Background(), cfg.GCS) + default: + return nil, nil, fmt.Errorf("Unrecognized rule storage mode %v, choose one of: configdb, gcs", cfg.Type) + } + + tracker, err := usertracker.NewTracker(cfg.Tracker) + if err != nil { + return nil, nil, err + } + + p, err := newTrackedPoller(tracker, ruleStore) + + return p, p.trackedRuleStore(), err +} diff --git a/pkg/ruler/store/compat.go b/pkg/ruler/store/compat.go new file mode 100644 index 00000000000..596f9f0f45b --- /dev/null +++ b/pkg/ruler/store/compat.go @@ -0,0 +1,98 @@ +package store + +import ( + time "time" + + "github.com/cortexproject/cortex/pkg/ingester/client" + + "github.com/golang/protobuf/proto" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/rulefmt" + "github.com/prometheus/prometheus/rules" +) + +// ProtoRuleUpdateDescFactory makes new RuleUpdateDesc +func ProtoRuleUpdateDescFactory() proto.Message { + return NewRuleUpdateDesc() +} + +// NewRuleUpdateDesc returns an empty *distributor.RuleUpdateDesc. +func NewRuleUpdateDesc() *RuleUpdateDesc { + return &RuleUpdateDesc{} +} + +// ToProto transforms a formatted prometheus rulegroup to a rule group protobuf +func ToProto(user string, namespace string, rl rulefmt.RuleGroup) RuleGroupDesc { + dur := time.Duration(rl.Interval) + rg := RuleGroupDesc{ + Name: rl.Name, + Namespace: namespace, + Interval: &dur, + Rules: formattedRuleToProto(rl.Rules), + User: user, + } + return rg +} + +func formattedRuleToProto(rls []rulefmt.Rule) []*Rule { + rules := make([]*Rule, len(rls)) + for i := range rls { + f := time.Duration(rls[i].For) + + rules[i] = &Rule{ + Expr: rls[i].Expr, + Record: rls[i].Record, + Alert: rls[i].Alert, + + For: &f, + Labels: client.FromLabelsToLabelAdapaters(labels.FromMap(rls[i].Labels)), + Annotations: client.FromLabelsToLabelAdapaters(labels.FromMap(rls[i].Labels)), + } + } + + return rules +} + +// FromProto generates a rulefmt RuleGroup +func FromProto(rg *RuleGroupDesc) *rulefmt.RuleGroup { + formattedRuleGroup := rulefmt.RuleGroup{ + Name: rg.GetName(), + Interval: model.Duration(*rg.Interval), + Rules: make([]rulefmt.Rule, len(rg.GetRules())), + } + + for i, rl := range rg.GetRules() { + formattedRuleGroup.Rules[i] = rulefmt.Rule{ + Record: rl.GetRecord(), + Alert: rl.GetAlert(), + Expr: rl.GetExpr(), + For: model.Duration(*rl.GetFor()), + Labels: client.FromLabelAdaptersToLabels(rl.Labels).Map(), + Annotations: client.FromLabelAdaptersToLabels(rl.Annotations).Map(), + } + } + + return &formattedRuleGroup +} + +// ToRuleGroup returns a functional rulegroup from a proto +func ToRuleGroup(rg *RuleGroupDesc) *Group { + return &Group{ + name: rg.GetName(), + namespace: rg.GetNamespace(), + user: rg.GetUser(), + interval: *rg.Interval, + rules: rg.Rules, + } +} + +// FormattedToRuleGroup transforms a formatted prometheus rulegroup to a rule group protobuf +func FormattedToRuleGroup(user string, namespace string, name string, rls []rules.Rule) *Group { + return &Group{ + name: name, + namespace: namespace, + user: user, + activeRules: rls, + } +} diff --git a/pkg/ruler/store/group.go b/pkg/ruler/store/group.go new file mode 100644 index 00000000000..db201d41eaf --- /dev/null +++ b/pkg/ruler/store/group.go @@ -0,0 +1,110 @@ +package store + +import ( + "context" + time "time" + + "github.com/cortexproject/cortex/pkg/ingester/client" + "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/rulefmt" + "github.com/prometheus/prometheus/promql" + "github.com/prometheus/prometheus/rules" +) + +// TODO: Add a lazy rule group that only loads rules when they are needed +// TODO: The cortex project should implement a separate Group struct from +// the prometheus project. This will allow for more precise instrumentation + +type Group struct { + name string + namespace string + user string + interval time.Duration + rules []*Rule + + // TODO: Allows for the support of the configdb client + activeRules []rules.Rule +} + +func (g *Group) Rules(ctx context.Context) ([]rules.Rule, error) { + // Used to be compatible with configdb client + if g.rules == nil && g.activeRules != nil { + return g.activeRules, nil + } + + rls := make([]rules.Rule, 0, len(g.rules)) + for _, rl := range g.rules { + expr, err := promql.ParseExpr(rl.GetExpr()) + if err != nil { + return nil, err + } + + if rl.Alert != "" { + rls = append(rls, rules.NewAlertingRule( + rl.Alert, + expr, + *rl.GetFor(), + client.FromLabelAdaptersToLabels(rl.Labels), + client.FromLabelAdaptersToLabels(rl.Annotations), + true, + log.With(util.Logger, "alert", rl.Alert), + )) + continue + } + rls = append(rls, rules.NewRecordingRule( + rl.Record, + expr, + client.FromLabelAdaptersToLabels(rl.Labels), + )) + } + return rls, nil +} + +func (g *Group) ID() string { + return g.namespace + "/" + g.name +} + +func (g *Group) Name() string { + return g.name +} + +func (g *Group) Namespace() string { + return g.namespace +} + +func (g *Group) User() string { + return g.user +} + +func (g *Group) Formatted() rulefmt.RuleGroup { + formattedRuleGroup := rulefmt.RuleGroup{ + Name: g.name, + Interval: model.Duration(g.interval), + Rules: make([]rulefmt.Rule, len(g.rules)), + } + + for i, rl := range g.rules { + formattedRuleGroup.Rules[i] = rulefmt.Rule{ + Record: rl.GetRecord(), + Alert: rl.GetAlert(), + Expr: rl.GetExpr(), + For: model.Duration(*rl.GetFor()), + Labels: client.FromLabelAdaptersToLabels(rl.Labels).Map(), + Annotations: client.FromLabelAdaptersToLabels(rl.Annotations).Map(), + } + } + + return formattedRuleGroup +} + +// NewGroup returns a Group +func NewRuleGroup(name, namespace, user string, rls []rulefmt.Rule) *Group { + return &Group{ + name: name, + namespace: namespace, + user: user, + rules: formattedRuleToProto(rls), + } +} diff --git a/pkg/ruler/rule_store.go b/pkg/ruler/store/store.go similarity index 81% rename from pkg/ruler/rule_store.go rename to pkg/ruler/store/store.go index 3822613bee5..1a193a5ff99 100644 --- a/pkg/ruler/rule_store.go +++ b/pkg/ruler/store/store.go @@ -1,4 +1,4 @@ -package ruler +package store import ( "context" @@ -15,6 +15,12 @@ var ( ErrUserNotFound = errors.New("no rule groups found for user") ) +// RulePoller is used to poll for recently updated rules +type RulePoller interface { + PollRules(ctx context.Context) (map[string][]RuleGroup, error) + Stop() +} + // RuleStoreConditions are used to filter retrieived results from a rule store type RuleStoreConditions struct { // UserID specifies to only retrieve rules with this ID @@ -25,20 +31,10 @@ type RuleStoreConditions struct { Namespace string } -type RulePoller interface { - PollRules(ctx context.Context) (map[string][]RuleGroup, error) - - // RuleStore returns the rule store client used by the poller, this allows a Poller - // to be used for scheduling, and an associated rule store to be used by the API. - RuleStore() RuleStore -} - // RuleStore is used to store and retrieve rules type RuleStore interface { - RulePoller - - ListRuleGroups(ctx context.Context, options RuleStoreConditions) (map[string]RuleNamespace, error) - GetRuleGroup(ctx context.Context, userID, namespace, group string) (*rulefmt.RuleGroup, error) + ListRuleGroups(ctx context.Context, options RuleStoreConditions) (RuleGroupList, error) + GetRuleGroup(ctx context.Context, userID, namespace, group string) (RuleGroup, error) SetRuleGroup(ctx context.Context, userID, namespace string, group rulefmt.RuleGroup) error DeleteRuleGroup(ctx context.Context, userID, namespace string, group string) error } @@ -47,8 +43,29 @@ type RuleStore interface { // an interface is used to allow for lazy evaluation implementations type RuleGroup interface { Rules(ctx context.Context) ([]rules.Rule, error) + ID() string Name() string + Namespace() string User() string + Formatted() rulefmt.RuleGroup +} + +type RuleGroupList []RuleGroup + +func (l RuleGroupList) Formatted(user string) map[string][]rulefmt.RuleGroup { + ruleMap := map[string][]rulefmt.RuleGroup{} + for _, g := range l { + if g.User() != user { + continue + } + + if _, exists := ruleMap[g.Namespace()]; !exists { + ruleMap[g.Namespace()] = []rulefmt.RuleGroup{g.Formatted()} + } + ruleMap[g.Namespace()] = append(ruleMap[g.Namespace()], g.Formatted()) + + } + return ruleMap } // RuleNamespace is used to parse a slightly modified prometheus diff --git a/pkg/ruler/group/rulegroup.pb.go b/pkg/ruler/store/store.pb.go similarity index 59% rename from pkg/ruler/group/rulegroup.pb.go rename to pkg/ruler/store/store.pb.go index 49317a79d0f..55c553da559 100644 --- a/pkg/ruler/group/rulegroup.pb.go +++ b/pkg/ruler/store/store.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: rulegroup.proto +// source: store.proto -package group +package store import ( fmt "fmt" @@ -30,25 +30,77 @@ var _ = time.Kitchen // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package -type RuleGroup struct { +type RuleUpdateDesc struct { + User string `protobuf:"bytes,1,opt,name=user,proto3" json:"user,omitempty"` + UpdatedAt int64 `protobuf:"varint,2,opt,name=updatedAt,proto3" json:"updatedAt,omitempty"` +} + +func (m *RuleUpdateDesc) Reset() { *m = RuleUpdateDesc{} } +func (*RuleUpdateDesc) ProtoMessage() {} +func (*RuleUpdateDesc) Descriptor() ([]byte, []int) { + return fileDescriptor_98bbca36ef968dfc, []int{0} +} +func (m *RuleUpdateDesc) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RuleUpdateDesc) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_RuleUpdateDesc.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *RuleUpdateDesc) XXX_Merge(src proto.Message) { + xxx_messageInfo_RuleUpdateDesc.Merge(m, src) +} +func (m *RuleUpdateDesc) XXX_Size() int { + return m.Size() +} +func (m *RuleUpdateDesc) XXX_DiscardUnknown() { + xxx_messageInfo_RuleUpdateDesc.DiscardUnknown(m) +} + +var xxx_messageInfo_RuleUpdateDesc proto.InternalMessageInfo + +func (m *RuleUpdateDesc) GetUser() string { + if m != nil { + return m.User + } + return "" +} + +func (m *RuleUpdateDesc) GetUpdatedAt() int64 { + if m != nil { + return m.UpdatedAt + } + return 0 +} + +type RuleGroupDesc struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Namespace string `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` Interval *time.Duration `protobuf:"bytes,3,opt,name=interval,proto3,stdduration" json:"interval,omitempty"` Rules []*Rule `protobuf:"bytes,4,rep,name=rules,proto3" json:"rules,omitempty"` Deleted bool `protobuf:"varint,5,opt,name=deleted,proto3" json:"deleted,omitempty"` + User string `protobuf:"bytes,6,opt,name=user,proto3" json:"user,omitempty"` } -func (m *RuleGroup) Reset() { *m = RuleGroup{} } -func (*RuleGroup) ProtoMessage() {} -func (*RuleGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_3324c775b6f77a4a, []int{0} +func (m *RuleGroupDesc) Reset() { *m = RuleGroupDesc{} } +func (*RuleGroupDesc) ProtoMessage() {} +func (*RuleGroupDesc) Descriptor() ([]byte, []int) { + return fileDescriptor_98bbca36ef968dfc, []int{1} } -func (m *RuleGroup) XXX_Unmarshal(b []byte) error { +func (m *RuleGroupDesc) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *RuleGroup) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *RuleGroupDesc) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_RuleGroup.Marshal(b, m, deterministic) + return xxx_messageInfo_RuleGroupDesc.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalTo(b) @@ -58,53 +110,60 @@ func (m *RuleGroup) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return b[:n], nil } } -func (m *RuleGroup) XXX_Merge(src proto.Message) { - xxx_messageInfo_RuleGroup.Merge(m, src) +func (m *RuleGroupDesc) XXX_Merge(src proto.Message) { + xxx_messageInfo_RuleGroupDesc.Merge(m, src) } -func (m *RuleGroup) XXX_Size() int { +func (m *RuleGroupDesc) XXX_Size() int { return m.Size() } -func (m *RuleGroup) XXX_DiscardUnknown() { - xxx_messageInfo_RuleGroup.DiscardUnknown(m) +func (m *RuleGroupDesc) XXX_DiscardUnknown() { + xxx_messageInfo_RuleGroupDesc.DiscardUnknown(m) } -var xxx_messageInfo_RuleGroup proto.InternalMessageInfo +var xxx_messageInfo_RuleGroupDesc proto.InternalMessageInfo -func (m *RuleGroup) GetName() string { +func (m *RuleGroupDesc) GetName() string { if m != nil { return m.Name } return "" } -func (m *RuleGroup) GetNamespace() string { +func (m *RuleGroupDesc) GetNamespace() string { if m != nil { return m.Namespace } return "" } -func (m *RuleGroup) GetInterval() *time.Duration { +func (m *RuleGroupDesc) GetInterval() *time.Duration { if m != nil { return m.Interval } return nil } -func (m *RuleGroup) GetRules() []*Rule { +func (m *RuleGroupDesc) GetRules() []*Rule { if m != nil { return m.Rules } return nil } -func (m *RuleGroup) GetDeleted() bool { +func (m *RuleGroupDesc) GetDeleted() bool { if m != nil { return m.Deleted } return false } +func (m *RuleGroupDesc) GetUser() string { + if m != nil { + return m.User + } + return "" +} + type Rule struct { Expr string `protobuf:"bytes,1,opt,name=expr,proto3" json:"expr,omitempty"` Record string `protobuf:"bytes,2,opt,name=record,proto3" json:"record,omitempty"` @@ -117,7 +176,7 @@ type Rule struct { func (m *Rule) Reset() { *m = Rule{} } func (*Rule) ProtoMessage() {} func (*Rule) Descriptor() ([]byte, []int) { - return fileDescriptor_3324c775b6f77a4a, []int{1} + return fileDescriptor_98bbca36ef968dfc, []int{2} } func (m *Rule) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -175,52 +234,82 @@ func (m *Rule) GetFor() *time.Duration { } func init() { - proto.RegisterType((*RuleGroup)(nil), "group.RuleGroup") - proto.RegisterType((*Rule)(nil), "group.Rule") + proto.RegisterType((*RuleUpdateDesc)(nil), "store.RuleUpdateDesc") + proto.RegisterType((*RuleGroupDesc)(nil), "store.RuleGroupDesc") + proto.RegisterType((*Rule)(nil), "store.Rule") } -func init() { proto.RegisterFile("rulegroup.proto", fileDescriptor_3324c775b6f77a4a) } +func init() { proto.RegisterFile("store.proto", fileDescriptor_98bbca36ef968dfc) } -var fileDescriptor_3324c775b6f77a4a = []byte{ - // 436 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x92, 0xb1, 0x6e, 0xd4, 0x30, - 0x18, 0xc7, 0xe3, 0x5e, 0x72, 0xf4, 0x7c, 0x03, 0xc2, 0x42, 0xc8, 0x54, 0xc8, 0x17, 0x3a, 0x65, - 0x21, 0x11, 0x65, 0xec, 0x02, 0x27, 0x24, 0x18, 0x18, 0x50, 0x46, 0x36, 0x27, 0xf9, 0x6a, 0x02, - 0x6e, 0x1c, 0x39, 0x0e, 0x62, 0x41, 0xe2, 0x11, 0x18, 0x79, 0x04, 0xde, 0x80, 0x57, 0xe8, 0x78, - 0x63, 0xc5, 0x50, 0xb8, 0xdc, 0xc2, 0x46, 0x1f, 0x01, 0xd9, 0x4e, 0x68, 0x47, 0x84, 0xc4, 0x94, - 0xef, 0xef, 0xff, 0xe7, 0xef, 0xfb, 0xe5, 0x9f, 0xe0, 0x9b, 0xba, 0x97, 0x20, 0xb4, 0xea, 0xdb, - 0xb4, 0xd5, 0xca, 0x28, 0x12, 0x39, 0x71, 0xf0, 0x40, 0xd4, 0xe6, 0x75, 0x5f, 0xa4, 0xa5, 0x3a, - 0xcd, 0x84, 0x12, 0x2a, 0x73, 0x6e, 0xd1, 0x9f, 0x38, 0xe5, 0x84, 0xab, 0xfc, 0xad, 0x03, 0x26, - 0x94, 0x12, 0x12, 0xae, 0xba, 0xaa, 0x5e, 0x73, 0x53, 0xab, 0x66, 0xf4, 0x1f, 0x5f, 0x1b, 0x57, - 0x2a, 0x6d, 0xe0, 0x7d, 0xab, 0xd5, 0x1b, 0x28, 0xcd, 0xa8, 0xb2, 0xf6, 0xad, 0xc8, 0xea, 0x46, - 0x40, 0x67, 0x40, 0x67, 0xa5, 0xac, 0xa1, 0x99, 0x2c, 0x3f, 0xe1, 0xf0, 0x2b, 0xc2, 0x8b, 0xbc, - 0x97, 0xf0, 0xcc, 0xe2, 0x11, 0x82, 0xc3, 0x86, 0x9f, 0x02, 0x45, 0x31, 0x4a, 0x16, 0xb9, 0xab, - 0xc9, 0x3d, 0xbc, 0xb0, 0xcf, 0xae, 0xe5, 0x25, 0xd0, 0x3d, 0x67, 0x5c, 0x1d, 0x90, 0x63, 0xbc, - 0x5f, 0x37, 0x06, 0xf4, 0x3b, 0x2e, 0xe9, 0x2c, 0x46, 0xc9, 0xf2, 0xe8, 0x6e, 0xea, 0xa1, 0xd3, - 0x09, 0x3a, 0x7d, 0x3a, 0x42, 0xaf, 0xc3, 0xcf, 0xdf, 0x57, 0x28, 0xff, 0x73, 0x81, 0xdc, 0xc7, - 0x91, 0xcd, 0xa9, 0xa3, 0x61, 0x3c, 0x4b, 0x96, 0x47, 0xcb, 0xd4, 0x27, 0x66, 0x79, 0x72, 0xef, - 0x10, 0x8a, 0x6f, 0x54, 0x20, 0xc1, 0x40, 0x45, 0xa3, 0x18, 0x25, 0xfb, 0xf9, 0x24, 0x0f, 0x7f, - 0xed, 0xe1, 0xd0, 0x76, 0x5a, 0x68, 0xfb, 0xde, 0x13, 0xb4, 0xad, 0xc9, 0x1d, 0x3c, 0xd7, 0x50, - 0x2a, 0x5d, 0x8d, 0xc4, 0xa3, 0x22, 0xb7, 0x71, 0xc4, 0x25, 0x68, 0xe3, 0x58, 0x17, 0xb9, 0x17, - 0xe4, 0x21, 0x9e, 0x9d, 0x28, 0x4d, 0xc3, 0xbf, 0xe3, 0xb7, 0xbd, 0xa4, 0xc3, 0x73, 0xc9, 0x0b, - 0x90, 0x1d, 0x8d, 0x1c, 0xfb, 0xad, 0x74, 0x8c, 0xf5, 0x85, 0x3d, 0x7d, 0xc9, 0x6b, 0xbd, 0x7e, - 0x7e, 0x76, 0xb1, 0x0a, 0xbe, 0x5d, 0xac, 0xfe, 0xe5, 0x23, 0xf9, 0x31, 0x4f, 0x2a, 0xde, 0x1a, - 0xd0, 0xf9, 0xb8, 0x8a, 0x7c, 0xc0, 0x4b, 0xde, 0x34, 0xca, 0x38, 0x9a, 0x8e, 0xce, 0xff, 0xff, - 0xe6, 0xeb, 0xfb, 0xd6, 0xc7, 0x9b, 0x2d, 0x0b, 0xce, 0xb7, 0x2c, 0xb8, 0xdc, 0x32, 0xf4, 0x71, - 0x60, 0xe8, 0xcb, 0xc0, 0xd0, 0xd9, 0xc0, 0xd0, 0x66, 0x60, 0xe8, 0xc7, 0xc0, 0xd0, 0xcf, 0x81, - 0x05, 0x97, 0x03, 0x43, 0x9f, 0x76, 0x2c, 0xd8, 0xec, 0x58, 0x70, 0xbe, 0x63, 0xc1, 0x2b, 0xff, - 0xe7, 0x17, 0x73, 0x17, 0xe7, 0xa3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x7f, 0xbc, 0xc2, 0x52, - 0x1a, 0x03, 0x00, 0x00, +var fileDescriptor_98bbca36ef968dfc = []byte{ + // 474 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x53, 0xb1, 0x6e, 0xd4, 0x40, + 0x10, 0xf5, 0xe6, 0x6c, 0x93, 0x5b, 0x0b, 0x24, 0x56, 0x08, 0x99, 0x08, 0xed, 0x99, 0xab, 0xdc, + 0x60, 0x8b, 0x50, 0xa6, 0x21, 0xa7, 0x48, 0x50, 0x50, 0x20, 0x4b, 0x34, 0x74, 0x7b, 0xf6, 0xc4, + 0x18, 0x36, 0x5e, 0x6b, 0x77, 0x8d, 0x68, 0x90, 0xf8, 0x04, 0x4a, 0x3e, 0x81, 0x4f, 0x49, 0xc7, + 0x95, 0x11, 0x45, 0xe0, 0x7c, 0x0d, 0x1d, 0xf9, 0x04, 0xb4, 0x6b, 0x3b, 0x77, 0x25, 0x42, 0x4a, + 0xe5, 0x79, 0xf3, 0x66, 0xde, 0xcc, 0x1b, 0xdb, 0x38, 0x50, 0x5a, 0x48, 0x48, 0x1a, 0x29, 0xb4, + 0x20, 0x9e, 0x05, 0x07, 0x8f, 0xcb, 0x4a, 0xbf, 0x6d, 0x97, 0x49, 0x2e, 0xce, 0xd2, 0x52, 0x94, + 0x22, 0xb5, 0xec, 0xb2, 0x3d, 0xb5, 0xc8, 0x02, 0x1b, 0xf5, 0x5d, 0x07, 0xb4, 0x14, 0xa2, 0xe4, + 0xb0, 0xad, 0x2a, 0x5a, 0xc9, 0x74, 0x25, 0xea, 0x81, 0x7f, 0xb6, 0x23, 0x97, 0x0b, 0xa9, 0xe1, + 0x63, 0x23, 0xc5, 0x3b, 0xc8, 0xf5, 0x80, 0xd2, 0xe6, 0x7d, 0x99, 0x56, 0x75, 0x09, 0x4a, 0x83, + 0x4c, 0x73, 0x5e, 0x41, 0x3d, 0x52, 0xbd, 0xc2, 0x7c, 0x81, 0xef, 0x64, 0x2d, 0x87, 0xd7, 0x4d, + 0xc1, 0x34, 0x9c, 0x80, 0xca, 0x09, 0xc1, 0x6e, 0xab, 0x40, 0x86, 0x28, 0x42, 0xf1, 0x34, 0xb3, + 0x31, 0x79, 0x88, 0xa7, 0xad, 0xad, 0x28, 0x8e, 0x75, 0xb8, 0x17, 0xa1, 0x78, 0x92, 0x6d, 0x13, + 0xf3, 0xef, 0x08, 0xdf, 0x36, 0x22, 0xcf, 0xa5, 0x68, 0x9b, 0x51, 0xa3, 0x66, 0x67, 0x30, 0x6a, + 0x98, 0xd8, 0x68, 0x98, 0xa7, 0x6a, 0x58, 0x0e, 0x56, 0x63, 0x9a, 0x6d, 0x13, 0xe4, 0x08, 0xef, + 0x57, 0xb5, 0x06, 0xf9, 0x81, 0xf1, 0x70, 0x12, 0xa1, 0x38, 0x38, 0x7c, 0x90, 0xf4, 0xe6, 0x93, + 0xd1, 0x7c, 0x72, 0x32, 0x98, 0x5f, 0xb8, 0x5f, 0x7f, 0xce, 0x50, 0x76, 0xdd, 0x40, 0x1e, 0x61, + 0x4f, 0xb6, 0x1c, 0x54, 0xe8, 0x46, 0x93, 0x38, 0x38, 0x0c, 0x92, 0xfe, 0xf2, 0x66, 0xa7, 0xac, + 0x67, 0x48, 0x88, 0x6f, 0x15, 0xc0, 0x41, 0x43, 0x11, 0x7a, 0x11, 0x8a, 0xf7, 0xb3, 0x11, 0x5e, + 0xfb, 0xf5, 0xb7, 0x7e, 0xe7, 0x7f, 0xf6, 0xb0, 0x6b, 0xba, 0x0d, 0x69, 0x6e, 0x3a, 0x1a, 0x31, + 0x31, 0xb9, 0x8f, 0x7d, 0x09, 0xb9, 0x90, 0xc5, 0xe0, 0x62, 0x40, 0xe4, 0x1e, 0xf6, 0x18, 0x07, + 0xa9, 0xed, 0xfe, 0xd3, 0xac, 0x07, 0xe4, 0x09, 0x9e, 0x9c, 0x0a, 0x19, 0xba, 0xff, 0xe6, 0xc9, + 0xd4, 0x12, 0x85, 0x7d, 0xce, 0x96, 0xc0, 0x55, 0xe8, 0x59, 0x3f, 0x77, 0x93, 0xe1, 0x95, 0xbd, + 0x34, 0xd9, 0x57, 0xac, 0x92, 0x8b, 0x17, 0xe7, 0x97, 0x33, 0xe7, 0xc7, 0xe5, 0xec, 0x7f, 0x3e, + 0x80, 0x5e, 0xe6, 0xb8, 0x60, 0x8d, 0x06, 0x99, 0x0d, 0xa3, 0xc8, 0x27, 0x1c, 0xb0, 0xba, 0x16, + 0xda, 0x6e, 0xa3, 0x42, 0xff, 0xe6, 0x27, 0xef, 0xce, 0x5b, 0x1c, 0xad, 0xd6, 0xd4, 0xb9, 0x58, + 0x53, 0xe7, 0x6a, 0x4d, 0xd1, 0xe7, 0x8e, 0xa2, 0x6f, 0x1d, 0x45, 0xe7, 0x1d, 0x45, 0xab, 0x8e, + 0xa2, 0x5f, 0x1d, 0x45, 0xbf, 0x3b, 0xea, 0x5c, 0x75, 0x14, 0x7d, 0xd9, 0x50, 0x67, 0xb5, 0xa1, + 0xce, 0xc5, 0x86, 0x3a, 0x6f, 0xfa, 0xbf, 0x6a, 0xe9, 0xdb, 0x73, 0x3e, 0xfd, 0x1b, 0x00, 0x00, + 0xff, 0xff, 0x86, 0x6d, 0x96, 0x1c, 0x72, 0x03, 0x00, 0x00, } -func (this *RuleGroup) Equal(that interface{}) bool { +func (this *RuleUpdateDesc) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*RuleUpdateDesc) + if !ok { + that2, ok := that.(RuleUpdateDesc) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.User != that1.User { + return false + } + if this.UpdatedAt != that1.UpdatedAt { + return false + } + return true +} +func (this *RuleGroupDesc) Equal(that interface{}) bool { if that == nil { return this == nil } - that1, ok := that.(*RuleGroup) + that1, ok := that.(*RuleGroupDesc) if !ok { - that2, ok := that.(RuleGroup) + that2, ok := that.(RuleGroupDesc) if ok { that1 = &that2 } else { @@ -258,6 +347,9 @@ func (this *RuleGroup) Equal(that interface{}) bool { if this.Deleted != that1.Deleted { return false } + if this.User != that1.User { + return false + } return true } func (this *Rule) Equal(that interface{}) bool { @@ -315,12 +407,23 @@ func (this *Rule) Equal(that interface{}) bool { } return true } -func (this *RuleGroup) GoString() string { +func (this *RuleUpdateDesc) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 6) + s = append(s, "&store.RuleUpdateDesc{") + s = append(s, "User: "+fmt.Sprintf("%#v", this.User)+",\n") + s = append(s, "UpdatedAt: "+fmt.Sprintf("%#v", this.UpdatedAt)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func (this *RuleGroupDesc) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 9) - s = append(s, "&group.RuleGroup{") + s := make([]string, 0, 10) + s = append(s, "&store.RuleGroupDesc{") s = append(s, "Name: "+fmt.Sprintf("%#v", this.Name)+",\n") s = append(s, "Namespace: "+fmt.Sprintf("%#v", this.Namespace)+",\n") s = append(s, "Interval: "+fmt.Sprintf("%#v", this.Interval)+",\n") @@ -328,6 +431,7 @@ func (this *RuleGroup) GoString() string { s = append(s, "Rules: "+fmt.Sprintf("%#v", this.Rules)+",\n") } s = append(s, "Deleted: "+fmt.Sprintf("%#v", this.Deleted)+",\n") + s = append(s, "User: "+fmt.Sprintf("%#v", this.User)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -336,7 +440,7 @@ func (this *Rule) GoString() string { return "nil" } s := make([]string, 0, 10) - s = append(s, "&group.Rule{") + s = append(s, "&store.Rule{") s = append(s, "Expr: "+fmt.Sprintf("%#v", this.Expr)+",\n") s = append(s, "Record: "+fmt.Sprintf("%#v", this.Record)+",\n") s = append(s, "Alert: "+fmt.Sprintf("%#v", this.Alert)+",\n") @@ -346,7 +450,7 @@ func (this *Rule) GoString() string { s = append(s, "}") return strings.Join(s, "") } -func valueToGoStringRulegroup(v interface{}, typ string) string { +func valueToGoStringStore(v interface{}, typ string) string { rv := reflect.ValueOf(v) if rv.IsNil() { return "nil" @@ -354,7 +458,36 @@ func valueToGoStringRulegroup(v interface{}, typ string) string { pv := reflect.Indirect(rv).Interface() return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) } -func (m *RuleGroup) Marshal() (dAtA []byte, err error) { +func (m *RuleUpdateDesc) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RuleUpdateDesc) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.User) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintStore(dAtA, i, uint64(len(m.User))) + i += copy(dAtA[i:], m.User) + } + if m.UpdatedAt != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintStore(dAtA, i, uint64(m.UpdatedAt)) + } + return i, nil +} + +func (m *RuleGroupDesc) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -364,7 +497,7 @@ func (m *RuleGroup) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *RuleGroup) MarshalTo(dAtA []byte) (int, error) { +func (m *RuleGroupDesc) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -372,19 +505,19 @@ func (m *RuleGroup) MarshalTo(dAtA []byte) (int, error) { if len(m.Name) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintRulegroup(dAtA, i, uint64(len(m.Name))) + i = encodeVarintStore(dAtA, i, uint64(len(m.Name))) i += copy(dAtA[i:], m.Name) } if len(m.Namespace) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintRulegroup(dAtA, i, uint64(len(m.Namespace))) + i = encodeVarintStore(dAtA, i, uint64(len(m.Namespace))) i += copy(dAtA[i:], m.Namespace) } if m.Interval != nil { dAtA[i] = 0x1a i++ - i = encodeVarintRulegroup(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdDuration(*m.Interval))) + i = encodeVarintStore(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdDuration(*m.Interval))) n1, err := github_com_gogo_protobuf_types.StdDurationMarshalTo(*m.Interval, dAtA[i:]) if err != nil { return 0, err @@ -395,7 +528,7 @@ func (m *RuleGroup) MarshalTo(dAtA []byte) (int, error) { for _, msg := range m.Rules { dAtA[i] = 0x22 i++ - i = encodeVarintRulegroup(dAtA, i, uint64(msg.Size())) + i = encodeVarintStore(dAtA, i, uint64(msg.Size())) n, err := msg.MarshalTo(dAtA[i:]) if err != nil { return 0, err @@ -413,6 +546,12 @@ func (m *RuleGroup) MarshalTo(dAtA []byte) (int, error) { } i++ } + if len(m.User) > 0 { + dAtA[i] = 0x32 + i++ + i = encodeVarintStore(dAtA, i, uint64(len(m.User))) + i += copy(dAtA[i:], m.User) + } return i, nil } @@ -434,25 +573,25 @@ func (m *Rule) MarshalTo(dAtA []byte) (int, error) { if len(m.Expr) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintRulegroup(dAtA, i, uint64(len(m.Expr))) + i = encodeVarintStore(dAtA, i, uint64(len(m.Expr))) i += copy(dAtA[i:], m.Expr) } if len(m.Record) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintRulegroup(dAtA, i, uint64(len(m.Record))) + i = encodeVarintStore(dAtA, i, uint64(len(m.Record))) i += copy(dAtA[i:], m.Record) } if len(m.Alert) > 0 { dAtA[i] = 0x1a i++ - i = encodeVarintRulegroup(dAtA, i, uint64(len(m.Alert))) + i = encodeVarintStore(dAtA, i, uint64(len(m.Alert))) i += copy(dAtA[i:], m.Alert) } if m.For != nil { dAtA[i] = 0x22 i++ - i = encodeVarintRulegroup(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdDuration(*m.For))) + i = encodeVarintStore(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdDuration(*m.For))) n2, err := github_com_gogo_protobuf_types.StdDurationMarshalTo(*m.For, dAtA[i:]) if err != nil { return 0, err @@ -463,7 +602,7 @@ func (m *Rule) MarshalTo(dAtA []byte) (int, error) { for _, msg := range m.Labels { dAtA[i] = 0x2a i++ - i = encodeVarintRulegroup(dAtA, i, uint64(msg.Size())) + i = encodeVarintStore(dAtA, i, uint64(msg.Size())) n, err := msg.MarshalTo(dAtA[i:]) if err != nil { return 0, err @@ -475,7 +614,7 @@ func (m *Rule) MarshalTo(dAtA []byte) (int, error) { for _, msg := range m.Annotations { dAtA[i] = 0x32 i++ - i = encodeVarintRulegroup(dAtA, i, uint64(msg.Size())) + i = encodeVarintStore(dAtA, i, uint64(msg.Size())) n, err := msg.MarshalTo(dAtA[i:]) if err != nil { return 0, err @@ -486,7 +625,7 @@ func (m *Rule) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeVarintRulegroup(dAtA []byte, offset int, v uint64) int { +func encodeVarintStore(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 @@ -495,7 +634,23 @@ func encodeVarintRulegroup(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return offset + 1 } -func (m *RuleGroup) Size() (n int) { +func (m *RuleUpdateDesc) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.User) + if l > 0 { + n += 1 + l + sovStore(uint64(l)) + } + if m.UpdatedAt != 0 { + n += 1 + sovStore(uint64(m.UpdatedAt)) + } + return n +} + +func (m *RuleGroupDesc) Size() (n int) { if m == nil { return 0 } @@ -503,25 +658,29 @@ func (m *RuleGroup) Size() (n int) { _ = l l = len(m.Name) if l > 0 { - n += 1 + l + sovRulegroup(uint64(l)) + n += 1 + l + sovStore(uint64(l)) } l = len(m.Namespace) if l > 0 { - n += 1 + l + sovRulegroup(uint64(l)) + n += 1 + l + sovStore(uint64(l)) } if m.Interval != nil { l = github_com_gogo_protobuf_types.SizeOfStdDuration(*m.Interval) - n += 1 + l + sovRulegroup(uint64(l)) + n += 1 + l + sovStore(uint64(l)) } if len(m.Rules) > 0 { for _, e := range m.Rules { l = e.Size() - n += 1 + l + sovRulegroup(uint64(l)) + n += 1 + l + sovStore(uint64(l)) } } if m.Deleted { n += 2 } + l = len(m.User) + if l > 0 { + n += 1 + l + sovStore(uint64(l)) + } return n } @@ -533,36 +692,36 @@ func (m *Rule) Size() (n int) { _ = l l = len(m.Expr) if l > 0 { - n += 1 + l + sovRulegroup(uint64(l)) + n += 1 + l + sovStore(uint64(l)) } l = len(m.Record) if l > 0 { - n += 1 + l + sovRulegroup(uint64(l)) + n += 1 + l + sovStore(uint64(l)) } l = len(m.Alert) if l > 0 { - n += 1 + l + sovRulegroup(uint64(l)) + n += 1 + l + sovStore(uint64(l)) } if m.For != nil { l = github_com_gogo_protobuf_types.SizeOfStdDuration(*m.For) - n += 1 + l + sovRulegroup(uint64(l)) + n += 1 + l + sovStore(uint64(l)) } if len(m.Labels) > 0 { for _, e := range m.Labels { l = e.Size() - n += 1 + l + sovRulegroup(uint64(l)) + n += 1 + l + sovStore(uint64(l)) } } if len(m.Annotations) > 0 { for _, e := range m.Annotations { l = e.Size() - n += 1 + l + sovRulegroup(uint64(l)) + n += 1 + l + sovStore(uint64(l)) } } return n } -func sovRulegroup(x uint64) (n int) { +func sovStore(x uint64) (n int) { for { n++ x >>= 7 @@ -572,19 +731,31 @@ func sovRulegroup(x uint64) (n int) { } return n } -func sozRulegroup(x uint64) (n int) { - return sovRulegroup(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +func sozStore(x uint64) (n int) { + return sovStore(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *RuleUpdateDesc) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&RuleUpdateDesc{`, + `User:` + fmt.Sprintf("%v", this.User) + `,`, + `UpdatedAt:` + fmt.Sprintf("%v", this.UpdatedAt) + `,`, + `}`, + }, "") + return s } -func (this *RuleGroup) String() string { +func (this *RuleGroupDesc) String() string { if this == nil { return "nil" } - s := strings.Join([]string{`&RuleGroup{`, + s := strings.Join([]string{`&RuleGroupDesc{`, `Name:` + fmt.Sprintf("%v", this.Name) + `,`, `Namespace:` + fmt.Sprintf("%v", this.Namespace) + `,`, `Interval:` + strings.Replace(fmt.Sprintf("%v", this.Interval), "Duration", "duration.Duration", 1) + `,`, `Rules:` + strings.Replace(fmt.Sprintf("%v", this.Rules), "Rule", "Rule", 1) + `,`, `Deleted:` + fmt.Sprintf("%v", this.Deleted) + `,`, + `User:` + fmt.Sprintf("%v", this.User) + `,`, `}`, }, "") return s @@ -604,7 +775,7 @@ func (this *Rule) String() string { }, "") return s } -func valueToStringRulegroup(v interface{}) string { +func valueToStringStore(v interface{}) string { rv := reflect.ValueOf(v) if rv.IsNil() { return "nil" @@ -612,7 +783,111 @@ func valueToStringRulegroup(v interface{}) string { pv := reflect.Indirect(rv).Interface() return fmt.Sprintf("*%v", pv) } -func (m *RuleGroup) Unmarshal(dAtA []byte) error { +func (m *RuleUpdateDesc) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStore + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RuleUpdateDesc: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RuleUpdateDesc: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field User", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStore + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthStore + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthStore + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.User = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdatedAt", wireType) + } + m.UpdatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStore + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.UpdatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipStore(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthStore + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthStore + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -620,7 +895,7 @@ func (m *RuleGroup) Unmarshal(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowRulegroup + return ErrIntOverflowStore } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -635,10 +910,10 @@ func (m *RuleGroup) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: RuleGroup: wiretype end group for non-group") + return fmt.Errorf("proto: RuleGroupDesc: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: RuleGroup: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: RuleGroupDesc: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -648,7 +923,7 @@ func (m *RuleGroup) Unmarshal(dAtA []byte) error { var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowRulegroup + return ErrIntOverflowStore } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -662,11 +937,11 @@ func (m *RuleGroup) Unmarshal(dAtA []byte) error { } intStringLen := int(stringLen) if intStringLen < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } postIndex := iNdEx + intStringLen if postIndex < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } if postIndex > l { return io.ErrUnexpectedEOF @@ -680,7 +955,7 @@ func (m *RuleGroup) Unmarshal(dAtA []byte) error { var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowRulegroup + return ErrIntOverflowStore } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -694,11 +969,11 @@ func (m *RuleGroup) Unmarshal(dAtA []byte) error { } intStringLen := int(stringLen) if intStringLen < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } postIndex := iNdEx + intStringLen if postIndex < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } if postIndex > l { return io.ErrUnexpectedEOF @@ -712,7 +987,7 @@ func (m *RuleGroup) Unmarshal(dAtA []byte) error { var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowRulegroup + return ErrIntOverflowStore } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -725,11 +1000,11 @@ func (m *RuleGroup) Unmarshal(dAtA []byte) error { } } if msglen < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } postIndex := iNdEx + msglen if postIndex < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } if postIndex > l { return io.ErrUnexpectedEOF @@ -748,7 +1023,7 @@ func (m *RuleGroup) Unmarshal(dAtA []byte) error { var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowRulegroup + return ErrIntOverflowStore } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -761,11 +1036,11 @@ func (m *RuleGroup) Unmarshal(dAtA []byte) error { } } if msglen < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } postIndex := iNdEx + msglen if postIndex < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } if postIndex > l { return io.ErrUnexpectedEOF @@ -782,7 +1057,7 @@ func (m *RuleGroup) Unmarshal(dAtA []byte) error { var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowRulegroup + return ErrIntOverflowStore } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -795,17 +1070,49 @@ func (m *RuleGroup) Unmarshal(dAtA []byte) error { } } m.Deleted = bool(v != 0) + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field User", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStore + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthStore + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthStore + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.User = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skipRulegroup(dAtA[iNdEx:]) + skippy, err := skipStore(dAtA[iNdEx:]) if err != nil { return err } if skippy < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } if (iNdEx + skippy) < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -827,7 +1134,7 @@ func (m *Rule) Unmarshal(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowRulegroup + return ErrIntOverflowStore } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -855,7 +1162,7 @@ func (m *Rule) Unmarshal(dAtA []byte) error { var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowRulegroup + return ErrIntOverflowStore } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -869,11 +1176,11 @@ func (m *Rule) Unmarshal(dAtA []byte) error { } intStringLen := int(stringLen) if intStringLen < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } postIndex := iNdEx + intStringLen if postIndex < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } if postIndex > l { return io.ErrUnexpectedEOF @@ -887,7 +1194,7 @@ func (m *Rule) Unmarshal(dAtA []byte) error { var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowRulegroup + return ErrIntOverflowStore } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -901,11 +1208,11 @@ func (m *Rule) Unmarshal(dAtA []byte) error { } intStringLen := int(stringLen) if intStringLen < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } postIndex := iNdEx + intStringLen if postIndex < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } if postIndex > l { return io.ErrUnexpectedEOF @@ -919,7 +1226,7 @@ func (m *Rule) Unmarshal(dAtA []byte) error { var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowRulegroup + return ErrIntOverflowStore } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -933,11 +1240,11 @@ func (m *Rule) Unmarshal(dAtA []byte) error { } intStringLen := int(stringLen) if intStringLen < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } postIndex := iNdEx + intStringLen if postIndex < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } if postIndex > l { return io.ErrUnexpectedEOF @@ -951,7 +1258,7 @@ func (m *Rule) Unmarshal(dAtA []byte) error { var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowRulegroup + return ErrIntOverflowStore } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -964,11 +1271,11 @@ func (m *Rule) Unmarshal(dAtA []byte) error { } } if msglen < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } postIndex := iNdEx + msglen if postIndex < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } if postIndex > l { return io.ErrUnexpectedEOF @@ -987,7 +1294,7 @@ func (m *Rule) Unmarshal(dAtA []byte) error { var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowRulegroup + return ErrIntOverflowStore } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1000,11 +1307,11 @@ func (m *Rule) Unmarshal(dAtA []byte) error { } } if msglen < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } postIndex := iNdEx + msglen if postIndex < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } if postIndex > l { return io.ErrUnexpectedEOF @@ -1021,7 +1328,7 @@ func (m *Rule) Unmarshal(dAtA []byte) error { var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowRulegroup + return ErrIntOverflowStore } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1034,11 +1341,11 @@ func (m *Rule) Unmarshal(dAtA []byte) error { } } if msglen < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } postIndex := iNdEx + msglen if postIndex < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } if postIndex > l { return io.ErrUnexpectedEOF @@ -1050,15 +1357,15 @@ func (m *Rule) Unmarshal(dAtA []byte) error { iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skipRulegroup(dAtA[iNdEx:]) + skippy, err := skipStore(dAtA[iNdEx:]) if err != nil { return err } if skippy < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } if (iNdEx + skippy) < 0 { - return ErrInvalidLengthRulegroup + return ErrInvalidLengthStore } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -1072,14 +1379,14 @@ func (m *Rule) Unmarshal(dAtA []byte) error { } return nil } -func skipRulegroup(dAtA []byte) (n int, err error) { +func skipStore(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return 0, ErrIntOverflowRulegroup + return 0, ErrIntOverflowStore } if iNdEx >= l { return 0, io.ErrUnexpectedEOF @@ -1096,7 +1403,7 @@ func skipRulegroup(dAtA []byte) (n int, err error) { case 0: for shift := uint(0); ; shift += 7 { if shift >= 64 { - return 0, ErrIntOverflowRulegroup + return 0, ErrIntOverflowStore } if iNdEx >= l { return 0, io.ErrUnexpectedEOF @@ -1114,7 +1421,7 @@ func skipRulegroup(dAtA []byte) (n int, err error) { var length int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return 0, ErrIntOverflowRulegroup + return 0, ErrIntOverflowStore } if iNdEx >= l { return 0, io.ErrUnexpectedEOF @@ -1127,11 +1434,11 @@ func skipRulegroup(dAtA []byte) (n int, err error) { } } if length < 0 { - return 0, ErrInvalidLengthRulegroup + return 0, ErrInvalidLengthStore } iNdEx += length if iNdEx < 0 { - return 0, ErrInvalidLengthRulegroup + return 0, ErrInvalidLengthStore } return iNdEx, nil case 3: @@ -1140,7 +1447,7 @@ func skipRulegroup(dAtA []byte) (n int, err error) { var start int = iNdEx for shift := uint(0); ; shift += 7 { if shift >= 64 { - return 0, ErrIntOverflowRulegroup + return 0, ErrIntOverflowStore } if iNdEx >= l { return 0, io.ErrUnexpectedEOF @@ -1156,13 +1463,13 @@ func skipRulegroup(dAtA []byte) (n int, err error) { if innerWireType == 4 { break } - next, err := skipRulegroup(dAtA[start:]) + next, err := skipStore(dAtA[start:]) if err != nil { return 0, err } iNdEx = start + next if iNdEx < 0 { - return 0, ErrInvalidLengthRulegroup + return 0, ErrInvalidLengthStore } } return iNdEx, nil @@ -1179,6 +1486,6 @@ func skipRulegroup(dAtA []byte) (n int, err error) { } var ( - ErrInvalidLengthRulegroup = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowRulegroup = fmt.Errorf("proto: integer overflow") + ErrInvalidLengthStore = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowStore = fmt.Errorf("proto: integer overflow") ) diff --git a/pkg/ruler/store/store.proto b/pkg/ruler/store/store.proto new file mode 100644 index 00000000000..69faeb9e9ef --- /dev/null +++ b/pkg/ruler/store/store.proto @@ -0,0 +1,38 @@ +syntax = "proto3"; + +package store; + +option go_package = "store"; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; +import "google/protobuf/duration.proto"; +import "github.com/cortexproject/cortex/pkg/ingester/client/cortex.proto"; + +option (gogoproto.marshaler_all) = true; +option (gogoproto.unmarshaler_all) = true; + +message RuleUpdateDesc { + string user = 1; + int64 updatedAt = 2; +} + +message RuleGroupDesc { + string name = 1; + string namespace = 2; + google.protobuf.Duration interval = 3 [(gogoproto.stdduration) = true]; + + repeated Rule rules = 4; + + bool deleted = 5; + + string user = 6; + } + + message Rule { + string expr = 1; + string record = 2; + string alert = 3; + google.protobuf.Duration for = 4 [(gogoproto.stdduration) = true]; + repeated cortex.LabelPair labels = 5 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter"];; + repeated cortex.LabelPair annotations = 6 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter"];; + } \ No newline at end of file diff --git a/pkg/ruler/worker.go b/pkg/ruler/worker.go index fed6b27188d..49ba484440d 100644 --- a/pkg/ruler/worker.go +++ b/pkg/ruler/worker.go @@ -75,7 +75,7 @@ func (w *worker) Evaluate(userID string, item *workItem) { ctx := user.InjectOrgID(context.Background(), userID) logger := util.WithContext(ctx, util.Logger) if w.ruler.cfg.EnableSharding && !w.ruler.ownsRule(item.hash) { - level.Debug(util.Logger).Log("msg", "ruler: skipping evaluation, not owned", "user_id", item.userID, "group", item.groupName) + level.Debug(util.Logger).Log("msg", "ruler: skipping evaluation, not owned", "user_id", item.userID, "group", item.groupID) return } level.Debug(logger).Log("msg", "evaluating rules...", "num_rules", len(item.group.Rules())) @@ -83,7 +83,7 @@ func (w *worker) Evaluate(userID string, item *workItem) { instrument.CollectedRequest(ctx, "Evaluate", evalDuration, nil, func(ctx native_ctx.Context) error { if span := opentracing.SpanFromContext(ctx); span != nil { span.SetTag("instance", userID) - span.SetTag("groupName", item.groupName) + span.SetTag("groupID", item.groupID) } item.group.Eval(ctx, time.Now()) return nil diff --git a/pkg/storage/clients/aws/s3_config_client.go b/pkg/storage/clients/aws/s3_config_client.go index 2765e55031d..d9d46a55dd4 100644 --- a/pkg/storage/clients/aws/s3_config_client.go +++ b/pkg/storage/clients/aws/s3_config_client.go @@ -2,58 +2,40 @@ package aws import ( "context" - "flag" + "encoding/json" "fmt" + "io/ioutil" "strings" - "time" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/rulefmt" - "github.com/cortexproject/cortex/pkg/configs" + alertstore "github.com/cortexproject/cortex/pkg/alertmanager/storage" + "github.com/cortexproject/cortex/pkg/ruler/store" "github.com/cortexproject/cortex/pkg/util/flagext" awscommon "github.com/weaveworks/common/aws" - "github.com/weaveworks/common/instrument" ) var ( - s3RequestDuration = instrument.NewHistogramCollector(prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "cortex", - Name: "s3_request_duration_seconds", - Help: "Time spent doing S3 requests.", - Buckets: []float64{.025, .05, .1, .25, .5, 1, 2}, - }, []string{"operation", "status_code"})) + alertPrefix = "alerts/" + rulePrefix = "rules/" ) -func init() { - s3RequestDuration.Register() -} - -// S3ClientConfig is config for the GCS Chunk Client. -type S3ClientConfig struct { - BucketName string `yaml:"bucket_name"` - S3 flagext.URLValue `yaml:"url"` - S3ForcePathStyle bool -} - -// RegisterFlagsWithPrefix registers flags. -func (cfg *S3ClientConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { - f.StringVar(&cfg.BucketName, prefix+"s3.bucketname", "", "Name of S3 bucket to put chunks in.") -} - type s3ConfigClient struct { bucketName string S3 s3iface.S3API +} - alertPolled time.Time - rulePolled time.Time +// S3Config configures the s3ConfigClient +type S3Config struct { + S3 flagext.URLValue + S3ForcePathStyle bool } // NewS3ConfigClient makes a new S3-backed ObjectClient. -func NewS3ConfigClient(cfg S3ClientConfig) (configs.ConfigStore, error) { +func NewS3ConfigClient(cfg S3Config) (alertstore.AlertStore, error) { if cfg.S3.URL == nil { return nil, fmt.Errorf("no URL specified for S3") } @@ -70,45 +52,96 @@ func NewS3ConfigClient(cfg S3ClientConfig) (configs.ConfigStore, error) { client := &s3ConfigClient{ S3: s3Client, bucketName: bucketName, - - alertPolled: time.Unix(0, 0), - rulePolled: time.Unix(0, 0), } + return client, nil } -func (a *s3ConfigClient) PollAlertConfigs(ctx context.Context) (map[string]configs.AlertConfig, error) { - panic("not implemented") +func (s *s3ConfigClient) ListAlertConfigs(ctx context.Context) (map[string]alertstore.AlertConfig, error) { + var continuation *string + alerts := map[string]alertstore.AlertConfig{} + + for { + result, err := s.listAlerts(ctx, continuation) + if err != nil { + return nil, err + } + + if !*result.IsTruncated { + break + } + + for _, alertObj := range result.Contents { + cfg, err := s.getAlertConfig(ctx, alertObj.Key) + if err != nil { + return nil, err + } + user := strings.TrimPrefix(*alertObj.Key, alertPrefix) + alerts[user] = cfg + } + } + + return alerts, nil } -func (a *s3ConfigClient) GetAlertConfig(ctx context.Context, userID string) (configs.AlertConfig, error) { - panic("not implemented") +func (s *s3ConfigClient) listAlerts(ctx context.Context, continuation *string) (*s3.ListObjectsV2Output, error) { + return s.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: &s.bucketName, + Prefix: &alertPrefix, + ContinuationToken: continuation, + }) +} + +func (s *s3ConfigClient) getAlertConfig(ctx context.Context, key *string) (alertstore.AlertConfig, error) { + cfg, err := s.S3.GetObject(&s3.GetObjectInput{ + Bucket: &s.bucketName, + Key: key, + }) + + if err != nil { + return alertstore.AlertConfig{}, err + } + + defer cfg.Body.Close() + + buf, err := ioutil.ReadAll(cfg.Body) + if err != nil { + return alertstore.AlertConfig{}, err + } + + config := alertstore.AlertConfig{} + err = json.Unmarshal(buf, &config) + if err != nil { + return alertstore.AlertConfig{}, err + } + + return config, nil } -func (a *s3ConfigClient) SetAlertConfig(ctx context.Context, userID string, config configs.AlertConfig) error { +func (s *s3ConfigClient) GetAlertConfig(ctx context.Context, id string) (alertstore.AlertConfig, error) { panic("not implemented") } -func (a *s3ConfigClient) DeleteAlertConfig(ctx context.Context, userID string) error { +func (s *s3ConfigClient) SetAlertConfig(ctx context.Context, id string, cfg alertstore.AlertConfig) error { panic("not implemented") } -func (a *s3ConfigClient) PollRules(ctx context.Context) (map[string][]storage.RuleGroup error) { +func (s *s3ConfigClient) DeleteAlertConfig(ctx context.Context, id string) error { panic("not implemented") } -func (a *s3ConfigClient) ListRuleGroups(ctx context.Context, options configs.RuleStoreConditions) (map[string]configs.RuleNamespace, error) { +func (s *s3ConfigClient) ListRuleGroups(ctx context.Context, options store.RuleStoreConditions) (store.RuleGroupList, error) { panic("not implemented") } -func (a *s3ConfigClient) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (rulefmt.RuleGroup, error) { +func (s *s3ConfigClient) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (store.RuleGroup, error) { panic("not implemented") } -func (a *s3ConfigClient) SetRuleGroup(ctx context.Context, userID string, namespace string, group rulefmt.RuleGroup) error { +func (s *s3ConfigClient) SetRuleGroup(ctx context.Context, userID string, namespace string, group rulefmt.RuleGroup) error { panic("not implemented") } -func (a *s3ConfigClient) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { +func (s *s3ConfigClient) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { panic("not implemented") } diff --git a/pkg/storage/clients/client_test.go b/pkg/storage/clients/client_test.go index df3e8568ffa..f9e105a5a91 100644 --- a/pkg/storage/clients/client_test.go +++ b/pkg/storage/clients/client_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/pkg/alertmanager" - "github.com/cortexproject/cortex/pkg/ruler" + alertStore "github.com/cortexproject/cortex/pkg/alertmanager/storage" + "github.com/cortexproject/cortex/pkg/ruler/store" "github.com/cortexproject/cortex/pkg/storage/clients/gcp" "github.com/cortexproject/cortex/pkg/storage/testutils" "github.com/prometheus/prometheus/pkg/rulefmt" @@ -26,7 +26,7 @@ var ( ) func TestRuleStoreBasic(t *testing.T) { - forAllFixtures(t, func(t *testing.T, _ alertmanager.AlertStore, client ruler.RuleStore) { + forAllFixtures(t, func(t *testing.T, _ alertStore.AlertStore, client store.RuleStore) { const batchSize = 5 ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -36,7 +36,7 @@ func TestRuleStoreBasic(t *testing.T) { rg, err := client.GetRuleGroup(ctx, userID, namespace, exampleRuleGrp.Name) require.NoError(t, err) - assert.Equal(t, exampleRuleGrp.Name, rg.Name) + assert.Equal(t, exampleRuleGrp.Name, rg.Name()) err = client.DeleteRuleGroup(ctx, userID, namespace, exampleRuleGrp.Name) require.NoError(t, err) @@ -47,7 +47,7 @@ func TestRuleStoreBasic(t *testing.T) { }) } -func forAllFixtures(t *testing.T, clientTest func(*testing.T, alertmanager.AlertStore, ruler.RuleStore)) { +func forAllFixtures(t *testing.T, clientTest func(*testing.T, alertStore.AlertStore, store.RuleStore)) { var fixtures []testutils.Fixture fixtures = append(fixtures, gcp.Fixtures...) diff --git a/pkg/storage/clients/configdb/client.go b/pkg/storage/clients/configdb/client.go index f11c3c6f606..62fdd7faf78 100644 --- a/pkg/storage/clients/configdb/client.go +++ b/pkg/storage/clients/configdb/client.go @@ -10,10 +10,9 @@ import ( "strings" "time" - "github.com/cortexproject/cortex/pkg/alertmanager" + alertstore "github.com/cortexproject/cortex/pkg/alertmanager/storage" "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/ruler" - "github.com/cortexproject/cortex/pkg/ruler/group" + "github.com/cortexproject/cortex/pkg/ruler/store" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/go-kit/kit/log/level" @@ -70,6 +69,9 @@ func (c *ConfigsClient) GetRules(ctx context.Context, since configs.ID) (map[str return configs, nil } +// Stop stops rthe config client +func (c *ConfigsClient) Stop() {} + // GetAlerts implements ConfigClient. func (c *ConfigsClient) GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) { suffix := "" @@ -127,15 +129,16 @@ func (c ConfigsResponse) GetLatestConfigID() configs.ID { return latest } -func (c *ConfigsClient) PollAlertConfigs(ctx context.Context) (map[string]alertmanager.AlertConfig, error) { +// PollAlerts polls the configdb for updated alerts +func (c *ConfigsClient) PollAlerts(ctx context.Context) (map[string]alertstore.AlertConfig, error) { resp, err := c.GetAlerts(ctx, c.lastPoll) if err != nil { return nil, err } - newConfigs := map[string]alertmanager.AlertConfig{} + newConfigs := map[string]alertstore.AlertConfig{} for user, c := range resp.Configs { - newConfigs[user] = alertmanager.AlertConfig{ + newConfigs[user] = alertstore.AlertConfig{ TemplateFiles: c.Config.TemplateFiles, AlertmanagerConfig: c.Config.AlertmanagerConfig, } @@ -147,24 +150,30 @@ func (c *ConfigsClient) PollAlertConfigs(ctx context.Context) (map[string]alertm } // PollRules polls the configdb server and returns the updated rule groups -func (c *ConfigsClient) PollRules(ctx context.Context) (map[string][]ruler.RuleGroup, error) { - resp, err := c.GetAlerts(ctx, c.lastPoll) +func (c *ConfigsClient) PollRules(ctx context.Context) (map[string][]store.RuleGroup, error) { + resp, err := c.GetRules(ctx, c.lastPoll) if err != nil { return nil, err } - newRules := map[string][]ruler.RuleGroup{} + newRules := map[string][]store.RuleGroup{} - for user, cfg := range resp.Configs { - userRules := []ruler.RuleGroup{} - rls := cfg.GetVersionedRulesConfig() - rMap, err := rls.Config.Parse() + var highestID configs.ID + for user, cfg := range resp { + if cfg.ID > highestID { + highestID = cfg.ID + } + userRules := []store.RuleGroup{} + if cfg.IsDeleted() { + newRules[user] = []store.RuleGroup{} + } + rMap, err := cfg.Config.Parse() if err != nil { return nil, err } for groupSlug, r := range rMap { name, file := decomposeGroupSlug(groupSlug) - userRules = append(userRules, group.NewRuleGroup(name, file, user, r)) + userRules = append(userRules, store.FormattedToRuleGroup(user, file, name, r)) } newRules[user] = userRules } @@ -173,21 +182,11 @@ func (c *ConfigsClient) PollRules(ctx context.Context) (map[string][]ruler.RuleG return nil, err } - c.lastPoll = resp.GetLatestConfigID() + c.lastPoll = highestID return newRules, nil } -// AlertStore returns an AlertStore from the client -func (c *ConfigsClient) AlertStore() alertmanager.AlertStore { - return nil -} - -// RuleStore returns an RuleStore from the client -func (c *ConfigsClient) RuleStore() ruler.RuleStore { - return nil -} - // decomposeGroupSlug breaks the group slug from Parse // into it's group name and file name func decomposeGroupSlug(slug string) (string, string) { diff --git a/pkg/storage/clients/gcp/config_client.go b/pkg/storage/clients/gcp/config_client.go index c6250e1b652..6976d1c57e8 100644 --- a/pkg/storage/clients/gcp/config_client.go +++ b/pkg/storage/clients/gcp/config_client.go @@ -7,11 +7,9 @@ import ( "flag" "io/ioutil" "strings" - "time" - "github.com/cortexproject/cortex/pkg/alertmanager" - "github.com/cortexproject/cortex/pkg/ruler" - "github.com/cortexproject/cortex/pkg/ruler/group" + alertStore "github.com/cortexproject/cortex/pkg/alertmanager/storage" + "github.com/cortexproject/cortex/pkg/ruler/store" "github.com/cortexproject/cortex/pkg/util" gstorage "cloud.google.com/go/storage" @@ -45,9 +43,6 @@ func (cfg *GCSConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { type GCSClient struct { client *gstorage.Client bucket *gstorage.BucketHandle - - alertPolled time.Time - rulePolled time.Time } // NewGCSClient makes a new chunk.ObjectClient that writes chunks to GCS. @@ -66,81 +61,71 @@ func newGCSClient(cfg GCSConfig, client *gstorage.Client) *GCSClient { return &GCSClient{ client: client, bucket: bucket, + } +} + +func (g *GCSClient) ListAlertConfigs(ctx context.Context) (map[string]alertStore.AlertConfig, error) { + it := g.bucket.Objects(ctx, &gstorage.Query{ + Prefix: alertPrefix, + }) + + configs := map[string]alertStore.AlertConfig{} - alertPolled: time.Unix(0, 0), - rulePolled: time.Unix(0, 0), + for { + obj, err := it.Next() + if err == iterator.Done { + break + } + + if err != nil { + return nil, err + } + + alertConfig, err := g.getAlertConfig(ctx, obj.Name) + if err != nil { + return nil, err + } + + user := strings.TrimPrefix(obj.Name, alertPrefix) + + configs[user] = alertConfig } + + return configs, nil } -func (g *GCSClient) getAlertConfig(ctx context.Context, obj string) (alertmanager.AlertConfig, error) { +func (g *GCSClient) getAlertConfig(ctx context.Context, obj string) (alertStore.AlertConfig, error) { reader, err := g.bucket.Object(obj).NewReader(ctx) if err == gstorage.ErrObjectNotExist { level.Debug(util.Logger).Log("msg", "object does not exist", "name", obj) - return alertmanager.AlertConfig{}, nil + return alertStore.AlertConfig{}, nil } if err != nil { - return alertmanager.AlertConfig{}, err + return alertStore.AlertConfig{}, err } defer reader.Close() buf, err := ioutil.ReadAll(reader) if err != nil { - return alertmanager.AlertConfig{}, err + return alertStore.AlertConfig{}, err } - config := alertmanager.AlertConfig{} + config := alertStore.AlertConfig{} err = json.Unmarshal(buf, &config) if err != nil { - return alertmanager.AlertConfig{}, err + return alertStore.AlertConfig{}, err } return config, nil } -// PollAlertConfigs returns any recently updated alert configurations -func (g *GCSClient) PollAlertConfigs(ctx context.Context) (map[string]alertmanager.AlertConfig, error) { - objs := g.bucket.Objects(ctx, &gstorage.Query{ - Prefix: alertPrefix, - }) - - alertMap := map[string]alertmanager.AlertConfig{} - for { - objAttrs, err := objs.Next() - if err == iterator.Done { - break - } - level.Debug(util.Logger).Log("msg", "checking gcs config", "config", objAttrs.Name) - - if err != nil { - return nil, err - } - - if objAttrs.Updated.After(g.alertPolled) { - level.Debug(util.Logger).Log("msg", "adding updated gcs config", "config", objAttrs.Name) - rls, err := g.getAlertConfig(ctx, objAttrs.Name) - if err != nil { - return nil, err - } - alertMap[strings.TrimPrefix(objAttrs.Name, alertPrefix)] = rls - } - } - - g.alertPolled = time.Now() - return alertMap, nil -} - -// AlertStore returns an AlertStore from the client -func (g *GCSClient) AlertStore() alertmanager.AlertStore { - return g -} - // GetAlertConfig returns a specified users alertmanager configuration -func (g *GCSClient) GetAlertConfig(ctx context.Context, userID string) (alertmanager.AlertConfig, error) { +func (g *GCSClient) GetAlertConfig(ctx context.Context, userID string) (alertStore.AlertConfig, error) { return g.getAlertConfig(ctx, alertPrefix+userID) } // SetAlertConfig sets a specified users alertmanager configuration -func (g *GCSClient) SetAlertConfig(ctx context.Context, userID string, cfg alertmanager.AlertConfig) error { +func (g *GCSClient) SetAlertConfig(ctx context.Context, userID string, cfg alertStore.AlertConfig) error { cfgBytes, err := json.Marshal(cfg) if err != nil { return err @@ -169,91 +154,12 @@ func (g *GCSClient) DeleteAlertConfig(ctx context.Context, userID string) error return nil } -// PollRules returns a users recently updated rule set -func (g *GCSClient) PollRules(ctx context.Context) (map[string][]ruler.RuleGroup, error) { - objs := g.bucket.Objects(ctx, &gstorage.Query{ - Prefix: rulePrefix, - }) - - updatedUsers := []string{} - for { - handle, err := objs.Next() - if err == iterator.Done { - break - } - user, _, _, err := decomposeRuleHande(handle.Name) - if err != nil { - level.Error(util.Logger).Log("msg", "unable to poll user for rules", "user", user) - } - - level.Debug(util.Logger).Log("msg", "checking gcs for updated rules", "user", user) - - if err != nil { - return nil, err - } - - updated, err := g.checkUser(ctx, user) - if err != nil { - return nil, err - } - - if updated { - level.Info(util.Logger).Log("msg", "updated rules found", "user", user) - updatedUsers = append(updatedUsers, user) - } - } - - ruleMap := map[string][]ruler.RuleGroup{} - - for _, user := range updatedUsers { - rgs, err := g.getAllRuleGroups(ctx, user) - if err != nil { - return nil, err - } - - ruleMap[user] = rgs - } - - g.rulePolled = time.Now() - return ruleMap, nil -} - -// RuleStore returns an RuleStore from the client -func (g *GCSClient) RuleStore() ruler.RuleStore { - return g -} - -func (g *GCSClient) checkUser(ctx context.Context, userID string) (bool, error) { - objs := g.bucket.Objects(ctx, &gstorage.Query{ - Prefix: generateRuleHandle(userID, "", ""), - }) - - for { - rg, err := objs.Next() - if err == iterator.Done { - break - } - level.Debug(util.Logger).Log("msg", "checking gcs config", "config", rg.Name) - - if err != nil { - return false, err - } - - if rg.Updated.After(g.rulePolled) { - level.Debug(util.Logger).Log("msg", "updated rulegroups found", "user", userID) - return true, nil - } - } - - return false, nil -} - -func (g *GCSClient) getAllRuleGroups(ctx context.Context, userID string) ([]ruler.RuleGroup, error) { +func (g *GCSClient) getAllRuleGroups(ctx context.Context, userID string) ([]store.RuleGroup, error) { it := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: generateRuleHandle(userID, "", ""), }) - rgs := []ruler.RuleGroup{} + rgs := []store.RuleGroup{} for { obj, err := it.Next() @@ -262,31 +168,26 @@ func (g *GCSClient) getAllRuleGroups(ctx context.Context, userID string) ([]rule } if err != nil { - return []ruler.RuleGroup{}, err + return []store.RuleGroup{}, err } rgProto, err := g.getRuleGroup(ctx, obj.Name) if err != nil { - return []ruler.RuleGroup{}, err - } - - rg, err := group.GenerateRuleGroup(userID, rgProto) - if err != nil { - return []ruler.RuleGroup{}, err + return []store.RuleGroup{}, err } - rgs = append(rgs, rg) + rgs = append(rgs, store.ToRuleGroup(rgProto)) } return rgs, nil } -func (g *GCSClient) ListRuleGroups(ctx context.Context, options ruler.RuleStoreConditions) (map[string]ruler.RuleNamespace, error) { +func (g *GCSClient) ListRuleGroups(ctx context.Context, options store.RuleStoreConditions) (store.RuleGroupList, error) { it := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: generateRuleHandle(options.UserID, options.Namespace, ""), }) - namespaces := map[string]bool{} + groups := []store.RuleGroup{} for { obj, err := it.Next() if err == iterator.Done { @@ -297,39 +198,23 @@ func (g *GCSClient) ListRuleGroups(ctx context.Context, options ruler.RuleStoreC return nil, err } - level.Debug(util.Logger).Log("msg", "listing rule groups", "handle", obj.Name) - - _, namespace, _, err := decomposeRuleHande(obj.Name) - if err != nil { - return nil, err - } - if namespace == options.Namespace || options.Namespace == "" { - namespaces[namespace] = true - } - } - - nss := map[string]ruler.RuleNamespace{} + level.Debug(util.Logger).Log("msg", "listing rule group", "handle", obj.Name) - for namespace := range namespaces { - level.Debug(util.Logger).Log("msg", "retrieving rule namespace", "user", options.UserID, "namespace", namespace) - ns, err := g.getRuleNamespace(ctx, options.UserID, namespace) + rg, err := g.getRuleGroup(ctx, obj.Name) if err != nil { return nil, err } - nss[namespace] = ns + groups = append(groups, store.ToRuleGroup(rg)) } - - return nss, nil + return groups, nil } -func (g *GCSClient) getRuleNamespace(ctx context.Context, userID string, namespace string) (ruler.RuleNamespace, error) { +func (g *GCSClient) getRuleNamespace(ctx context.Context, userID string, namespace string) ([]*store.RuleGroupDesc, error) { it := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: generateRuleHandle(userID, namespace, ""), }) - ns := ruler.RuleNamespace{ - Groups: []rulefmt.RuleGroup{}, - } + groups := []*store.RuleGroupDesc{} for { obj, err := it.Next() @@ -338,21 +223,21 @@ func (g *GCSClient) getRuleNamespace(ctx context.Context, userID string, namespa } if err != nil { - return ruler.RuleNamespace{}, err + return nil, err } rg, err := g.getRuleGroup(ctx, obj.Name) if err != nil { - return ruler.RuleNamespace{}, err + return nil, err } - ns.Groups = append(ns.Groups, *group.FromProto(rg)) + groups = append(groups, rg) } - return ns, nil + return groups, nil } -func (g *GCSClient) GetRuleGroup(ctx context.Context, userID string, namespace string, grp string) (*rulefmt.RuleGroup, error) { +func (g *GCSClient) GetRuleGroup(ctx context.Context, userID string, namespace string, grp string) (store.RuleGroup, error) { handle := generateRuleHandle(userID, namespace, grp) rg, err := g.getRuleGroup(ctx, handle) if err != nil { @@ -360,12 +245,13 @@ func (g *GCSClient) GetRuleGroup(ctx context.Context, userID string, namespace s } if rg == nil { - return nil, ruler.ErrGroupNotFound + return nil, store.ErrGroupNotFound } - return group.FromProto(rg), nil + + return store.ToRuleGroup(rg), nil } -func (g *GCSClient) getRuleGroup(ctx context.Context, handle string) (*group.RuleGroup, error) { +func (g *GCSClient) getRuleGroup(ctx context.Context, handle string) (*store.RuleGroupDesc, error) { reader, err := g.bucket.Object(handle).NewReader(ctx) if err == gstorage.ErrObjectNotExist { level.Debug(util.Logger).Log("msg", "rule group does not exist", "name", handle) @@ -381,7 +267,7 @@ func (g *GCSClient) getRuleGroup(ctx context.Context, handle string) (*group.Rul return nil, err } - rg := &group.RuleGroup{} + rg := &store.RuleGroupDesc{} err = proto.Unmarshal(buf, rg) if err != nil { @@ -392,7 +278,7 @@ func (g *GCSClient) getRuleGroup(ctx context.Context, handle string) (*group.Rul } func (g *GCSClient) SetRuleGroup(ctx context.Context, userID string, namespace string, grp rulefmt.RuleGroup) error { - rg := group.ToProto(namespace, grp) + rg := store.ToProto(userID, namespace, grp) rgBytes, err := proto.Marshal(&rg) if err != nil { return err @@ -424,20 +310,12 @@ func (g *GCSClient) DeleteRuleGroup(ctx context.Context, userID string, namespac } func generateRuleHandle(id, namespace, name string) string { + if id == "" { + return rulePrefix + } prefix := rulePrefix + id + "/" if namespace == "" { return prefix } return prefix + namespace + "/" + name } - -func decomposeRuleHande(handle string) (string, string, string, error) { - components := strings.Split(handle, "/") - - if len(components) != 4 { - return "", "", "", errBadRuleGroup - } - - // Return `user, namespace, group_name` - return components[1], components[2], components[3], nil -} diff --git a/pkg/storage/clients/gcp/fixtures.go b/pkg/storage/clients/gcp/fixtures.go index bb5462d16f4..c176ba25dff 100644 --- a/pkg/storage/clients/gcp/fixtures.go +++ b/pkg/storage/clients/gcp/fixtures.go @@ -1,8 +1,8 @@ package gcp import ( - "github.com/cortexproject/cortex/pkg/alertmanager" - "github.com/cortexproject/cortex/pkg/ruler" + alertStore "github.com/cortexproject/cortex/pkg/alertmanager/storage" + "github.com/cortexproject/cortex/pkg/ruler/store" "github.com/cortexproject/cortex/pkg/storage/testutils" "github.com/fsouza/fake-gcs-server/fakestorage" ) @@ -21,7 +21,7 @@ func (f *fixture) Name() string { return f.name } -func (f *fixture) Clients() (alertmanager.AlertStore, ruler.RuleStore, error) { +func (f *fixture) Clients() (alertStore.AlertStore, store.RuleStore, error) { f.gcssrv = fakestorage.NewServer(nil) f.gcssrv.CreateBucket("configdb") cli := newGCSClient(GCSConfig{ diff --git a/pkg/storage/factory.go b/pkg/storage/factory.go deleted file mode 100644 index 850f66056bf..00000000000 --- a/pkg/storage/factory.go +++ /dev/null @@ -1,75 +0,0 @@ -package storage - -import ( - "context" - "flag" - "fmt" - - "github.com/cortexproject/cortex/pkg/alertmanager" - "github.com/cortexproject/cortex/pkg/ruler" - "github.com/cortexproject/cortex/pkg/storage/clients/configdb" - "github.com/cortexproject/cortex/pkg/storage/clients/gcp" -) - -// Config is used to configure the alert and rule store config clients -type Config struct { - AlertStoreConfig ClientConfig - RuleStoreConfig ClientConfig -} - -// RegisterFlags registers flags. -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - cfg.AlertStoreConfig.RegisterFlagsWithPrefix("alertmanager", f) - cfg.RuleStoreConfig.RegisterFlagsWithPrefix("ruler", f) -} - -// ClientConfig is used to config an config client -type ClientConfig struct { - BackendType string - - ConfigDB configdb.Config - GCS gcp.GCSConfig -} - -// RegisterFlagsWithPrefix registers flags. -func (cfg *ClientConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { - f.StringVar(&cfg.BackendType, prefix+".store.backend", "configdb", "backend to use for storing and retrieving alerts") - cfg.ConfigDB.RegisterFlagsWithPrefix(prefix, f) - cfg.GCS.RegisterFlagsWithPrefix(prefix+".store.", f) -} - -// NewRuleStore returns a rule store -func NewRuleStore(cfg Config) (ruler.RulePoller, error) { - var ( - store ruler.RulePoller - err error - ) - switch cfg.RuleStoreConfig.BackendType { - case "configdb": - store, err = configdb.New(cfg.RuleStoreConfig.ConfigDB) - case "gcp": - store, err = gcp.NewGCSClient(context.Background(), cfg.RuleStoreConfig.GCS) - default: - return nil, fmt.Errorf("Unrecognized rule store client %v, choose one of: client, gcp", cfg.RuleStoreConfig.BackendType) - } - - return store, err -} - -// NewAlertStore returns a alert store -func NewAlertStore(cfg Config) (alertmanager.AlertPoller, error) { - var ( - store alertmanager.AlertPoller - err error - ) - switch cfg.AlertStoreConfig.BackendType { - case "configdb": - store, err = configdb.New(cfg.AlertStoreConfig.ConfigDB) - case "gcp": - store, err = gcp.NewGCSClient(context.Background(), cfg.AlertStoreConfig.GCS) - default: - return nil, fmt.Errorf("Unrecognized config storage client %v, choose one of: client, gcp", cfg.AlertStoreConfig.BackendType) - } - - return store, err -} diff --git a/pkg/storage/testutils/testutils.go b/pkg/storage/testutils/testutils.go index edb54f7e7ce..b2d9ef1b505 100644 --- a/pkg/storage/testutils/testutils.go +++ b/pkg/storage/testutils/testutils.go @@ -1,13 +1,13 @@ package testutils import ( - "github.com/cortexproject/cortex/pkg/alertmanager" - "github.com/cortexproject/cortex/pkg/ruler" + alertStore "github.com/cortexproject/cortex/pkg/alertmanager/storage" + "github.com/cortexproject/cortex/pkg/ruler/store" ) // Fixture type for per-backend testing. type Fixture interface { Name() string - Clients() (alertmanager.AlertStore, ruler.RuleStore, error) + Clients() (alertStore.AlertStore, store.RuleStore, error) Teardown() error } diff --git a/pkg/util/usertracker/tracker.go b/pkg/util/usertracker/tracker.go new file mode 100644 index 00000000000..3cfef441095 --- /dev/null +++ b/pkg/util/usertracker/tracker.go @@ -0,0 +1,115 @@ +package usertracker + +import ( + "context" + "flag" + "sync" + "time" + + "github.com/cortexproject/cortex/pkg/ring/kv" + "github.com/cortexproject/cortex/pkg/ring/kv/codec" + "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log" + "github.com/golang/protobuf/proto" +) + +// ProtoUserUpdateDescFactory makes new UserUpdateDesc +func ProtoUserUpdateDescFactory() proto.Message { + return NewUserUpdateDesc() +} + +// NewUserUpdateDesc returns an empty *distributor.UserUpdateDesc. +func NewUserUpdateDesc() *UserUpdateDesc { + return &UserUpdateDesc{} +} + +// Tracker tracks when users update their configs to allow for efficient polling +type Tracker struct { + logger log.Logger + cfg Config + client kv.Client + + // Replicas we are accepting samples from. + updatedUsersMtx sync.RWMutex + updatedUsers map[string]UserUpdateDesc + done chan struct{} + cancel context.CancelFunc +} + +// Config contains the configuration require to create a User Tracker. +type Config struct { + KVStore kv.Config +} + +// RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet +func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + // We want the ability to use different Consul instances for the ring and for HA cluster tracking. + cfg.KVStore.RegisterFlagsWithPrefix(prefix+"user-tracker.", f) +} + +// NewTracker returns a new HA cluster tracker using either Consul +// or in-memory KV store. +func NewTracker(cfg Config) (*Tracker, error) { + codec := codec.Proto{Factory: ProtoUserUpdateDescFactory} + + client, err := kv.NewClient(cfg.KVStore, codec) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithCancel(context.Background()) + t := Tracker{ + logger: util.Logger, + cfg: cfg, + done: make(chan struct{}), + updatedUsers: map[string]UserUpdateDesc{}, + client: client, + cancel: cancel, + } + go t.loop(ctx) + return &t, nil +} + +// Follows pattern used by ring for WatchKey. +func (c *Tracker) loop(ctx context.Context) { + defer close(c.done) + // The KVStore config we gave when creating c should have contained a prefix, + // which would have given us a prefixed KVStore client. So, we can pass empty string here. + c.client.WatchPrefix(ctx, "", func(key string, value interface{}) bool { + replica := value.(*UserUpdateDesc) + c.updatedUsersMtx.Lock() + defer c.updatedUsersMtx.Unlock() + c.updatedUsers[key] = *replica + return true + }) +} + +// Stop ends calls the trackers cancel function, which will end the loop for WatchPrefix. +func (c *Tracker) Stop() { + c.cancel() + <-c.done +} + +func (c *Tracker) GetUpdatedUsers(ctx context.Context) []string { + c.updatedUsersMtx.Lock() + defer c.updatedUsersMtx.Unlock() + + users := make([]string, len(c.updatedUsers)) + for u := range c.updatedUsers { + users = append(users, u) + } + + c.updatedUsers = map[string]UserUpdateDesc{} + + return users +} + +func (c *Tracker) UpdateUser(ctx context.Context, userID string) error { + return c.client.CAS(ctx, userID, func(in interface{}) (out interface{}, retry bool, err error) { + // Add an entry to mark an update to a users rule configs + return &UserUpdateDesc{ + User: userID, + UpdatedAt: time.Now().UnixNano(), + }, true, nil + }) +} diff --git a/pkg/util/usertracker/tracker.pb.go b/pkg/util/usertracker/tracker.pb.go new file mode 100644 index 00000000000..960d852cdf4 --- /dev/null +++ b/pkg/util/usertracker/tracker.pb.go @@ -0,0 +1,446 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: tracker.proto + +package usertracker + +import ( + fmt "fmt" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + reflect "reflect" + strings "strings" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +type UserUpdateDesc struct { + User string `protobuf:"bytes,1,opt,name=user,proto3" json:"user,omitempty"` + UpdatedAt int64 `protobuf:"varint,2,opt,name=updatedAt,proto3" json:"updatedAt,omitempty"` +} + +func (m *UserUpdateDesc) Reset() { *m = UserUpdateDesc{} } +func (*UserUpdateDesc) ProtoMessage() {} +func (*UserUpdateDesc) Descriptor() ([]byte, []int) { + return fileDescriptor_a0ba8625d8751af3, []int{0} +} +func (m *UserUpdateDesc) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *UserUpdateDesc) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_UserUpdateDesc.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *UserUpdateDesc) XXX_Merge(src proto.Message) { + xxx_messageInfo_UserUpdateDesc.Merge(m, src) +} +func (m *UserUpdateDesc) XXX_Size() int { + return m.Size() +} +func (m *UserUpdateDesc) XXX_DiscardUnknown() { + xxx_messageInfo_UserUpdateDesc.DiscardUnknown(m) +} + +var xxx_messageInfo_UserUpdateDesc proto.InternalMessageInfo + +func (m *UserUpdateDesc) GetUser() string { + if m != nil { + return m.User + } + return "" +} + +func (m *UserUpdateDesc) GetUpdatedAt() int64 { + if m != nil { + return m.UpdatedAt + } + return 0 +} + +func init() { + proto.RegisterType((*UserUpdateDesc)(nil), "usertracker.UserUpdateDesc") +} + +func init() { proto.RegisterFile("tracker.proto", fileDescriptor_a0ba8625d8751af3) } + +var fileDescriptor_a0ba8625d8751af3 = []byte{ + // 195 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2d, 0x29, 0x4a, 0x4c, + 0xce, 0x4e, 0x2d, 0xd2, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x2e, 0x2d, 0x4e, 0x2d, 0x82, + 0x0a, 0x49, 0xe9, 0xa6, 0x67, 0x96, 0x64, 0x94, 0x26, 0xe9, 0x25, 0xe7, 0xe7, 0xea, 0xa7, 0xe7, + 0xa7, 0xe7, 0xeb, 0x83, 0xd5, 0x24, 0x95, 0xa6, 0x81, 0x79, 0x60, 0x0e, 0x98, 0x05, 0xd1, 0xab, + 0xe4, 0xc4, 0xc5, 0x17, 0x5a, 0x9c, 0x5a, 0x14, 0x5a, 0x90, 0x92, 0x58, 0x92, 0xea, 0x92, 0x5a, + 0x9c, 0x2c, 0x24, 0xc4, 0xc5, 0x02, 0x32, 0x4f, 0x82, 0x51, 0x81, 0x51, 0x83, 0x33, 0x08, 0xcc, + 0x16, 0x92, 0xe1, 0xe2, 0x2c, 0x05, 0xab, 0x48, 0x71, 0x2c, 0x91, 0x60, 0x52, 0x60, 0xd4, 0x60, + 0x0e, 0x42, 0x08, 0x38, 0x39, 0x5e, 0x78, 0x28, 0xc7, 0x70, 0xe3, 0xa1, 0x1c, 0xc3, 0x87, 0x87, + 0x72, 0x8c, 0x0d, 0x8f, 0xe4, 0x18, 0x57, 0x3c, 0x92, 0x63, 0x3c, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, + 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x5f, 0x3c, 0x92, 0x63, 0xf8, 0xf0, 0x48, 0x8e, 0x71, + 0xc2, 0x63, 0x39, 0x86, 0x0b, 0x8f, 0xe5, 0x18, 0x6e, 0x3c, 0x96, 0x63, 0x88, 0x42, 0x76, 0x75, + 0x12, 0x1b, 0xd8, 0x35, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x8a, 0xdb, 0xd4, 0x4d, 0xda, + 0x00, 0x00, 0x00, +} + +func (this *UserUpdateDesc) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*UserUpdateDesc) + if !ok { + that2, ok := that.(UserUpdateDesc) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.User != that1.User { + return false + } + if this.UpdatedAt != that1.UpdatedAt { + return false + } + return true +} +func (this *UserUpdateDesc) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 6) + s = append(s, "&usertracker.UserUpdateDesc{") + s = append(s, "User: "+fmt.Sprintf("%#v", this.User)+",\n") + s = append(s, "UpdatedAt: "+fmt.Sprintf("%#v", this.UpdatedAt)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func valueToGoStringTracker(v interface{}, typ string) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) +} +func (m *UserUpdateDesc) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *UserUpdateDesc) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.User) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintTracker(dAtA, i, uint64(len(m.User))) + i += copy(dAtA[i:], m.User) + } + if m.UpdatedAt != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintTracker(dAtA, i, uint64(m.UpdatedAt)) + } + return i, nil +} + +func encodeVarintTracker(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *UserUpdateDesc) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.User) + if l > 0 { + n += 1 + l + sovTracker(uint64(l)) + } + if m.UpdatedAt != 0 { + n += 1 + sovTracker(uint64(m.UpdatedAt)) + } + return n +} + +func sovTracker(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozTracker(x uint64) (n int) { + return sovTracker(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *UserUpdateDesc) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&UserUpdateDesc{`, + `User:` + fmt.Sprintf("%v", this.User) + `,`, + `UpdatedAt:` + fmt.Sprintf("%v", this.UpdatedAt) + `,`, + `}`, + }, "") + return s +} +func valueToStringTracker(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *UserUpdateDesc) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTracker + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: UserUpdateDesc: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: UserUpdateDesc: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field User", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTracker + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTracker + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthTracker + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.User = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdatedAt", wireType) + } + m.UpdatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTracker + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.UpdatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipTracker(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTracker + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthTracker + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipTracker(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowTracker + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowTracker + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowTracker + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthTracker + } + iNdEx += length + if iNdEx < 0 { + return 0, ErrInvalidLengthTracker + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowTracker + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipTracker(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + if iNdEx < 0 { + return 0, ErrInvalidLengthTracker + } + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthTracker = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowTracker = fmt.Errorf("proto: integer overflow") +) diff --git a/pkg/util/usertracker/tracker.proto b/pkg/util/usertracker/tracker.proto new file mode 100644 index 00000000000..d35b52ce5d4 --- /dev/null +++ b/pkg/util/usertracker/tracker.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +package usertracker; + +option go_package = "usertracker"; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; + + +option (gogoproto.marshaler_all) = true; +option (gogoproto.unmarshaler_all) = true; + +message UserUpdateDesc { + string user = 1; + int64 updatedAt = 2; +} \ No newline at end of file From 962db7952182162c849963d1dc6dd2f2fb919f1c Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 19 Jul 2019 12:57:46 -0400 Subject: [PATCH 49/60] add logs and comments Signed-off-by: Jacob Lisi --- pkg/ruler/kv_poller.go | 10 ++++++++-- pkg/util/usertracker/tracker.go | 4 ++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/ruler/kv_poller.go b/pkg/ruler/kv_poller.go index bd4ebe2601e..a807585dbe5 100644 --- a/pkg/ruler/kv_poller.go +++ b/pkg/ruler/kv_poller.go @@ -4,7 +4,9 @@ import ( "context" "github.com/cortexproject/cortex/pkg/ruler/store" + "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/usertracker" + "github.com/go-kit/kit/log/level" "github.com/prometheus/prometheus/pkg/rulefmt" ) @@ -34,11 +36,14 @@ func (p *trackedPoller) trackedRuleStore() *trackedRuleStore { func (p *trackedPoller) PollRules(ctx context.Context) (map[string][]store.RuleGroup, error) { updatedRules := map[string][]store.RuleGroup{} + level.Debug(util.Logger).Log("msg", "polling for new rules") + // First poll will return all rule groups if !p.initialized { + level.Debug(util.Logger).Log("msg", "first poll, loading all rules") rgs, err := p.store.ListRuleGroups(ctx, store.RuleStoreConditions{}) if err != nil { - return nil, nil + return nil, err } for _, rg := range rgs { if _, exists := updatedRules[rg.User()]; !exists { @@ -51,11 +56,12 @@ func (p *trackedPoller) PollRules(ctx context.Context) (map[string][]store.RuleG } else { users := p.tracker.GetUpdatedUsers(ctx) for _, u := range users { + level.Debug(util.Logger).Log("msg", "poll found updated user", "user", u) rgs, err := p.store.ListRuleGroups(ctx, store.RuleStoreConditions{ UserID: u, }) if err != nil { - return nil, nil + return nil, err } updatedRules[u] = rgs diff --git a/pkg/util/usertracker/tracker.go b/pkg/util/usertracker/tracker.go index 3cfef441095..8e7dcef315e 100644 --- a/pkg/util/usertracker/tracker.go +++ b/pkg/util/usertracker/tracker.go @@ -90,6 +90,8 @@ func (c *Tracker) Stop() { <-c.done } +// GetUpdatedUsers returns all of the users updated since the last +// poll func (c *Tracker) GetUpdatedUsers(ctx context.Context) []string { c.updatedUsersMtx.Lock() defer c.updatedUsersMtx.Unlock() @@ -104,6 +106,8 @@ func (c *Tracker) GetUpdatedUsers(ctx context.Context) []string { return users } +// UpdateUser pushes a change to the kvstore to signal the user has been +// updated func (c *Tracker) UpdateUser(ctx context.Context, userID string) error { return c.client.CAS(ctx, userID, func(in interface{}) (out interface{}, retry bool, err error) { // Add an entry to mark an update to a users rule configs From e6d8952cee28caaf58d7c6af66e6677e1e968506 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 19 Jul 2019 19:35:33 -0400 Subject: [PATCH 50/60] return 404 when rule group is not found Signed-off-by: Jacob Lisi --- pkg/ruler/api.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 669bb0f9728..1f24e5c64e9 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -143,6 +143,7 @@ func (r *Ruler) getRuleGroup(w http.ResponseWriter, req *http.Request) { if err != nil { if err == store.ErrGroupNotFound { http.Error(w, err.Error(), http.StatusNotFound) + return } http.Error(w, err.Error(), http.StatusBadRequest) return From 18f96cedff3085eeee63d89f6b61e2f8dbf00342 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 19 Jul 2019 19:45:47 -0400 Subject: [PATCH 51/60] add delete rule group feature Signed-off-by: Jacob Lisi --- pkg/ruler/api.go | 46 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 1f24e5c64e9..09812813251 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -127,19 +127,19 @@ func (r *Ruler) getRuleGroup(w http.ResponseWriter, req *http.Request) { } vars := mux.Vars(req) - ns, exists := vars["namespace"] + namespace, exists := vars["namespace"] if !exists { http.Error(w, ErrNoNamespace.Error(), http.StatusUnauthorized) return } - gn, exists := vars["groupName"] + groupName, exists := vars["groupName"] if !exists { http.Error(w, ErrNoGroupName.Error(), http.StatusUnauthorized) return } - rg, err := r.store.GetRuleGroup(req.Context(), userID, ns, gn) + rg, err := r.store.GetRuleGroup(req.Context(), userID, namespace, groupName) if err != nil { if err == store.ErrGroupNotFound { http.Error(w, err.Error(), http.StatusNotFound) @@ -229,5 +229,45 @@ func (r *Ruler) createRuleGroup(w http.ResponseWriter, req *http.Request) { } func (r *Ruler) deleteRuleGroup(w http.ResponseWriter, req *http.Request) { + userID, _, err := user.ExtractOrgIDFromHTTPRequest(req) + if err != nil { + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + logger := util.WithContext(req.Context(), util.Logger) + if err != nil { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if userID == "" { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + + vars := mux.Vars(req) + namespace, exists := vars["namespace"] + if !exists { + http.Error(w, ErrNoNamespace.Error(), http.StatusUnauthorized) + return + } + + groupName, exists := vars["groupName"] + if !exists { + http.Error(w, ErrNoGroupName.Error(), http.StatusUnauthorized) + return + } + + err = r.store.DeleteRuleGroup(req.Context(), userID, namespace, groupName) + if err != nil { + level.Error(logger).Log("err", err.Error()) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Return a status accepted because the rule has been stored and queued for polling, but is not currently active + w.WriteHeader(http.StatusAccepted) } From 1c0405592bce38dc72db1e39fd9a0e24216e46be Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Sun, 21 Jul 2019 20:48:53 -0400 Subject: [PATCH 52/60] fix linting and tests Signed-off-by: Jacob Lisi --- pkg/alertmanager/api.go | 2 +- pkg/alertmanager/kv_poller.go | 4 +- pkg/alertmanager/storage.go | 1 + pkg/ruler/api.go | 56 +++++++----------------- pkg/ruler/ruler.go | 8 +++- pkg/ruler/ruler_test.go | 5 ++- pkg/ruler/scheduler.go | 24 +++++----- pkg/ruler/scheduling_queue.go | 2 +- pkg/ruler/storage.go | 3 ++ pkg/ruler/store/group.go | 30 ++++++++----- pkg/ruler/store/store.go | 10 ++++- pkg/ruler/worker.go | 7 ++- pkg/storage/clients/gcp/config_client.go | 5 +++ 13 files changed, 84 insertions(+), 73 deletions(-) diff --git a/pkg/alertmanager/api.go b/pkg/alertmanager/api.go index c42c80b2204..74d61e24e75 100644 --- a/pkg/alertmanager/api.go +++ b/pkg/alertmanager/api.go @@ -14,7 +14,7 @@ import ( // RegisterRoutes registers the configs API HTTP routes with the provided Router. func (am *MultitenantAlertmanager) RegisterRoutes(r *mux.Router) { - // if no storeis set return without regisering routes + // if no store is set return without regisering routes if am.store == nil { return } diff --git a/pkg/alertmanager/kv_poller.go b/pkg/alertmanager/kv_poller.go index 15afab33166..3ceffab39a4 100644 --- a/pkg/alertmanager/kv_poller.go +++ b/pkg/alertmanager/kv_poller.go @@ -8,7 +8,7 @@ import ( ) // trackedAlertPoller checks for updated user configs and -// retreives the updated configuration from the backend +// Retrieves the updated configuration from the backend type trackedAlertPoller struct { tracker *usertracker.Tracker store storage.AlertStore @@ -46,7 +46,7 @@ func (p *trackedAlertPoller) PollAlerts(ctx context.Context) (map[string]storage // Get the changed users from the user update tracker users := p.tracker.GetUpdatedUsers(ctx) - // Retreive user configuration from the rule store + // Retrieve user configuration from the rule store // TODO: Add Retry logic for failed requests // TODO: store users that were failed to be updated and reattempt to retrieve on the next poll for _, u := range users { diff --git a/pkg/alertmanager/storage.go b/pkg/alertmanager/storage.go index 7c3800c11c3..98d04a2d60a 100644 --- a/pkg/alertmanager/storage.go +++ b/pkg/alertmanager/storage.go @@ -11,6 +11,7 @@ import ( "github.com/cortexproject/cortex/pkg/util/usertracker" ) +// AlertStoreConfig configures the alertmanager backend type AlertStoreConfig struct { Type string `yaml:"type"` diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 09812813251..56d164ab4f1 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -16,9 +16,14 @@ import ( ) var ( - ErrNoNamespace = errors.New("a namespace must be provided in the url") - ErrNoGroupName = errors.New("a matching group name must be provided in the url") + // ErrNoNamespace signals the requested namespace does not exist + ErrNoNamespace = errors.New("a namespace must be provided in the url") + // ErrNoGroupName signals a group name url parameter was not found + ErrNoGroupName = errors.New("a matching group name must be provided in the url") + // ErrNoRuleGroups signals the rule group requested does not exist ErrNoRuleGroups = errors.New("no rule groups found") + // ErrNoUserID is returned when no user ID is provided + ErrNoUserID = errors.New("no id provided") ) // RegisterRoutes registers the configs API HTTP routes with the provided Router. @@ -33,7 +38,7 @@ func (r *Ruler) RegisterRoutes(router *mux.Router) { handler http.HandlerFunc }{ {"list_rules", "GET", "/api/prom/rules", r.listRules}, - {"list_rules_namespace", "GET", "/api/prom/rules/{namespace}", r.listRules}, + {"getRuleNamespace", "GET", "/api/prom/rules/{namespace}", r.listRules}, {"get_rulegroup", "GET", "/api/prom/rules/{namespace}/{groupName}", r.getRuleGroup}, {"set_rulegroup", "POST", "/api/prom/rules/{namespace}", r.createRuleGroup}, {"delete_rulegroup", "DELETE", "/api/prom/rules/{namespace}/{groupName}", r.deleteRuleGroup}, @@ -43,22 +48,15 @@ func (r *Ruler) RegisterRoutes(router *mux.Router) { } func (r *Ruler) listRules(w http.ResponseWriter, req *http.Request) { + logger := util.WithContext(req.Context(), util.Logger) userID, _, err := user.ExtractOrgIDFromHTTPRequest(req) if err != nil { http.Error(w, err.Error(), http.StatusUnauthorized) return } - logger := util.WithContext(req.Context(), util.Logger) - if err != nil { - level.Error(logger).Log("err", err.Error()) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - if userID == "" { - level.Error(logger).Log("err", err.Error()) - http.Error(w, err.Error(), http.StatusUnauthorized) + http.Error(w, ErrNoUserID.Error(), http.StatusUnauthorized) return } @@ -107,22 +105,16 @@ func (r *Ruler) listRules(w http.ResponseWriter, req *http.Request) { } func (r *Ruler) getRuleGroup(w http.ResponseWriter, req *http.Request) { + logger := util.WithContext(req.Context(), util.Logger) + userID, _, err := user.ExtractOrgIDFromHTTPRequest(req) if err != nil { http.Error(w, err.Error(), http.StatusUnauthorized) return } - logger := util.WithContext(req.Context(), util.Logger) - if err != nil { - level.Error(logger).Log("err", err.Error()) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - if userID == "" { - level.Error(logger).Log("err", err.Error()) - http.Error(w, err.Error(), http.StatusUnauthorized) + http.Error(w, ErrNoUserID.Error(), http.StatusUnauthorized) return } @@ -165,22 +157,15 @@ func (r *Ruler) getRuleGroup(w http.ResponseWriter, req *http.Request) { } func (r *Ruler) createRuleGroup(w http.ResponseWriter, req *http.Request) { + logger := util.WithContext(req.Context(), util.Logger) userID, _, err := user.ExtractOrgIDFromHTTPRequest(req) if err != nil { http.Error(w, err.Error(), http.StatusUnauthorized) return } - logger := util.WithContext(req.Context(), util.Logger) - if err != nil { - level.Error(logger).Log("err", err.Error()) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - if userID == "" { - level.Error(logger).Log("err", err.Error()) - http.Error(w, err.Error(), http.StatusUnauthorized) + http.Error(w, ErrNoUserID.Error(), http.StatusUnauthorized) return } @@ -229,22 +214,15 @@ func (r *Ruler) createRuleGroup(w http.ResponseWriter, req *http.Request) { } func (r *Ruler) deleteRuleGroup(w http.ResponseWriter, req *http.Request) { + logger := util.WithContext(req.Context(), util.Logger) userID, _, err := user.ExtractOrgIDFromHTTPRequest(req) if err != nil { http.Error(w, err.Error(), http.StatusUnauthorized) return } - logger := util.WithContext(req.Context(), util.Logger) - if err != nil { - level.Error(logger).Log("err", err.Error()) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - if userID == "" { - level.Error(logger).Log("err", err.Error()) - http.Error(w, err.Error(), http.StatusUnauthorized) + http.Error(w, ErrNoUserID.Error(), http.StatusUnauthorized) return } diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index a6651fff7bb..c910d726fc2 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -142,6 +142,9 @@ func NewRuler(cfg Config, engine *promql.Engine, queryable promStorage.Queryable } rulePoller, ruleStore, err := NewRuleStorage(cfg.StoreConfig) + if err != nil { + return nil, err + } ruler := &Ruler{ cfg: cfg, @@ -350,6 +353,9 @@ func (r *Ruler) ServeHTTP(w http.ResponseWriter, req *http.Request) { ` w.WriteHeader(http.StatusOK) - w.Write([]byte(unshardedPage)) + _, err := w.Write([]byte(unshardedPage)) + if err != nil { + level.Error(util.Logger).Log("msg", "unable to serve status page", "err", err) + } } } diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index e970aed185a..c125a85beb1 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/cortexproject/cortex/pkg/querier" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ring/kv/codec" @@ -69,7 +71,8 @@ func TestNotifierSendsUserIDHeader(t *testing.T) { defer ts.Close() cfg := defaultRulerConfig() - cfg.AlertmanagerURL.Set(ts.URL) + err := cfg.AlertmanagerURL.Set(ts.URL) + require.NoError(t, err) cfg.AlertmanagerDiscovery = false r := newTestRuler(t, cfg) diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index 6a34d79a0cd..ea19ccae124 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -17,15 +17,6 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" ) -type Scheduler interface { - Next() *WorkItem - Done(WorkItem) -} - -type WorkItem interface { - Evaluate(context.Context) -} - const ( timeLogFormat = "2006-01-02T15:04:05" ) @@ -178,7 +169,12 @@ func (s *scheduler) addUserConfig(ctx context.Context, userID string, rgs []stor } ringHasher.Reset() - ringHasher.Write([]byte(rg.ID())) + _, err = ringHasher.Write([]byte(rg.ID())) + if err != nil { + level.Error(util.Logger).Log("msg", "scheduler: failed to create group for user", "user_id", userID, "group", rg.ID(), "err", err) + return + } + hash := ringHasher.Sum32() workItems = append(workItems, workItem{userID, rg.ID(), hash, grp, evalTime, userChan}) } @@ -206,8 +202,6 @@ func (s *scheduler) updateUserConfig(ctx context.Context, cfg userConfig) { if exists { close(curr.done) // If a previous configuration exists, ensure it is closed } - - return } func (s *scheduler) determineEvalTime(userID string) time.Time { @@ -222,7 +216,11 @@ func computeNextEvalTime(hasher hash.Hash64, now time.Time, intervalNanos float6 currentEvalCyclePoint := math.Mod(float64(now.UnixNano()), intervalNanos) hasher.Reset() - hasher.Write([]byte(userID)) + _, err := hasher.Write([]byte(userID)) + if err != nil { + // if an error occurs just return the current time plus a minute + return now.Add(time.Minute) + } offset := math.Mod( // We subtract our current point in the cycle to cause the entries // before 'now' to wrap around to the end. diff --git a/pkg/ruler/scheduling_queue.go b/pkg/ruler/scheduling_queue.go index d4c0233916d..23808d8685c 100644 --- a/pkg/ruler/scheduling_queue.go +++ b/pkg/ruler/scheduling_queue.go @@ -73,7 +73,7 @@ func (q *queueState) Enqueue(op ScheduledItem) { func (q *queueState) Dequeue() ScheduledItem { item := heap.Pop(q).(ScheduledItem) - itemEvaluationLatency.Observe(time.Now().Sub(item.Scheduled()).Seconds()) + itemEvaluationLatency.Observe(time.Since(item.Scheduled()).Seconds()) return item } diff --git a/pkg/ruler/storage.go b/pkg/ruler/storage.go index a5268a70efe..8aed879fcc6 100644 --- a/pkg/ruler/storage.go +++ b/pkg/ruler/storage.go @@ -45,6 +45,9 @@ func NewRuleStorage(cfg RuleStoreConfig) (store.RulePoller, store.RuleStore, err return poller, nil, err case "gcs": ruleStore, err = gcp.NewGCSClient(context.Background(), cfg.GCS) + if err != nil { + return nil, nil, err + } default: return nil, nil, fmt.Errorf("Unrecognized rule storage mode %v, choose one of: configdb, gcs", cfg.Type) } diff --git a/pkg/ruler/store/group.go b/pkg/ruler/store/group.go index db201d41eaf..a1ba363425f 100644 --- a/pkg/ruler/store/group.go +++ b/pkg/ruler/store/group.go @@ -17,6 +17,7 @@ import ( // TODO: The cortex project should implement a separate Group struct from // the prometheus project. This will allow for more precise instrumentation +// Group is used as a compatibility format between storage and evaluation type Group struct { name string namespace string @@ -24,10 +25,22 @@ type Group struct { interval time.Duration rules []*Rule - // TODO: Allows for the support of the configdb client + // activeRules allows for the support of the configdb client + // TODO: figure out a better way to accomplish this activeRules []rules.Rule } +// NewRuleGroup returns a Group +func NewRuleGroup(name, namespace, user string, rls []rulefmt.Rule) *Group { + return &Group{ + name: name, + namespace: namespace, + user: user, + rules: formattedRuleToProto(rls), + } +} + +// Rules returns eval ready prometheus rules func (g *Group) Rules(ctx context.Context) ([]rules.Rule, error) { // Used to be compatible with configdb client if g.rules == nil && g.activeRules != nil { @@ -62,22 +75,27 @@ func (g *Group) Rules(ctx context.Context) ([]rules.Rule, error) { return rls, nil } +// ID returns a unique group identifier with the namespace and name func (g *Group) ID() string { return g.namespace + "/" + g.name } +// Name returns the name of the rule group func (g *Group) Name() string { return g.name } +// Namespace returns the Namespace of the rule group func (g *Group) Namespace() string { return g.namespace } +// User returns the User of the rule group func (g *Group) User() string { return g.user } +// Formatted returns a prometheus rulefmt formatted rule group func (g *Group) Formatted() rulefmt.RuleGroup { formattedRuleGroup := rulefmt.RuleGroup{ Name: g.name, @@ -98,13 +116,3 @@ func (g *Group) Formatted() rulefmt.RuleGroup { return formattedRuleGroup } - -// NewGroup returns a Group -func NewRuleGroup(name, namespace, user string, rls []rulefmt.Rule) *Group { - return &Group{ - name: name, - namespace: namespace, - user: user, - rules: formattedRuleToProto(rls), - } -} diff --git a/pkg/ruler/store/store.go b/pkg/ruler/store/store.go index 1a193a5ff99..5dfc2d0ada1 100644 --- a/pkg/ruler/store/store.go +++ b/pkg/ruler/store/store.go @@ -10,9 +10,12 @@ import ( ) var ( - ErrGroupNotFound = errors.New("group does not exist") + // ErrGroupNotFound is returned if a rule group does not exist + ErrGroupNotFound = errors.New("group does not exist") + // ErrGroupNamespaceNotFound is returned if a namespace does not exist ErrGroupNamespaceNotFound = errors.New("group namespace does not exist") - ErrUserNotFound = errors.New("no rule groups found for user") + // ErrUserNotFound is returned if the user does not currently exist + ErrUserNotFound = errors.New("no rule groups found for user") ) // RulePoller is used to poll for recently updated rules @@ -50,8 +53,11 @@ type RuleGroup interface { Formatted() rulefmt.RuleGroup } +// RuleGroupList contains a set of rule groups type RuleGroupList []RuleGroup +// Formatted returns the rule group list as a set of formatted rule groups mapped +// by namespace func (l RuleGroupList) Formatted(user string) map[string][]rulefmt.RuleGroup { ruleMap := map[string][]rulefmt.RuleGroup{} for _, g := range l { diff --git a/pkg/ruler/worker.go b/pkg/ruler/worker.go index 49ba484440d..29be0f707d1 100644 --- a/pkg/ruler/worker.go +++ b/pkg/ruler/worker.go @@ -55,7 +55,7 @@ func (w *worker) Run() { item := w.scheduler.nextWorkItem() blockedWorkers.Dec() - waitElapsed := time.Now().Sub(waitStart) + waitElapsed := time.Since(waitStart) workerIdleTime.Add(waitElapsed.Seconds()) // If no item is returned, worker is safe to terminate @@ -80,7 +80,7 @@ func (w *worker) Evaluate(userID string, item *workItem) { } level.Debug(logger).Log("msg", "evaluating rules...", "num_rules", len(item.group.Rules())) - instrument.CollectedRequest(ctx, "Evaluate", evalDuration, nil, func(ctx native_ctx.Context) error { + err := instrument.CollectedRequest(ctx, "Evaluate", evalDuration, nil, func(ctx native_ctx.Context) error { if span := opentracing.SpanFromContext(ctx); span != nil { span.SetTag("instance", userID) span.SetTag("groupID", item.groupID) @@ -88,4 +88,7 @@ func (w *worker) Evaluate(userID string, item *workItem) { item.group.Eval(ctx, time.Now()) return nil }) + if err != nil { + level.Debug(logger).Log("msg", "failed instrumented worker evaluation", "err", err) + } } diff --git a/pkg/storage/clients/gcp/config_client.go b/pkg/storage/clients/gcp/config_client.go index 6976d1c57e8..bbfa729de1a 100644 --- a/pkg/storage/clients/gcp/config_client.go +++ b/pkg/storage/clients/gcp/config_client.go @@ -64,6 +64,7 @@ func newGCSClient(cfg GCSConfig, client *gstorage.Client) *GCSClient { } } +// ListAlertConfigs returns all of the active alert configus in this store func (g *GCSClient) ListAlertConfigs(ctx context.Context) (map[string]alertStore.AlertConfig, error) { it := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: alertPrefix, @@ -182,6 +183,7 @@ func (g *GCSClient) getAllRuleGroups(ctx context.Context, userID string) ([]stor return rgs, nil } +// ListRuleGroups returns all the active rule groups for a user func (g *GCSClient) ListRuleGroups(ctx context.Context, options store.RuleStoreConditions) (store.RuleGroupList, error) { it := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: generateRuleHandle(options.UserID, options.Namespace, ""), @@ -237,6 +239,7 @@ func (g *GCSClient) getRuleNamespace(ctx context.Context, userID string, namespa return groups, nil } +// GetRuleGroup returns the requested rule group func (g *GCSClient) GetRuleGroup(ctx context.Context, userID string, namespace string, grp string) (store.RuleGroup, error) { handle := generateRuleHandle(userID, namespace, grp) rg, err := g.getRuleGroup(ctx, handle) @@ -277,6 +280,7 @@ func (g *GCSClient) getRuleGroup(ctx context.Context, handle string) (*store.Rul return rg, nil } +// SetRuleGroup sets provided rule group func (g *GCSClient) SetRuleGroup(ctx context.Context, userID string, namespace string, grp rulefmt.RuleGroup) error { rg := store.ToProto(userID, namespace, grp) rgBytes, err := proto.Marshal(&rg) @@ -299,6 +303,7 @@ func (g *GCSClient) SetRuleGroup(ctx context.Context, userID string, namespace s return nil } +// DeleteRuleGroup deletes the specified rule group func (g *GCSClient) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { handle := generateRuleHandle(userID, namespace, group) err := g.bucket.Object(handle).Delete(ctx) From 8ea8cc7d210f1fb450bc6ae6bbcf96d75744d96f Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Mon, 22 Jul 2019 10:03:35 -0400 Subject: [PATCH 53/60] fix ruler tests by adding mutex to mock store Signed-off-by: Jacob Lisi --- pkg/ruler/mock_store.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/ruler/mock_store.go b/pkg/ruler/mock_store.go index ec4e1d422a4..553639f2fe2 100644 --- a/pkg/ruler/mock_store.go +++ b/pkg/ruler/mock_store.go @@ -3,18 +3,22 @@ package ruler import ( "context" "strings" + "sync" "github.com/cortexproject/cortex/pkg/ruler/store" "github.com/prometheus/prometheus/pkg/rulefmt" ) type mockRuleStore struct { + sync.Mutex rules map[string]store.RuleGroup pollPayload map[string][]store.RuleGroup } func (m *mockRuleStore) PollRules(ctx context.Context) (map[string][]store.RuleGroup, error) { + m.Lock() + defer m.Unlock() pollPayload := m.pollPayload m.pollPayload = map[string][]store.RuleGroup{} return pollPayload, nil @@ -28,6 +32,9 @@ func (m *mockRuleStore) RuleStore() store.RuleStore { } func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options store.RuleStoreConditions) (store.RuleGroupList, error) { + m.Lock() + defer m.Unlock() + groupPrefix := options.UserID + ":" namespaces := []string{} @@ -76,6 +83,9 @@ func (m *mockRuleStore) getRuleNamespace(ctx context.Context, userID string, nam } func (m *mockRuleStore) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (store.RuleGroup, error) { + m.Lock() + defer m.Unlock() + groupID := userID + ":" + namespace + ":" + group g, ok := m.rules[groupID] @@ -88,12 +98,18 @@ func (m *mockRuleStore) GetRuleGroup(ctx context.Context, userID string, namespa } func (m *mockRuleStore) SetRuleGroup(ctx context.Context, userID string, namespace string, group rulefmt.RuleGroup) error { + m.Lock() + defer m.Unlock() + groupID := userID + ":" + namespace + ":" + group.Name m.rules[groupID] = store.NewRuleGroup(group.Name, namespace, userID, group.Rules) return nil } func (m *mockRuleStore) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { + m.Lock() + defer m.Unlock() + groupID := userID + ":" + namespace + ":" + group delete(m.rules, groupID) return nil From 234411b87c16357133363df0df107eaf268cc8a1 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Mon, 22 Jul 2019 10:17:13 -0400 Subject: [PATCH 54/60] mod tidy Signed-off-by: Jacob Lisi --- go.sum | 1 - vendor/modules.txt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/go.sum b/go.sum index f949008782d..ff991c71dee 100644 --- a/go.sum +++ b/go.sum @@ -402,7 +402,6 @@ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/prometheus v0.0.0-20190417125241-3cc5f9d88062 h1:3ssjgrtXRvb5iJmsBgFNAUqQFeU/2MpraZjfw+bPOBs= github.com/prometheus/prometheus v0.0.0-20190417125241-3cc5f9d88062/go.mod h1:nqVG0x8dUO5cW+A9KWkaCaaMx7ERCDanTEjTm9i+g3o= -github.com/prometheus/prometheus v2.5.0+incompatible h1:7QPitgO2kOFG8ecuRn9O/4L9+10He72rVRJvMXrE9Hg= github.com/prometheus/tsdb v0.7.0/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.2-0.20190506134726-2ae028114c89 h1:r2TOLePIzxV3KQHQuFz5wPllXq+9Y3g0pjj989nizJo= github.com/prometheus/tsdb v0.7.2-0.20190506134726-2ae028114c89/go.mod h1:fSI0j+IUQrDd7+ZtR9WKIGtoYAYAJUKcKhYLG25tN4g= diff --git a/vendor/modules.txt b/vendor/modules.txt index d2dd5064b14..76998bd12bd 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -588,8 +588,8 @@ golang.org/x/tools/cover # google.golang.org/api v0.4.0 google.golang.org/api/option google.golang.org/api/transport/http -google.golang.org/api/cloudresourcemanager/v1 google.golang.org/api/iterator +google.golang.org/api/cloudresourcemanager/v1 google.golang.org/api/transport/grpc google.golang.org/api/googleapi google.golang.org/api/storage/v1 From 5172d088e9a6e02bd1209efda09334fb30993926 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Mon, 22 Jul 2019 10:46:45 -0400 Subject: [PATCH 55/60] remove aws config backend until later Signed-off-by: Jacob Lisi --- pkg/storage/clients/aws/s3_config_client.go | 147 -------------------- 1 file changed, 147 deletions(-) delete mode 100644 pkg/storage/clients/aws/s3_config_client.go diff --git a/pkg/storage/clients/aws/s3_config_client.go b/pkg/storage/clients/aws/s3_config_client.go deleted file mode 100644 index d9d46a55dd4..00000000000 --- a/pkg/storage/clients/aws/s3_config_client.go +++ /dev/null @@ -1,147 +0,0 @@ -package aws - -import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" - "strings" - - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3iface" - "github.com/prometheus/prometheus/pkg/rulefmt" - - alertstore "github.com/cortexproject/cortex/pkg/alertmanager/storage" - "github.com/cortexproject/cortex/pkg/ruler/store" - "github.com/cortexproject/cortex/pkg/util/flagext" - awscommon "github.com/weaveworks/common/aws" -) - -var ( - alertPrefix = "alerts/" - rulePrefix = "rules/" -) - -type s3ConfigClient struct { - bucketName string - S3 s3iface.S3API -} - -// S3Config configures the s3ConfigClient -type S3Config struct { - S3 flagext.URLValue - S3ForcePathStyle bool -} - -// NewS3ConfigClient makes a new S3-backed ObjectClient. -func NewS3ConfigClient(cfg S3Config) (alertstore.AlertStore, error) { - if cfg.S3.URL == nil { - return nil, fmt.Errorf("no URL specified for S3") - } - s3Config, err := awscommon.ConfigFromURL(cfg.S3.URL) - if err != nil { - return nil, err - } - - s3Config = s3Config.WithS3ForcePathStyle(cfg.S3ForcePathStyle) // support for Path Style S3 url if has the flag - - s3Config = s3Config.WithMaxRetries(0) // We do our own retries, so we can monitor them - s3Client := s3.New(session.New(s3Config)) - bucketName := strings.TrimPrefix(cfg.S3.URL.Path, "/") - client := &s3ConfigClient{ - S3: s3Client, - bucketName: bucketName, - } - - return client, nil -} - -func (s *s3ConfigClient) ListAlertConfigs(ctx context.Context) (map[string]alertstore.AlertConfig, error) { - var continuation *string - alerts := map[string]alertstore.AlertConfig{} - - for { - result, err := s.listAlerts(ctx, continuation) - if err != nil { - return nil, err - } - - if !*result.IsTruncated { - break - } - - for _, alertObj := range result.Contents { - cfg, err := s.getAlertConfig(ctx, alertObj.Key) - if err != nil { - return nil, err - } - user := strings.TrimPrefix(*alertObj.Key, alertPrefix) - alerts[user] = cfg - } - } - - return alerts, nil -} - -func (s *s3ConfigClient) listAlerts(ctx context.Context, continuation *string) (*s3.ListObjectsV2Output, error) { - return s.S3.ListObjectsV2(&s3.ListObjectsV2Input{ - Bucket: &s.bucketName, - Prefix: &alertPrefix, - ContinuationToken: continuation, - }) -} - -func (s *s3ConfigClient) getAlertConfig(ctx context.Context, key *string) (alertstore.AlertConfig, error) { - cfg, err := s.S3.GetObject(&s3.GetObjectInput{ - Bucket: &s.bucketName, - Key: key, - }) - - if err != nil { - return alertstore.AlertConfig{}, err - } - - defer cfg.Body.Close() - - buf, err := ioutil.ReadAll(cfg.Body) - if err != nil { - return alertstore.AlertConfig{}, err - } - - config := alertstore.AlertConfig{} - err = json.Unmarshal(buf, &config) - if err != nil { - return alertstore.AlertConfig{}, err - } - - return config, nil -} - -func (s *s3ConfigClient) GetAlertConfig(ctx context.Context, id string) (alertstore.AlertConfig, error) { - panic("not implemented") -} - -func (s *s3ConfigClient) SetAlertConfig(ctx context.Context, id string, cfg alertstore.AlertConfig) error { - panic("not implemented") -} - -func (s *s3ConfigClient) DeleteAlertConfig(ctx context.Context, id string) error { - panic("not implemented") -} - -func (s *s3ConfigClient) ListRuleGroups(ctx context.Context, options store.RuleStoreConditions) (store.RuleGroupList, error) { - panic("not implemented") -} - -func (s *s3ConfigClient) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (store.RuleGroup, error) { - panic("not implemented") -} - -func (s *s3ConfigClient) SetRuleGroup(ctx context.Context, userID string, namespace string, group rulefmt.RuleGroup) error { - panic("not implemented") -} - -func (s *s3ConfigClient) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { - panic("not implemented") -} From 12d98fc62cee8560500684c09f3ee633e882436c Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Tue, 23 Jul 2019 10:42:54 -0400 Subject: [PATCH 56/60] remove unused package Signed-off-by: Jacob Lisi --- pkg/configs/legacy_configs/configs.go | 298 --------------------- pkg/configs/legacy_configs/configs_test.go | 105 -------- 2 files changed, 403 deletions(-) delete mode 100644 pkg/configs/legacy_configs/configs.go delete mode 100644 pkg/configs/legacy_configs/configs_test.go diff --git a/pkg/configs/legacy_configs/configs.go b/pkg/configs/legacy_configs/configs.go deleted file mode 100644 index fcf9c593125..00000000000 --- a/pkg/configs/legacy_configs/configs.go +++ /dev/null @@ -1,298 +0,0 @@ -package configs - -import ( - "encoding/json" - "fmt" - "time" - - "github.com/go-kit/kit/log" - "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/pkg/rulefmt" - "github.com/prometheus/prometheus/promql" - "github.com/prometheus/prometheus/rules" - - legacy_promql "github.com/cortexproject/cortex/pkg/configs/legacy_promql" - "github.com/cortexproject/cortex/pkg/util" -) - -// An ID is the ID of a single users's Cortex configuration. When a -// configuration changes, it gets a new ID. -type ID int - -// RuleFormatVersion indicates which Prometheus rule format (v1 vs. v2) to use in parsing. -type RuleFormatVersion int - -const ( - // RuleFormatV1 is the Prometheus 1.x rule format. - RuleFormatV1 RuleFormatVersion = iota - // RuleFormatV2 is the Prometheus 2.x rule format. - RuleFormatV2 RuleFormatVersion = iota -) - -// IsValid returns whether the rules format version is a valid (known) version. -func (v RuleFormatVersion) IsValid() bool { - switch v { - case RuleFormatV1, RuleFormatV2: - return true - default: - return false - } -} - -// MarshalJSON implements json.Marshaler. -func (v RuleFormatVersion) MarshalJSON() ([]byte, error) { - switch v { - case RuleFormatV1: - return json.Marshal("1") - case RuleFormatV2: - return json.Marshal("2") - default: - return nil, fmt.Errorf("unknown rule format version %d", v) - } -} - -// UnmarshalJSON implements json.Unmarshaler. -func (v *RuleFormatVersion) UnmarshalJSON(data []byte) error { - var s string - if err := json.Unmarshal(data, &s); err != nil { - return err - } - switch s { - case "1": - *v = RuleFormatV1 - case "2": - *v = RuleFormatV2 - default: - return fmt.Errorf("unknown rule format version %q", string(data)) - } - return nil -} - -// A Config is a Cortex configuration for a single user. -type Config struct { - // RulesFiles maps from a rules filename to file contents. - RulesConfig RulesConfig - TemplateFiles map[string]string - AlertmanagerConfig string -} - -// configCompat is a compatibility struct to support old JSON config blobs -// saved in the config DB that didn't have a rule format version yet and -// just had a top-level field for the rule files. -type configCompat struct { - RulesFiles map[string]string `json:"rules_files"` - RuleFormatVersion RuleFormatVersion `json:"rule_format_version"` - TemplateFiles map[string]string `json:"template_files"` - AlertmanagerConfig string `json:"alertmanager_config"` -} - -// MarshalJSON implements json.Marshaler. -func (c Config) MarshalJSON() ([]byte, error) { - compat := &configCompat{ - RulesFiles: c.RulesConfig.Files, - RuleFormatVersion: c.RulesConfig.FormatVersion, - TemplateFiles: c.TemplateFiles, - AlertmanagerConfig: c.AlertmanagerConfig, - } - - return json.Marshal(compat) -} - -// UnmarshalJSON implements json.Unmarshaler. -func (c *Config) UnmarshalJSON(data []byte) error { - compat := configCompat{} - if err := json.Unmarshal(data, &compat); err != nil { - return err - } - *c = Config{ - RulesConfig: RulesConfig{ - Files: compat.RulesFiles, - FormatVersion: compat.RuleFormatVersion, - }, - TemplateFiles: compat.TemplateFiles, - AlertmanagerConfig: compat.AlertmanagerConfig, - } - return nil -} - -// View is what's returned from the Weave Cloud configs service -// when we ask for all Cortex configurations. -// -// The configs service is essentially a JSON blob store that gives each -// _version_ of a configuration a unique ID and guarantees that later versions -// have greater IDs. -type View struct { - ID ID `json:"id"` - Config Config `json:"config"` - DeletedAt time.Time `json:"deleted_at"` -} - -// GetVersionedRulesConfig specializes the view to just the rules config. -func (v View) GetVersionedRulesConfig() *VersionedRulesConfig { - if v.Config.RulesConfig.Files == nil { - return nil - } - return &VersionedRulesConfig{ - ID: v.ID, - Config: v.Config.RulesConfig, - DeletedAt: v.DeletedAt, - } -} - -// RulesConfig is the rules configuration for a particular organization. -type RulesConfig struct { - FormatVersion RuleFormatVersion `json:"format_version"` - Files map[string]string `json:"files"` -} - -// Equal compares two RulesConfigs for equality. -// -// instance Eq RulesConfig -func (c RulesConfig) Equal(o RulesConfig) bool { - if c.FormatVersion != o.FormatVersion { - return false - } - if len(o.Files) != len(c.Files) { - return false - } - for k, v1 := range c.Files { - v2, ok := o.Files[k] - if !ok || v1 != v2 { - return false - } - } - return true -} - -// Parse parses and validates the content of the rule files in a RulesConfig -// according to the passed rule format version. -func (c RulesConfig) Parse() (map[string][]rules.Rule, error) { - switch c.FormatVersion { - case RuleFormatV1: - return c.parseV1() - case RuleFormatV2: - return c.parseV2() - default: - return nil, fmt.Errorf("unknown rule format version %v", c.FormatVersion) - } -} - -// parseV2 parses and validates the content of the rule files in a RulesConfig -// according to the Prometheus 2.x rule format. -// -// NOTE: On one hand, we cannot return fully-fledged lists of rules.Group -// here yet, as creating a rules.Group requires already -// passing in rules.ManagerOptions options (which in turn require a -// notifier, appender, etc.), which we do not want to create simply -// for parsing. On the other hand, we should not return barebones -// rulefmt.RuleGroup sets here either, as only a fully-converted rules.Rule -// is able to track alert states over multiple rule evaluations. The caller -// would otherwise have to ensure to convert the rulefmt.RuleGroup only exactly -// once, not for every evaluation (or risk losing alert pending states). So -// it's probably better to just return a set of rules.Rule here. -func (c RulesConfig) parseV2() (map[string][]rules.Rule, error) { - groups := map[string][]rules.Rule{} - - for fn, content := range c.Files { - rgs, errs := rulefmt.Parse([]byte(content)) - if len(errs) > 0 { - return nil, fmt.Errorf("error parsing %s: %v", fn, errs[0]) - } - - for _, rg := range rgs.Groups { - rls := make([]rules.Rule, 0, len(rg.Rules)) - for _, rl := range rg.Rules { - expr, err := promql.ParseExpr(rl.Expr) - if err != nil { - return nil, err - } - - if rl.Alert != "" { - rls = append(rls, rules.NewAlertingRule( - rl.Alert, - expr, - time.Duration(rl.For), - labels.FromMap(rl.Labels), - labels.FromMap(rl.Annotations), - true, - log.With(util.Logger, "alert", rl.Alert), - )) - continue - } - rls = append(rls, rules.NewRecordingRule( - rl.Record, - expr, - labels.FromMap(rl.Labels), - )) - } - - // Group names have to be unique in Prometheus, but only within one rules file. - groups[rg.Name+";"+fn] = rls - } - } - - return groups, nil -} - -// parseV1 parses and validates the content of the rule files in a RulesConfig -// according to the Prometheus 1.x rule format. -// -// The same comment about rule groups as on ParseV2() applies here. -func (c RulesConfig) parseV1() (map[string][]rules.Rule, error) { - result := map[string][]rules.Rule{} - for fn, content := range c.Files { - stmts, err := legacy_promql.ParseStmts(content) - if err != nil { - return nil, fmt.Errorf("error parsing %s: %s", fn, err) - } - ra := []rules.Rule{} - for _, stmt := range stmts { - var rule rules.Rule - - switch r := stmt.(type) { - case *legacy_promql.AlertStmt: - // legacy_promql.ParseStmts has parsed the whole rule for us. - // Ideally we'd just use r.Expr and pass that to rules.NewAlertingRule, - // but it is of the type legacy_proql.Expr and not promql.Expr. - // So we convert it back to a string, and then parse it again with the - // upstream parser to get it into the right type. - expr, err := promql.ParseExpr(r.Expr.String()) - if err != nil { - return nil, err - } - - rule = rules.NewAlertingRule( - r.Name, expr, r.Duration, r.Labels, r.Annotations, true, - log.With(util.Logger, "alert", r.Name), - ) - - case *legacy_promql.RecordStmt: - expr, err := promql.ParseExpr(r.Expr.String()) - if err != nil { - return nil, err - } - - rule = rules.NewRecordingRule(r.Name, expr, r.Labels) - - default: - return nil, fmt.Errorf("ruler.GetRules: unknown statement type") - } - ra = append(ra, rule) - } - result[fn] = ra - } - return result, nil -} - -// VersionedRulesConfig is a RulesConfig together with a version. -// `data Versioned a = Versioned { id :: ID , config :: a }` -type VersionedRulesConfig struct { - ID ID `json:"id"` - Config RulesConfig `json:"config"` - DeletedAt time.Time `json:"deleted_at"` -} - -// IsDeleted tells you if the config is deleted. -func (vr VersionedRulesConfig) IsDeleted() bool { - return !vr.DeletedAt.IsZero() -} diff --git a/pkg/configs/legacy_configs/configs_test.go b/pkg/configs/legacy_configs/configs_test.go deleted file mode 100644 index d5c83c80041..00000000000 --- a/pkg/configs/legacy_configs/configs_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package configs - -import ( - "encoding/json" - "strconv" - "testing" - "time" - - "github.com/go-kit/kit/log" - "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/promql" - "github.com/prometheus/prometheus/rules" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/cortexproject/cortex/pkg/util" -) - -func TestUnmarshalLegacyConfigWithMissingRuleFormatVersionSucceeds(t *testing.T) { - actual := Config{} - buf := []byte(`{"rules_files": {"a": "b"}}`) - assert.Nil(t, json.Unmarshal(buf, &actual)) - - expected := Config{ - RulesConfig: RulesConfig{ - Files: map[string]string{ - "a": "b", - }, - FormatVersion: RuleFormatV1, - }, - } - - assert.Equal(t, expected, actual) -} - -func TestParseLegacyAlerts(t *testing.T) { - parsed, err := promql.ParseExpr("up == 0") - require.NoError(t, err) - rule := rules.NewAlertingRule( - "TestAlert", - parsed, - 5*time.Minute, - labels.Labels{ - labels.Label{Name: "severity", Value: "critical"}, - }, - labels.Labels{ - labels.Label{Name: "message", Value: "I am a message"}, - }, - true, - log.With(util.Logger, "alert", "TestAlert"), - ) - - for i, tc := range []struct { - cfg RulesConfig - expected map[string][]rules.Rule - }{ - { - cfg: RulesConfig{ - FormatVersion: RuleFormatV1, - Files: map[string]string{ - "legacy.rules": ` - ALERT TestAlert - IF up == 0 - FOR 5m - LABELS { severity = "critical" } - ANNOTATIONS { - message = "I am a message" - } - `, - }, - }, - expected: map[string][]rules.Rule{ - "legacy.rules": {rule}, - }, - }, - { - cfg: RulesConfig{ - FormatVersion: RuleFormatV2, - Files: map[string]string{ - "alerts.yaml": ` -groups: -- name: example - rules: - - alert: TestAlert - expr: up == 0 - for: 5m - labels: - severity: critical - annotations: - message: I am a message -`, - }, - }, - expected: map[string][]rules.Rule{ - "example;alerts.yaml": {rule}, - }, - }, - } { - t.Run(strconv.Itoa(i), func(t *testing.T) { - rules, err := tc.cfg.Parse() - require.NoError(t, err) - require.Equal(t, tc.expected, rules) - }) - } -} From 7f3d836d80af16a81358fcea2d604d413331636a Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Tue, 23 Jul 2019 13:01:11 -0400 Subject: [PATCH 57/60] move storage definitions to storage subpackages Signed-off-by: Jacob Lisi --- Makefile | 2 +- pkg/alertmanager/api.go | 4 +- pkg/alertmanager/kv_poller.go | 18 +- pkg/alertmanager/multitenant.go | 18 +- pkg/alertmanager/storage.go | 6 +- pkg/ruler/api.go | 2 +- pkg/ruler/kv_poller.go | 22 +- pkg/ruler/mock_store.go | 30 +- pkg/ruler/ruler.go | 25 +- pkg/ruler/scheduler.go | 12 +- pkg/ruler/scheduler_test.go | 10 +- pkg/ruler/storage.go | 6 +- .../storage => storage/alerts}/alert_store.go | 2 +- pkg/storage/clients/client_test.go | 8 +- pkg/storage/clients/configdb/client.go | 20 +- pkg/storage/clients/gcp/config_client.go | 56 +-- pkg/storage/clients/gcp/fixtures.go | 6 +- pkg/{ruler/store => storage/rules}/compat.go | 8 +- pkg/{ruler/store => storage/rules}/group.go | 4 +- .../store.pb.go => storage/rules/rules.pb.go} | 340 +++++++++--------- .../store.proto => storage/rules/rules.proto} | 8 +- pkg/{ruler/store => storage/rules}/store.go | 2 +- pkg/storage/testutils/testutils.go | 6 +- 23 files changed, 308 insertions(+), 307 deletions(-) rename pkg/{alertmanager/storage => storage/alerts}/alert_store.go (97%) rename pkg/{ruler/store => storage/rules}/compat.go (94%) rename pkg/{ruler/store => storage/rules}/group.go (98%) rename pkg/{ruler/store/store.pb.go => storage/rules/rules.pb.go} (76%) rename pkg/{ruler/store/store.proto => storage/rules/rules.proto} (91%) rename pkg/{ruler/store => storage/rules}/store.go (99%) diff --git a/Makefile b/Makefile index e3125c9c057..59257e45478 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ pkg/ring/ring.pb.go: pkg/ring/ring.proto pkg/querier/frontend/frontend.pb.go: pkg/querier/frontend/frontend.proto pkg/chunk/storage/caching_index_client.pb.go: pkg/chunk/storage/caching_index_client.proto pkg/distributor/ha_tracker.pb.go: pkg/distributor/ha_tracker.proto -pkg/ruler/store/store.pb.go: pkg/ruler/store/store.proto +pkg/storage/rules/rules.pb.go: pkg/storage/rules/rules.proto pkg/util/usertracker/usertracker.pb.go: pkg/util/usertracker/usertracker.proto all: $(UPTODATE_FILES) test: protos diff --git a/pkg/alertmanager/api.go b/pkg/alertmanager/api.go index 74d61e24e75..998ce80437e 100644 --- a/pkg/alertmanager/api.go +++ b/pkg/alertmanager/api.go @@ -4,7 +4,7 @@ import ( "io/ioutil" "net/http" - "github.com/cortexproject/cortex/pkg/alertmanager/storage" + "github.com/cortexproject/cortex/pkg/storage/alerts" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "github.com/gorilla/mux" @@ -93,7 +93,7 @@ func (am *MultitenantAlertmanager) setUserConfig(w http.ResponseWriter, r *http. return } - cfg := storage.AlertConfig{} + cfg := alerts.AlertConfig{} err = yaml.Unmarshal(payload, &cfg) if err != nil { level.Error(logger).Log("err", err.Error()) diff --git a/pkg/alertmanager/kv_poller.go b/pkg/alertmanager/kv_poller.go index 3ceffab39a4..7cb6119b832 100644 --- a/pkg/alertmanager/kv_poller.go +++ b/pkg/alertmanager/kv_poller.go @@ -3,7 +3,7 @@ package alertmanager import ( "context" - "github.com/cortexproject/cortex/pkg/alertmanager/storage" + "github.com/cortexproject/cortex/pkg/storage/alerts" "github.com/cortexproject/cortex/pkg/util/usertracker" ) @@ -11,12 +11,12 @@ import ( // Retrieves the updated configuration from the backend type trackedAlertPoller struct { tracker *usertracker.Tracker - store storage.AlertStore + store alerts.AlertStore initialized bool } -func newTrackedAlertPoller(tracker *usertracker.Tracker, store storage.AlertStore) (*trackedAlertPoller, error) { +func newTrackedAlertPoller(tracker *usertracker.Tracker, store alerts.AlertStore) (*trackedAlertPoller, error) { return &trackedAlertPoller{ tracker: tracker, store: store, @@ -34,8 +34,8 @@ func (p *trackedAlertPoller) trackedAlertStore() *trackedAlertStore { // PollAlerts returns the alerts changed since the last poll // All alert configurations are returned on the first poll -func (p *trackedAlertPoller) PollAlerts(ctx context.Context) (map[string]storage.AlertConfig, error) { - updatedConfigs := map[string]storage.AlertConfig{} +func (p *trackedAlertPoller) PollAlerts(ctx context.Context) (map[string]alerts.AlertConfig, error) { + updatedConfigs := map[string]alerts.AlertConfig{} // First poll will return all rule groups if !p.initialized { @@ -67,21 +67,21 @@ func (p *trackedAlertPoller) Stop() { type trackedAlertStore struct { tracker *usertracker.Tracker - store storage.AlertStore + store alerts.AlertStore } // ListAlertConfigs passes through to the embedded alert store -func (w *trackedAlertStore) ListAlertConfigs(ctx context.Context) (map[string]storage.AlertConfig, error) { +func (w *trackedAlertStore) ListAlertConfigs(ctx context.Context) (map[string]alerts.AlertConfig, error) { return w.store.ListAlertConfigs(ctx) } // GetAlertConfig passes through to the embedded alert store -func (w *trackedAlertStore) GetAlertConfig(ctx context.Context, id string) (storage.AlertConfig, error) { +func (w *trackedAlertStore) GetAlertConfig(ctx context.Context, id string) (alerts.AlertConfig, error) { return w.store.GetAlertConfig(ctx, id) } // SetAlertConfig passes through to the embedded alert store, and tracks a user change -func (w *trackedAlertStore) SetAlertConfig(ctx context.Context, id string, cfg storage.AlertConfig) error { +func (w *trackedAlertStore) SetAlertConfig(ctx context.Context, id string, cfg alerts.AlertConfig) error { err := w.store.SetAlertConfig(ctx, id, cfg) if err != nil { return err diff --git a/pkg/alertmanager/multitenant.go b/pkg/alertmanager/multitenant.go index b2bbf92dfe7..b10ade1c393 100644 --- a/pkg/alertmanager/multitenant.go +++ b/pkg/alertmanager/multitenant.go @@ -22,7 +22,7 @@ import ( "github.com/weaveworks/common/user" "github.com/weaveworks/mesh" - "github.com/cortexproject/cortex/pkg/alertmanager/storage" + "github.com/cortexproject/cortex/pkg/storage/alerts" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" ) @@ -215,8 +215,8 @@ func (cfg *MultitenantAlertmanagerConfig) RegisterFlags(f *flag.FlagSet) { type MultitenantAlertmanager struct { cfg *MultitenantAlertmanagerConfig - store storage.AlertStore - poller storage.AlertPoller + store alerts.AlertStore + poller alerts.AlertPoller // The fallback config is stored as a string and parsed every time it's needed // because we mutate the parsed results and don't want those changes to take @@ -224,7 +224,7 @@ type MultitenantAlertmanager struct { fallbackConfig string // All the organization configurations that we have. Only used for instrumentation. - cfgs map[string]storage.AlertConfig + cfgs map[string]alerts.AlertConfig alertmanagersMtx sync.Mutex alertmanagers map[string]*Alertmanager @@ -327,7 +327,7 @@ func (am *MultitenantAlertmanager) Stop() { // Load the full set of configurations from the server, retrying with backoff // until we can get them. -func (am *MultitenantAlertmanager) loadAllConfigs() map[string]storage.AlertConfig { +func (am *MultitenantAlertmanager) loadAllConfigs() map[string]alerts.AlertConfig { backoff := util.NewBackoff(context.Background(), backoffConfig) for { cfgs, err := am.poll() @@ -350,7 +350,7 @@ func (am *MultitenantAlertmanager) updateConfigs(now time.Time) error { } // poll the configuration server. Not re-entrant. -func (am *MultitenantAlertmanager) poll() (map[string]storage.AlertConfig, error) { +func (am *MultitenantAlertmanager) poll() (map[string]alerts.AlertConfig, error) { cfgs, err := am.poller.PollAlerts(context.Background()) if err != nil { level.Warn(util.Logger).Log("msg", "MultitenantAlertmanager: configs server poll failed", "err", err) @@ -359,7 +359,7 @@ func (am *MultitenantAlertmanager) poll() (map[string]storage.AlertConfig, error return cfgs, nil } -func (am *MultitenantAlertmanager) addNewConfigs(cfgs map[string]storage.AlertConfig) { +func (am *MultitenantAlertmanager) addNewConfigs(cfgs map[string]alerts.AlertConfig) { // TODO: instrument how many configs we have, both valid & invalid. level.Debug(util.Logger).Log("msg", "adding configurations", "num_configs", len(cfgs)) for userID, config := range cfgs { @@ -423,7 +423,7 @@ func (am *MultitenantAlertmanager) createTemplatesFile(userID, fn, content strin // setConfig applies the given configuration to the alertmanager for `userID`, // creating an alertmanager if it doesn't already exist. -func (am *MultitenantAlertmanager) setConfig(userID string, config storage.AlertConfig) error { +func (am *MultitenantAlertmanager) setConfig(userID string, config alerts.AlertConfig) error { _, hasExisting := am.alertmanagers[userID] var amConfig *amconfig.Config var err error @@ -486,7 +486,7 @@ func (am *MultitenantAlertmanager) setConfig(userID string, config storage.Alert } // alertmanagerConfigFromConfig returns the Alertmanager config from the Cortex configuration. -func alertmanagerConfigFromConfig(c storage.AlertConfig) (*amconfig.Config, error) { +func alertmanagerConfigFromConfig(c alerts.AlertConfig) (*amconfig.Config, error) { cfg, err := amconfig.Load(c.AlertmanagerConfig) if err != nil { return nil, fmt.Errorf("error parsing Alertmanager config: %s", err) diff --git a/pkg/alertmanager/storage.go b/pkg/alertmanager/storage.go index 98d04a2d60a..e75f59f53db 100644 --- a/pkg/alertmanager/storage.go +++ b/pkg/alertmanager/storage.go @@ -5,7 +5,7 @@ import ( "flag" "fmt" - "github.com/cortexproject/cortex/pkg/alertmanager/storage" + "github.com/cortexproject/cortex/pkg/storage/alerts" "github.com/cortexproject/cortex/pkg/storage/clients/configdb" "github.com/cortexproject/cortex/pkg/storage/clients/gcp" "github.com/cortexproject/cortex/pkg/util/usertracker" @@ -30,9 +30,9 @@ func (cfg *AlertStoreConfig) RegisterFlags(f *flag.FlagSet) { } // NewAlertStore returns a new rule storage backend poller and store -func NewAlertStore(cfg AlertStoreConfig) (storage.AlertPoller, storage.AlertStore, error) { +func NewAlertStore(cfg AlertStoreConfig) (alerts.AlertPoller, alerts.AlertStore, error) { var ( - alertStore storage.AlertStore + alertStore alerts.AlertStore err error ) switch cfg.Type { diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go index 56d164ab4f1..b9f98b7a8bc 100644 --- a/pkg/ruler/api.go +++ b/pkg/ruler/api.go @@ -7,7 +7,7 @@ import ( "github.com/prometheus/prometheus/pkg/rulefmt" - "github.com/cortexproject/cortex/pkg/ruler/store" + store "github.com/cortexproject/cortex/pkg/storage/rules" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "github.com/gorilla/mux" diff --git a/pkg/ruler/kv_poller.go b/pkg/ruler/kv_poller.go index a807585dbe5..d4f1958841f 100644 --- a/pkg/ruler/kv_poller.go +++ b/pkg/ruler/kv_poller.go @@ -3,7 +3,7 @@ package ruler import ( "context" - "github.com/cortexproject/cortex/pkg/ruler/store" + "github.com/cortexproject/cortex/pkg/storage/rules" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/usertracker" "github.com/go-kit/kit/log/level" @@ -12,12 +12,12 @@ import ( type trackedPoller struct { tracker *usertracker.Tracker - store store.RuleStore + store rules.RuleStore initialized bool } -func newTrackedPoller(tracker *usertracker.Tracker, store store.RuleStore) (*trackedPoller, error) { +func newTrackedPoller(tracker *usertracker.Tracker, store rules.RuleStore) (*trackedPoller, error) { return &trackedPoller{ tracker: tracker, store: store, @@ -33,21 +33,21 @@ func (p *trackedPoller) trackedRuleStore() *trackedRuleStore { } } -func (p *trackedPoller) PollRules(ctx context.Context) (map[string][]store.RuleGroup, error) { - updatedRules := map[string][]store.RuleGroup{} +func (p *trackedPoller) PollRules(ctx context.Context) (map[string][]rules.RuleGroup, error) { + updatedRules := map[string][]rules.RuleGroup{} level.Debug(util.Logger).Log("msg", "polling for new rules") // First poll will return all rule groups if !p.initialized { level.Debug(util.Logger).Log("msg", "first poll, loading all rules") - rgs, err := p.store.ListRuleGroups(ctx, store.RuleStoreConditions{}) + rgs, err := p.store.ListRuleGroups(ctx, rules.RuleStoreConditions{}) if err != nil { return nil, err } for _, rg := range rgs { if _, exists := updatedRules[rg.User()]; !exists { - updatedRules[rg.User()] = []store.RuleGroup{rg} + updatedRules[rg.User()] = []rules.RuleGroup{rg} } else { updatedRules[rg.User()] = append(updatedRules[rg.User()], rg) } @@ -57,7 +57,7 @@ func (p *trackedPoller) PollRules(ctx context.Context) (map[string][]store.RuleG users := p.tracker.GetUpdatedUsers(ctx) for _, u := range users { level.Debug(util.Logger).Log("msg", "poll found updated user", "user", u) - rgs, err := p.store.ListRuleGroups(ctx, store.RuleStoreConditions{ + rgs, err := p.store.ListRuleGroups(ctx, rules.RuleStoreConditions{ UserID: u, }) if err != nil { @@ -77,16 +77,16 @@ func (p *trackedPoller) Stop() { type trackedRuleStore struct { tracker *usertracker.Tracker - store store.RuleStore + store rules.RuleStore } // ListRuleGroups returns set of all rule groups matching the provided conditions -func (w *trackedRuleStore) ListRuleGroups(ctx context.Context, options store.RuleStoreConditions) (store.RuleGroupList, error) { +func (w *trackedRuleStore) ListRuleGroups(ctx context.Context, options rules.RuleStoreConditions) (rules.RuleGroupList, error) { return w.store.ListRuleGroups(ctx, options) } // GetRuleGroup retrieves the specified rule group from the backend store -func (w *trackedRuleStore) GetRuleGroup(ctx context.Context, userID, namespace, group string) (store.RuleGroup, error) { +func (w *trackedRuleStore) GetRuleGroup(ctx context.Context, userID, namespace, group string) (rules.RuleGroup, error) { return w.store.GetRuleGroup(ctx, userID, namespace, group) } diff --git a/pkg/ruler/mock_store.go b/pkg/ruler/mock_store.go index 553639f2fe2..73c24c82bac 100644 --- a/pkg/ruler/mock_store.go +++ b/pkg/ruler/mock_store.go @@ -5,40 +5,40 @@ import ( "strings" "sync" - "github.com/cortexproject/cortex/pkg/ruler/store" + "github.com/cortexproject/cortex/pkg/storage/rules" "github.com/prometheus/prometheus/pkg/rulefmt" ) type mockRuleStore struct { sync.Mutex - rules map[string]store.RuleGroup + rules map[string]rules.RuleGroup - pollPayload map[string][]store.RuleGroup + pollPayload map[string][]rules.RuleGroup } -func (m *mockRuleStore) PollRules(ctx context.Context) (map[string][]store.RuleGroup, error) { +func (m *mockRuleStore) PollRules(ctx context.Context) (map[string][]rules.RuleGroup, error) { m.Lock() defer m.Unlock() pollPayload := m.pollPayload - m.pollPayload = map[string][]store.RuleGroup{} + m.pollPayload = map[string][]rules.RuleGroup{} return pollPayload, nil } func (m *mockRuleStore) Stop() {} // RuleStore returns an RuleStore from the client -func (m *mockRuleStore) RuleStore() store.RuleStore { +func (m *mockRuleStore) RuleStore() rules.RuleStore { return m } -func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options store.RuleStoreConditions) (store.RuleGroupList, error) { +func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options rules.RuleStoreConditions) (rules.RuleGroupList, error) { m.Lock() defer m.Unlock() groupPrefix := options.UserID + ":" namespaces := []string{} - nss := store.RuleGroupList{} + nss := rules.RuleGroupList{} for n := range m.rules { if strings.HasPrefix(n, groupPrefix) { components := strings.Split(n, ":") @@ -50,7 +50,7 @@ func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options store.RuleSt } if len(namespaces) == 0 { - return nss, store.ErrUserNotFound + return nss, rules.ErrUserNotFound } for _, n := range namespaces { @@ -65,10 +65,10 @@ func (m *mockRuleStore) ListRuleGroups(ctx context.Context, options store.RuleSt return nss, nil } -func (m *mockRuleStore) getRuleNamespace(ctx context.Context, userID string, namespace string) (store.RuleGroupList, error) { +func (m *mockRuleStore) getRuleNamespace(ctx context.Context, userID string, namespace string) (rules.RuleGroupList, error) { groupPrefix := userID + ":" + namespace + ":" - ns := store.RuleGroupList{} + ns := rules.RuleGroupList{} for n, g := range m.rules { if strings.HasPrefix(n, groupPrefix) { ns = append(ns, g) @@ -76,13 +76,13 @@ func (m *mockRuleStore) getRuleNamespace(ctx context.Context, userID string, nam } if len(ns) == 0 { - return ns, store.ErrGroupNamespaceNotFound + return ns, rules.ErrGroupNamespaceNotFound } return ns, nil } -func (m *mockRuleStore) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (store.RuleGroup, error) { +func (m *mockRuleStore) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (rules.RuleGroup, error) { m.Lock() defer m.Unlock() @@ -90,7 +90,7 @@ func (m *mockRuleStore) GetRuleGroup(ctx context.Context, userID string, namespa g, ok := m.rules[groupID] if !ok { - return nil, store.ErrGroupNotFound + return nil, rules.ErrGroupNotFound } return g, nil @@ -102,7 +102,7 @@ func (m *mockRuleStore) SetRuleGroup(ctx context.Context, userID string, namespa defer m.Unlock() groupID := userID + ":" + namespace + ":" + group.Name - m.rules[groupID] = store.NewRuleGroup(group.Name, namespace, userID, group.Rules) + m.rules[groupID] = rules.NewRuleGroup(group.Name, namespace, userID, group.Rules) return nil } diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index c910d726fc2..87335f5a2a1 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -15,7 +15,7 @@ import ( "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/notifier" "github.com/prometheus/prometheus/promql" - "github.com/prometheus/prometheus/rules" + promRules "github.com/prometheus/prometheus/rules" promStorage "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/strutil" "golang.org/x/net/context" @@ -23,7 +23,8 @@ import ( "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/ring" - "github.com/cortexproject/cortex/pkg/ruler/store" + "github.com/cortexproject/cortex/pkg/storage/rules" + store "github.com/cortexproject/cortex/pkg/storage/rules" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/weaveworks/common/instrument" @@ -119,7 +120,7 @@ type Ruler struct { lifecycler *ring.Lifecycler ring *ring.Ring - store store.RuleStore + store rules.RuleStore // Per-user notifiers with separate queues. notifiersMtx sync.Mutex @@ -127,7 +128,7 @@ type Ruler struct { // Per-user rules metrics userMetricsMtx sync.Mutex - userMetrics map[string]*rules.Metrics + userMetrics map[string]*promRules.Metrics } // NewRuler creates a new ruler from a distributor and chunk store. @@ -155,7 +156,7 @@ func NewRuler(cfg Config, engine *promql.Engine, queryable promStorage.Queryable notifierCfg: ncfg, notifiers: map[string]*rulerNotifier{}, workerWG: &sync.WaitGroup{}, - userMetrics: map[string]*rules.Metrics{}, + userMetrics: map[string]*promRules.Metrics{}, store: ruleStore, } @@ -235,14 +236,14 @@ func (r *Ruler) newGroup(ctx context.Context, g store.RuleGroup) (*wrappedGroup, if !exists { // Wrap the default register with the users ID and pass reg := prometheus.WrapRegistererWith(prometheus.Labels{"user": user}, prometheus.DefaultRegisterer) - metrics = rules.NewGroupMetrics(reg) + metrics = promRules.NewGroupMetrics(reg) r.userMetrics[user] = metrics } r.userMetricsMtx.Unlock() - opts := &rules.ManagerOptions{ + opts := &promRules.ManagerOptions{ Appendable: appendable, - QueryFunc: rules.EngineQueryFunc(r.engine, r.queryable), + QueryFunc: promRules.EngineQueryFunc(r.engine, r.queryable), Context: context.Background(), ExternalURL: r.alertURL, NotifyFunc: sendAlerts(notifier, r.alertURL.String()), @@ -252,17 +253,17 @@ func (r *Ruler) newGroup(ctx context.Context, g store.RuleGroup) (*wrappedGroup, return newGroup(g.ID(), rls, appendable, opts), nil } -// sendAlerts implements a rules.NotifyFunc for a Notifier. +// sendAlerts implements a promRules.NotifyFunc for a Notifier. // It filters any non-firing alerts from the input. // // Copied from Prometheus's main.go. -func sendAlerts(n *notifier.Manager, externalURL string) rules.NotifyFunc { - return func(ctx native_ctx.Context, expr string, alerts ...*rules.Alert) { +func sendAlerts(n *notifier.Manager, externalURL string) promRules.NotifyFunc { + return func(ctx native_ctx.Context, expr string, alerts ...*promRules.Alert) { var res []*notifier.Alert for _, alert := range alerts { // Only send actually firing alerts. - if alert.State == rules.StatePending { + if alert.State == promRules.StatePending { continue } a := ¬ifier.Alert{ diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index ea19ccae124..fa89711924d 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -9,7 +9,7 @@ import ( "sync" "time" - "github.com/cortexproject/cortex/pkg/ruler/store" + "github.com/cortexproject/cortex/pkg/storage/rules" "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "github.com/jonboulle/clockwork" @@ -72,13 +72,13 @@ func (w workItem) String() string { type userConfig struct { done chan struct{} id string - rules []store.RuleGroup + rules []rules.RuleGroup } -type groupFactory func(context.Context, store.RuleGroup) (*wrappedGroup, error) +type groupFactory func(context.Context, rules.RuleGroup) (*wrappedGroup, error) type scheduler struct { - poller store.RulePoller + poller rules.RulePoller evaluationInterval time.Duration // how often we re-evaluate each rule set q *SchedulingQueue @@ -91,7 +91,7 @@ type scheduler struct { } // newScheduler makes a new scheduler. -func newScheduler(poller store.RulePoller, evaluationInterval, pollInterval time.Duration, groupFn groupFactory) *scheduler { +func newScheduler(poller rules.RulePoller, evaluationInterval, pollInterval time.Duration, groupFn groupFactory) *scheduler { return &scheduler{ poller: poller, evaluationInterval: evaluationInterval, @@ -150,7 +150,7 @@ func (s *scheduler) updateConfigs(ctx context.Context) error { return nil } -func (s *scheduler) addUserConfig(ctx context.Context, userID string, rgs []store.RuleGroup) { +func (s *scheduler) addUserConfig(ctx context.Context, userID string, rgs []rules.RuleGroup) { level.Info(util.Logger).Log("msg", "scheduler: updating rules for user", "user_id", userID, "num_groups", len(rgs)) // create a new userchan for rulegroups of this user diff --git a/pkg/ruler/scheduler_test.go b/pkg/ruler/scheduler_test.go index 2628c27f1cf..b3dae8de0fe 100644 --- a/pkg/ruler/scheduler_test.go +++ b/pkg/ruler/scheduler_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/pkg/ruler/store" + "github.com/cortexproject/cortex/pkg/storage/rules" "github.com/prometheus/prometheus/pkg/rulefmt" "github.com/stretchr/testify/assert" ) @@ -75,12 +75,12 @@ func TestSchedulerRulesOverlap(t *testing.T) { groupTwo := "test2" next := time.Now() - ruleSetsOne := []store.RuleGroup{ - store.NewRuleGroup(groupOne, "default", userID, []rulefmt.Rule{}), + ruleSetsOne := []rules.RuleGroup{ + rules.NewRuleGroup(groupOne, "default", userID, []rulefmt.Rule{}), } - ruleSetsTwo := []store.RuleGroup{ - store.NewRuleGroup(groupTwo, "default", userID, []rulefmt.Rule{}), + ruleSetsTwo := []rules.RuleGroup{ + rules.NewRuleGroup(groupTwo, "default", userID, []rulefmt.Rule{}), } userChanOne := make(chan struct{}) userChanTwo := make(chan struct{}) diff --git a/pkg/ruler/storage.go b/pkg/ruler/storage.go index 8aed879fcc6..7d2e1e7693a 100644 --- a/pkg/ruler/storage.go +++ b/pkg/ruler/storage.go @@ -5,9 +5,9 @@ import ( "flag" "fmt" - "github.com/cortexproject/cortex/pkg/ruler/store" "github.com/cortexproject/cortex/pkg/storage/clients/configdb" "github.com/cortexproject/cortex/pkg/storage/clients/gcp" + "github.com/cortexproject/cortex/pkg/storage/rules" "github.com/cortexproject/cortex/pkg/util/usertracker" ) @@ -30,13 +30,13 @@ func (cfg *RuleStoreConfig) RegisterFlags(f *flag.FlagSet) { } // NewRuleStorage returns a new rule storage backend poller and store -func NewRuleStorage(cfg RuleStoreConfig) (store.RulePoller, store.RuleStore, error) { +func NewRuleStorage(cfg RuleStoreConfig) (rules.RulePoller, rules.RuleStore, error) { if cfg.mock != nil { return cfg.mock, cfg.mock, nil } var ( - ruleStore store.RuleStore + ruleStore rules.RuleStore err error ) switch cfg.Type { diff --git a/pkg/alertmanager/storage/alert_store.go b/pkg/storage/alerts/alert_store.go similarity index 97% rename from pkg/alertmanager/storage/alert_store.go rename to pkg/storage/alerts/alert_store.go index a8246daaf89..b65043fb6a8 100644 --- a/pkg/alertmanager/storage/alert_store.go +++ b/pkg/storage/alerts/alert_store.go @@ -1,4 +1,4 @@ -package storage +package alerts import ( "context" diff --git a/pkg/storage/clients/client_test.go b/pkg/storage/clients/client_test.go index f9e105a5a91..304267196c5 100644 --- a/pkg/storage/clients/client_test.go +++ b/pkg/storage/clients/client_test.go @@ -5,9 +5,9 @@ import ( "testing" "time" - alertStore "github.com/cortexproject/cortex/pkg/alertmanager/storage" - "github.com/cortexproject/cortex/pkg/ruler/store" + "github.com/cortexproject/cortex/pkg/storage/alerts" "github.com/cortexproject/cortex/pkg/storage/clients/gcp" + "github.com/cortexproject/cortex/pkg/storage/rules" "github.com/cortexproject/cortex/pkg/storage/testutils" "github.com/prometheus/prometheus/pkg/rulefmt" "github.com/stretchr/testify/assert" @@ -26,7 +26,7 @@ var ( ) func TestRuleStoreBasic(t *testing.T) { - forAllFixtures(t, func(t *testing.T, _ alertStore.AlertStore, client store.RuleStore) { + forAllFixtures(t, func(t *testing.T, _ alerts.AlertStore, client rules.RuleStore) { const batchSize = 5 ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -47,7 +47,7 @@ func TestRuleStoreBasic(t *testing.T) { }) } -func forAllFixtures(t *testing.T, clientTest func(*testing.T, alertStore.AlertStore, store.RuleStore)) { +func forAllFixtures(t *testing.T, clientTest func(*testing.T, alerts.AlertStore, rules.RuleStore)) { var fixtures []testutils.Fixture fixtures = append(fixtures, gcp.Fixtures...) diff --git a/pkg/storage/clients/configdb/client.go b/pkg/storage/clients/configdb/client.go index 62fdd7faf78..3bf450a7938 100644 --- a/pkg/storage/clients/configdb/client.go +++ b/pkg/storage/clients/configdb/client.go @@ -10,9 +10,9 @@ import ( "strings" "time" - alertstore "github.com/cortexproject/cortex/pkg/alertmanager/storage" "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/ruler/store" + "github.com/cortexproject/cortex/pkg/storage/alerts" + "github.com/cortexproject/cortex/pkg/storage/rules" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/go-kit/kit/log/level" @@ -130,15 +130,15 @@ func (c ConfigsResponse) GetLatestConfigID() configs.ID { } // PollAlerts polls the configdb for updated alerts -func (c *ConfigsClient) PollAlerts(ctx context.Context) (map[string]alertstore.AlertConfig, error) { +func (c *ConfigsClient) PollAlerts(ctx context.Context) (map[string]alerts.AlertConfig, error) { resp, err := c.GetAlerts(ctx, c.lastPoll) if err != nil { return nil, err } - newConfigs := map[string]alertstore.AlertConfig{} + newConfigs := map[string]alerts.AlertConfig{} for user, c := range resp.Configs { - newConfigs[user] = alertstore.AlertConfig{ + newConfigs[user] = alerts.AlertConfig{ TemplateFiles: c.Config.TemplateFiles, AlertmanagerConfig: c.Config.AlertmanagerConfig, } @@ -150,22 +150,22 @@ func (c *ConfigsClient) PollAlerts(ctx context.Context) (map[string]alertstore.A } // PollRules polls the configdb server and returns the updated rule groups -func (c *ConfigsClient) PollRules(ctx context.Context) (map[string][]store.RuleGroup, error) { +func (c *ConfigsClient) PollRules(ctx context.Context) (map[string][]rules.RuleGroup, error) { resp, err := c.GetRules(ctx, c.lastPoll) if err != nil { return nil, err } - newRules := map[string][]store.RuleGroup{} + newRules := map[string][]rules.RuleGroup{} var highestID configs.ID for user, cfg := range resp { if cfg.ID > highestID { highestID = cfg.ID } - userRules := []store.RuleGroup{} + userRules := []rules.RuleGroup{} if cfg.IsDeleted() { - newRules[user] = []store.RuleGroup{} + newRules[user] = []rules.RuleGroup{} } rMap, err := cfg.Config.Parse() if err != nil { @@ -173,7 +173,7 @@ func (c *ConfigsClient) PollRules(ctx context.Context) (map[string][]store.RuleG } for groupSlug, r := range rMap { name, file := decomposeGroupSlug(groupSlug) - userRules = append(userRules, store.FormattedToRuleGroup(user, file, name, r)) + userRules = append(userRules, rules.FormattedToRuleGroup(user, file, name, r)) } newRules[user] = userRules } diff --git a/pkg/storage/clients/gcp/config_client.go b/pkg/storage/clients/gcp/config_client.go index bbfa729de1a..41524d71050 100644 --- a/pkg/storage/clients/gcp/config_client.go +++ b/pkg/storage/clients/gcp/config_client.go @@ -8,8 +8,8 @@ import ( "io/ioutil" "strings" - alertStore "github.com/cortexproject/cortex/pkg/alertmanager/storage" - "github.com/cortexproject/cortex/pkg/ruler/store" + "github.com/cortexproject/cortex/pkg/storage/alerts" + "github.com/cortexproject/cortex/pkg/storage/rules" "github.com/cortexproject/cortex/pkg/util" gstorage "cloud.google.com/go/storage" @@ -65,12 +65,12 @@ func newGCSClient(cfg GCSConfig, client *gstorage.Client) *GCSClient { } // ListAlertConfigs returns all of the active alert configus in this store -func (g *GCSClient) ListAlertConfigs(ctx context.Context) (map[string]alertStore.AlertConfig, error) { +func (g *GCSClient) ListAlertConfigs(ctx context.Context) (map[string]alerts.AlertConfig, error) { it := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: alertPrefix, }) - configs := map[string]alertStore.AlertConfig{} + configs := map[string]alerts.AlertConfig{} for { obj, err := it.Next() @@ -95,38 +95,38 @@ func (g *GCSClient) ListAlertConfigs(ctx context.Context) (map[string]alertStore return configs, nil } -func (g *GCSClient) getAlertConfig(ctx context.Context, obj string) (alertStore.AlertConfig, error) { +func (g *GCSClient) getAlertConfig(ctx context.Context, obj string) (alerts.AlertConfig, error) { reader, err := g.bucket.Object(obj).NewReader(ctx) if err == gstorage.ErrObjectNotExist { level.Debug(util.Logger).Log("msg", "object does not exist", "name", obj) - return alertStore.AlertConfig{}, nil + return alerts.AlertConfig{}, nil } if err != nil { - return alertStore.AlertConfig{}, err + return alerts.AlertConfig{}, err } defer reader.Close() buf, err := ioutil.ReadAll(reader) if err != nil { - return alertStore.AlertConfig{}, err + return alerts.AlertConfig{}, err } - config := alertStore.AlertConfig{} + config := alerts.AlertConfig{} err = json.Unmarshal(buf, &config) if err != nil { - return alertStore.AlertConfig{}, err + return alerts.AlertConfig{}, err } return config, nil } // GetAlertConfig returns a specified users alertmanager configuration -func (g *GCSClient) GetAlertConfig(ctx context.Context, userID string) (alertStore.AlertConfig, error) { +func (g *GCSClient) GetAlertConfig(ctx context.Context, userID string) (alerts.AlertConfig, error) { return g.getAlertConfig(ctx, alertPrefix+userID) } // SetAlertConfig sets a specified users alertmanager configuration -func (g *GCSClient) SetAlertConfig(ctx context.Context, userID string, cfg alertStore.AlertConfig) error { +func (g *GCSClient) SetAlertConfig(ctx context.Context, userID string, cfg alerts.AlertConfig) error { cfgBytes, err := json.Marshal(cfg) if err != nil { return err @@ -155,12 +155,12 @@ func (g *GCSClient) DeleteAlertConfig(ctx context.Context, userID string) error return nil } -func (g *GCSClient) getAllRuleGroups(ctx context.Context, userID string) ([]store.RuleGroup, error) { +func (g *GCSClient) getAllRuleGroups(ctx context.Context, userID string) ([]rules.RuleGroup, error) { it := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: generateRuleHandle(userID, "", ""), }) - rgs := []store.RuleGroup{} + rgs := []rules.RuleGroup{} for { obj, err := it.Next() @@ -169,27 +169,27 @@ func (g *GCSClient) getAllRuleGroups(ctx context.Context, userID string) ([]stor } if err != nil { - return []store.RuleGroup{}, err + return []rules.RuleGroup{}, err } rgProto, err := g.getRuleGroup(ctx, obj.Name) if err != nil { - return []store.RuleGroup{}, err + return []rules.RuleGroup{}, err } - rgs = append(rgs, store.ToRuleGroup(rgProto)) + rgs = append(rgs, rules.ToRuleGroup(rgProto)) } return rgs, nil } // ListRuleGroups returns all the active rule groups for a user -func (g *GCSClient) ListRuleGroups(ctx context.Context, options store.RuleStoreConditions) (store.RuleGroupList, error) { +func (g *GCSClient) ListRuleGroups(ctx context.Context, options rules.RuleStoreConditions) (rules.RuleGroupList, error) { it := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: generateRuleHandle(options.UserID, options.Namespace, ""), }) - groups := []store.RuleGroup{} + groups := []rules.RuleGroup{} for { obj, err := it.Next() if err == iterator.Done { @@ -206,17 +206,17 @@ func (g *GCSClient) ListRuleGroups(ctx context.Context, options store.RuleStoreC if err != nil { return nil, err } - groups = append(groups, store.ToRuleGroup(rg)) + groups = append(groups, rules.ToRuleGroup(rg)) } return groups, nil } -func (g *GCSClient) getRuleNamespace(ctx context.Context, userID string, namespace string) ([]*store.RuleGroupDesc, error) { +func (g *GCSClient) getRuleNamespace(ctx context.Context, userID string, namespace string) ([]*rules.RuleGroupDesc, error) { it := g.bucket.Objects(ctx, &gstorage.Query{ Prefix: generateRuleHandle(userID, namespace, ""), }) - groups := []*store.RuleGroupDesc{} + groups := []*rules.RuleGroupDesc{} for { obj, err := it.Next() @@ -240,7 +240,7 @@ func (g *GCSClient) getRuleNamespace(ctx context.Context, userID string, namespa } // GetRuleGroup returns the requested rule group -func (g *GCSClient) GetRuleGroup(ctx context.Context, userID string, namespace string, grp string) (store.RuleGroup, error) { +func (g *GCSClient) GetRuleGroup(ctx context.Context, userID string, namespace string, grp string) (rules.RuleGroup, error) { handle := generateRuleHandle(userID, namespace, grp) rg, err := g.getRuleGroup(ctx, handle) if err != nil { @@ -248,13 +248,13 @@ func (g *GCSClient) GetRuleGroup(ctx context.Context, userID string, namespace s } if rg == nil { - return nil, store.ErrGroupNotFound + return nil, rules.ErrGroupNotFound } - return store.ToRuleGroup(rg), nil + return rules.ToRuleGroup(rg), nil } -func (g *GCSClient) getRuleGroup(ctx context.Context, handle string) (*store.RuleGroupDesc, error) { +func (g *GCSClient) getRuleGroup(ctx context.Context, handle string) (*rules.RuleGroupDesc, error) { reader, err := g.bucket.Object(handle).NewReader(ctx) if err == gstorage.ErrObjectNotExist { level.Debug(util.Logger).Log("msg", "rule group does not exist", "name", handle) @@ -270,7 +270,7 @@ func (g *GCSClient) getRuleGroup(ctx context.Context, handle string) (*store.Rul return nil, err } - rg := &store.RuleGroupDesc{} + rg := &rules.RuleGroupDesc{} err = proto.Unmarshal(buf, rg) if err != nil { @@ -282,7 +282,7 @@ func (g *GCSClient) getRuleGroup(ctx context.Context, handle string) (*store.Rul // SetRuleGroup sets provided rule group func (g *GCSClient) SetRuleGroup(ctx context.Context, userID string, namespace string, grp rulefmt.RuleGroup) error { - rg := store.ToProto(userID, namespace, grp) + rg := rules.ToProto(userID, namespace, grp) rgBytes, err := proto.Marshal(&rg) if err != nil { return err diff --git a/pkg/storage/clients/gcp/fixtures.go b/pkg/storage/clients/gcp/fixtures.go index c176ba25dff..b2dc7d92262 100644 --- a/pkg/storage/clients/gcp/fixtures.go +++ b/pkg/storage/clients/gcp/fixtures.go @@ -1,8 +1,8 @@ package gcp import ( - alertStore "github.com/cortexproject/cortex/pkg/alertmanager/storage" - "github.com/cortexproject/cortex/pkg/ruler/store" + "github.com/cortexproject/cortex/pkg/storage/alerts" + "github.com/cortexproject/cortex/pkg/storage/rules" "github.com/cortexproject/cortex/pkg/storage/testutils" "github.com/fsouza/fake-gcs-server/fakestorage" ) @@ -21,7 +21,7 @@ func (f *fixture) Name() string { return f.name } -func (f *fixture) Clients() (alertStore.AlertStore, store.RuleStore, error) { +func (f *fixture) Clients() (alerts.AlertStore, rules.RuleStore, error) { f.gcssrv = fakestorage.NewServer(nil) f.gcssrv.CreateBucket("configdb") cli := newGCSClient(GCSConfig{ diff --git a/pkg/ruler/store/compat.go b/pkg/storage/rules/compat.go similarity index 94% rename from pkg/ruler/store/compat.go rename to pkg/storage/rules/compat.go index 596f9f0f45b..2aa8037a6dc 100644 --- a/pkg/ruler/store/compat.go +++ b/pkg/storage/rules/compat.go @@ -1,4 +1,4 @@ -package store +package rules import ( time "time" @@ -35,12 +35,12 @@ func ToProto(user string, namespace string, rl rulefmt.RuleGroup) RuleGroupDesc return rg } -func formattedRuleToProto(rls []rulefmt.Rule) []*Rule { - rules := make([]*Rule, len(rls)) +func formattedRuleToProto(rls []rulefmt.Rule) []*RuleDesc { + rules := make([]*RuleDesc, len(rls)) for i := range rls { f := time.Duration(rls[i].For) - rules[i] = &Rule{ + rules[i] = &RuleDesc{ Expr: rls[i].Expr, Record: rls[i].Record, Alert: rls[i].Alert, diff --git a/pkg/ruler/store/group.go b/pkg/storage/rules/group.go similarity index 98% rename from pkg/ruler/store/group.go rename to pkg/storage/rules/group.go index a1ba363425f..ab1e3d42aa3 100644 --- a/pkg/ruler/store/group.go +++ b/pkg/storage/rules/group.go @@ -1,4 +1,4 @@ -package store +package rules import ( "context" @@ -23,7 +23,7 @@ type Group struct { namespace string user string interval time.Duration - rules []*Rule + rules []*RuleDesc // activeRules allows for the support of the configdb client // TODO: figure out a better way to accomplish this diff --git a/pkg/ruler/store/store.pb.go b/pkg/storage/rules/rules.pb.go similarity index 76% rename from pkg/ruler/store/store.pb.go rename to pkg/storage/rules/rules.pb.go index 55c553da559..d4c15283775 100644 --- a/pkg/ruler/store/store.pb.go +++ b/pkg/storage/rules/rules.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: store.proto +// source: rules.proto -package store +package rules import ( fmt "fmt" @@ -38,7 +38,7 @@ type RuleUpdateDesc struct { func (m *RuleUpdateDesc) Reset() { *m = RuleUpdateDesc{} } func (*RuleUpdateDesc) ProtoMessage() {} func (*RuleUpdateDesc) Descriptor() ([]byte, []int) { - return fileDescriptor_98bbca36ef968dfc, []int{0} + return fileDescriptor_8e722d3e922f0937, []int{0} } func (m *RuleUpdateDesc) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -85,7 +85,7 @@ type RuleGroupDesc struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Namespace string `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` Interval *time.Duration `protobuf:"bytes,3,opt,name=interval,proto3,stdduration" json:"interval,omitempty"` - Rules []*Rule `protobuf:"bytes,4,rep,name=rules,proto3" json:"rules,omitempty"` + Rules []*RuleDesc `protobuf:"bytes,4,rep,name=rules,proto3" json:"rules,omitempty"` Deleted bool `protobuf:"varint,5,opt,name=deleted,proto3" json:"deleted,omitempty"` User string `protobuf:"bytes,6,opt,name=user,proto3" json:"user,omitempty"` } @@ -93,7 +93,7 @@ type RuleGroupDesc struct { func (m *RuleGroupDesc) Reset() { *m = RuleGroupDesc{} } func (*RuleGroupDesc) ProtoMessage() {} func (*RuleGroupDesc) Descriptor() ([]byte, []int) { - return fileDescriptor_98bbca36ef968dfc, []int{1} + return fileDescriptor_8e722d3e922f0937, []int{1} } func (m *RuleGroupDesc) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -143,7 +143,7 @@ func (m *RuleGroupDesc) GetInterval() *time.Duration { return nil } -func (m *RuleGroupDesc) GetRules() []*Rule { +func (m *RuleGroupDesc) GetRules() []*RuleDesc { if m != nil { return m.Rules } @@ -164,7 +164,7 @@ func (m *RuleGroupDesc) GetUser() string { return "" } -type Rule struct { +type RuleDesc struct { Expr string `protobuf:"bytes,1,opt,name=expr,proto3" json:"expr,omitempty"` Record string `protobuf:"bytes,2,opt,name=record,proto3" json:"record,omitempty"` Alert string `protobuf:"bytes,3,opt,name=alert,proto3" json:"alert,omitempty"` @@ -173,17 +173,17 @@ type Rule struct { Annotations []github_com_cortexproject_cortex_pkg_ingester_client.LabelAdapter `protobuf:"bytes,6,rep,name=annotations,proto3,customtype=github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter" json:"annotations"` } -func (m *Rule) Reset() { *m = Rule{} } -func (*Rule) ProtoMessage() {} -func (*Rule) Descriptor() ([]byte, []int) { - return fileDescriptor_98bbca36ef968dfc, []int{2} +func (m *RuleDesc) Reset() { *m = RuleDesc{} } +func (*RuleDesc) ProtoMessage() {} +func (*RuleDesc) Descriptor() ([]byte, []int) { + return fileDescriptor_8e722d3e922f0937, []int{2} } -func (m *Rule) XXX_Unmarshal(b []byte) error { +func (m *RuleDesc) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *Rule) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *RuleDesc) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_Rule.Marshal(b, m, deterministic) + return xxx_messageInfo_RuleDesc.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalTo(b) @@ -193,40 +193,40 @@ func (m *Rule) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return b[:n], nil } } -func (m *Rule) XXX_Merge(src proto.Message) { - xxx_messageInfo_Rule.Merge(m, src) +func (m *RuleDesc) XXX_Merge(src proto.Message) { + xxx_messageInfo_RuleDesc.Merge(m, src) } -func (m *Rule) XXX_Size() int { +func (m *RuleDesc) XXX_Size() int { return m.Size() } -func (m *Rule) XXX_DiscardUnknown() { - xxx_messageInfo_Rule.DiscardUnknown(m) +func (m *RuleDesc) XXX_DiscardUnknown() { + xxx_messageInfo_RuleDesc.DiscardUnknown(m) } -var xxx_messageInfo_Rule proto.InternalMessageInfo +var xxx_messageInfo_RuleDesc proto.InternalMessageInfo -func (m *Rule) GetExpr() string { +func (m *RuleDesc) GetExpr() string { if m != nil { return m.Expr } return "" } -func (m *Rule) GetRecord() string { +func (m *RuleDesc) GetRecord() string { if m != nil { return m.Record } return "" } -func (m *Rule) GetAlert() string { +func (m *RuleDesc) GetAlert() string { if m != nil { return m.Alert } return "" } -func (m *Rule) GetFor() *time.Duration { +func (m *RuleDesc) GetFor() *time.Duration { if m != nil { return m.For } @@ -234,45 +234,45 @@ func (m *Rule) GetFor() *time.Duration { } func init() { - proto.RegisterType((*RuleUpdateDesc)(nil), "store.RuleUpdateDesc") - proto.RegisterType((*RuleGroupDesc)(nil), "store.RuleGroupDesc") - proto.RegisterType((*Rule)(nil), "store.Rule") + proto.RegisterType((*RuleUpdateDesc)(nil), "rules.RuleUpdateDesc") + proto.RegisterType((*RuleGroupDesc)(nil), "rules.RuleGroupDesc") + proto.RegisterType((*RuleDesc)(nil), "rules.RuleDesc") } -func init() { proto.RegisterFile("store.proto", fileDescriptor_98bbca36ef968dfc) } +func init() { proto.RegisterFile("rules.proto", fileDescriptor_8e722d3e922f0937) } -var fileDescriptor_98bbca36ef968dfc = []byte{ - // 474 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x53, 0xb1, 0x6e, 0xd4, 0x40, - 0x10, 0xf5, 0xe6, 0x6c, 0x93, 0x5b, 0x0b, 0x24, 0x56, 0x08, 0x99, 0x08, 0xed, 0x99, 0xab, 0xdc, - 0x60, 0x8b, 0x50, 0xa6, 0x21, 0xa7, 0x48, 0x50, 0x50, 0x20, 0x4b, 0x34, 0x74, 0x7b, 0xf6, 0xc4, - 0x18, 0x36, 0x5e, 0x6b, 0x77, 0x8d, 0x68, 0x90, 0xf8, 0x04, 0x4a, 0x3e, 0x81, 0x4f, 0x49, 0xc7, - 0x95, 0x11, 0x45, 0xe0, 0x7c, 0x0d, 0x1d, 0xf9, 0x04, 0xb4, 0x6b, 0x3b, 0x77, 0x25, 0x42, 0x4a, - 0xe5, 0x79, 0xf3, 0x66, 0xde, 0xcc, 0x1b, 0xdb, 0x38, 0x50, 0x5a, 0x48, 0x48, 0x1a, 0x29, 0xb4, - 0x20, 0x9e, 0x05, 0x07, 0x8f, 0xcb, 0x4a, 0xbf, 0x6d, 0x97, 0x49, 0x2e, 0xce, 0xd2, 0x52, 0x94, - 0x22, 0xb5, 0xec, 0xb2, 0x3d, 0xb5, 0xc8, 0x02, 0x1b, 0xf5, 0x5d, 0x07, 0xb4, 0x14, 0xa2, 0xe4, - 0xb0, 0xad, 0x2a, 0x5a, 0xc9, 0x74, 0x25, 0xea, 0x81, 0x7f, 0xb6, 0x23, 0x97, 0x0b, 0xa9, 0xe1, - 0x63, 0x23, 0xc5, 0x3b, 0xc8, 0xf5, 0x80, 0xd2, 0xe6, 0x7d, 0x99, 0x56, 0x75, 0x09, 0x4a, 0x83, - 0x4c, 0x73, 0x5e, 0x41, 0x3d, 0x52, 0xbd, 0xc2, 0x7c, 0x81, 0xef, 0x64, 0x2d, 0x87, 0xd7, 0x4d, - 0xc1, 0x34, 0x9c, 0x80, 0xca, 0x09, 0xc1, 0x6e, 0xab, 0x40, 0x86, 0x28, 0x42, 0xf1, 0x34, 0xb3, - 0x31, 0x79, 0x88, 0xa7, 0xad, 0xad, 0x28, 0x8e, 0x75, 0xb8, 0x17, 0xa1, 0x78, 0x92, 0x6d, 0x13, - 0xf3, 0xef, 0x08, 0xdf, 0x36, 0x22, 0xcf, 0xa5, 0x68, 0x9b, 0x51, 0xa3, 0x66, 0x67, 0x30, 0x6a, - 0x98, 0xd8, 0x68, 0x98, 0xa7, 0x6a, 0x58, 0x0e, 0x56, 0x63, 0x9a, 0x6d, 0x13, 0xe4, 0x08, 0xef, - 0x57, 0xb5, 0x06, 0xf9, 0x81, 0xf1, 0x70, 0x12, 0xa1, 0x38, 0x38, 0x7c, 0x90, 0xf4, 0xe6, 0x93, - 0xd1, 0x7c, 0x72, 0x32, 0x98, 0x5f, 0xb8, 0x5f, 0x7f, 0xce, 0x50, 0x76, 0xdd, 0x40, 0x1e, 0x61, - 0x4f, 0xb6, 0x1c, 0x54, 0xe8, 0x46, 0x93, 0x38, 0x38, 0x0c, 0x92, 0xfe, 0xf2, 0x66, 0xa7, 0xac, - 0x67, 0x48, 0x88, 0x6f, 0x15, 0xc0, 0x41, 0x43, 0x11, 0x7a, 0x11, 0x8a, 0xf7, 0xb3, 0x11, 0x5e, - 0xfb, 0xf5, 0xb7, 0x7e, 0xe7, 0x7f, 0xf6, 0xb0, 0x6b, 0xba, 0x0d, 0x69, 0x6e, 0x3a, 0x1a, 0x31, - 0x31, 0xb9, 0x8f, 0x7d, 0x09, 0xb9, 0x90, 0xc5, 0xe0, 0x62, 0x40, 0xe4, 0x1e, 0xf6, 0x18, 0x07, - 0xa9, 0xed, 0xfe, 0xd3, 0xac, 0x07, 0xe4, 0x09, 0x9e, 0x9c, 0x0a, 0x19, 0xba, 0xff, 0xe6, 0xc9, - 0xd4, 0x12, 0x85, 0x7d, 0xce, 0x96, 0xc0, 0x55, 0xe8, 0x59, 0x3f, 0x77, 0x93, 0xe1, 0x95, 0xbd, - 0x34, 0xd9, 0x57, 0xac, 0x92, 0x8b, 0x17, 0xe7, 0x97, 0x33, 0xe7, 0xc7, 0xe5, 0xec, 0x7f, 0x3e, - 0x80, 0x5e, 0xe6, 0xb8, 0x60, 0x8d, 0x06, 0x99, 0x0d, 0xa3, 0xc8, 0x27, 0x1c, 0xb0, 0xba, 0x16, - 0xda, 0x6e, 0xa3, 0x42, 0xff, 0xe6, 0x27, 0xef, 0xce, 0x5b, 0x1c, 0xad, 0xd6, 0xd4, 0xb9, 0x58, - 0x53, 0xe7, 0x6a, 0x4d, 0xd1, 0xe7, 0x8e, 0xa2, 0x6f, 0x1d, 0x45, 0xe7, 0x1d, 0x45, 0xab, 0x8e, - 0xa2, 0x5f, 0x1d, 0x45, 0xbf, 0x3b, 0xea, 0x5c, 0x75, 0x14, 0x7d, 0xd9, 0x50, 0x67, 0xb5, 0xa1, - 0xce, 0xc5, 0x86, 0x3a, 0x6f, 0xfa, 0xbf, 0x6a, 0xe9, 0xdb, 0x73, 0x3e, 0xfd, 0x1b, 0x00, 0x00, - 0xff, 0xff, 0x86, 0x6d, 0x96, 0x1c, 0x72, 0x03, 0x00, 0x00, +var fileDescriptor_8e722d3e922f0937 = []byte{ + // 478 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x92, 0xbf, 0x8e, 0xd4, 0x30, + 0x10, 0xc6, 0xe3, 0xdb, 0x3f, 0xec, 0x7a, 0x05, 0x08, 0x0b, 0x21, 0x73, 0x42, 0xde, 0x68, 0x25, + 0xa4, 0x34, 0x24, 0xe2, 0x28, 0xaf, 0xe1, 0x56, 0x27, 0x41, 0x41, 0x81, 0x2c, 0xd1, 0xd0, 0x79, + 0x93, 0xb9, 0x10, 0xf0, 0xc5, 0x91, 0xe3, 0x20, 0x1a, 0x24, 0x1e, 0x81, 0x92, 0x47, 0xe0, 0x51, + 0xae, 0x5c, 0x89, 0xe6, 0x44, 0x71, 0xb0, 0xd9, 0x86, 0xf2, 0x24, 0x5e, 0x00, 0xd9, 0x4e, 0x6e, + 0xaf, 0x44, 0x48, 0x54, 0x99, 0xcf, 0x33, 0xfe, 0x66, 0x7e, 0xe3, 0xe0, 0x99, 0x6e, 0x24, 0xd4, + 0x71, 0xa5, 0x95, 0x51, 0x64, 0xe4, 0xc4, 0xfe, 0xa3, 0xbc, 0x30, 0x6f, 0x9a, 0x55, 0x9c, 0xaa, + 0xd3, 0x24, 0x57, 0xb9, 0x4a, 0x5c, 0x76, 0xd5, 0x9c, 0x38, 0xe5, 0x84, 0x8b, 0xfc, 0xad, 0x7d, + 0x96, 0x2b, 0x95, 0x4b, 0xd8, 0x55, 0x65, 0x8d, 0x16, 0xa6, 0x50, 0x65, 0x97, 0x7f, 0x7a, 0xcd, + 0x2e, 0x55, 0xda, 0xc0, 0x87, 0x4a, 0xab, 0xb7, 0x90, 0x9a, 0x4e, 0x25, 0xd5, 0xbb, 0x3c, 0x29, + 0xca, 0x1c, 0x6a, 0x03, 0x3a, 0x49, 0x65, 0x01, 0x65, 0x9f, 0xf2, 0x0e, 0x8b, 0x25, 0xbe, 0xc5, + 0x1b, 0x09, 0xaf, 0xaa, 0x4c, 0x18, 0x38, 0x86, 0x3a, 0x25, 0x04, 0x0f, 0x9b, 0x1a, 0x34, 0x45, + 0x21, 0x8a, 0xa6, 0xdc, 0xc5, 0xe4, 0x01, 0x9e, 0x36, 0xae, 0x22, 0x3b, 0x32, 0x74, 0x2f, 0x44, + 0xd1, 0x80, 0xef, 0x0e, 0x16, 0xdf, 0x10, 0xbe, 0x69, 0x4d, 0x9e, 0x69, 0xd5, 0x54, 0xbd, 0x47, + 0x29, 0x4e, 0xa1, 0xf7, 0xb0, 0xb1, 0xf5, 0xb0, 0xdf, 0xba, 0x12, 0x29, 0x38, 0x8f, 0x29, 0xdf, + 0x1d, 0x90, 0x43, 0x3c, 0x29, 0x4a, 0x03, 0xfa, 0xbd, 0x90, 0x74, 0x10, 0xa2, 0x68, 0x76, 0x70, + 0x3f, 0xf6, 0xf0, 0x71, 0x0f, 0x1f, 0x1f, 0x77, 0xf0, 0xcb, 0xe1, 0x97, 0x1f, 0x73, 0xc4, 0xaf, + 0x2e, 0x90, 0x87, 0xd8, 0xaf, 0x97, 0x0e, 0xc3, 0x41, 0x34, 0x3b, 0xb8, 0x1d, 0xfb, 0xcd, 0xdb, + 0x99, 0xec, 0x38, 0xdc, 0x67, 0x09, 0xc5, 0x37, 0x32, 0x90, 0x60, 0x20, 0xa3, 0xa3, 0x10, 0x45, + 0x13, 0xde, 0xcb, 0x2b, 0xe6, 0xf1, 0x8e, 0x79, 0xf1, 0x7b, 0x0f, 0x4f, 0x7a, 0x07, 0x5b, 0x60, + 0x77, 0xdb, 0x03, 0xd9, 0x98, 0xdc, 0xc3, 0x63, 0x0d, 0xa9, 0xd2, 0x59, 0x47, 0xd3, 0x29, 0x72, + 0x17, 0x8f, 0x84, 0x04, 0x6d, 0x1c, 0xc7, 0x94, 0x7b, 0x41, 0x1e, 0xe3, 0xc1, 0x89, 0xd2, 0x74, + 0xf8, 0x77, 0x6c, 0xb6, 0x96, 0xd4, 0x78, 0x2c, 0xc5, 0x0a, 0x64, 0x4d, 0x47, 0x8e, 0xeb, 0x4e, + 0xdc, 0x3d, 0xdd, 0x0b, 0x7b, 0xfa, 0x52, 0x14, 0x7a, 0xf9, 0xfc, 0xec, 0x62, 0x1e, 0x7c, 0xbf, + 0x98, 0xff, 0xcb, 0x8f, 0xe0, 0x6d, 0x8e, 0x32, 0x51, 0x19, 0xd0, 0xbc, 0x6b, 0x45, 0x3e, 0xe2, + 0x99, 0x28, 0x4b, 0x65, 0xdc, 0x34, 0x35, 0x1d, 0xff, 0xff, 0xce, 0xd7, 0xfb, 0x2d, 0x0f, 0xd7, + 0x1b, 0x16, 0x9c, 0x6f, 0x58, 0x70, 0xb9, 0x61, 0xe8, 0x53, 0xcb, 0xd0, 0xd7, 0x96, 0xa1, 0xb3, + 0x96, 0xa1, 0x75, 0xcb, 0xd0, 0xcf, 0x96, 0xa1, 0x5f, 0x2d, 0x0b, 0x2e, 0x5b, 0x86, 0x3e, 0x6f, + 0x59, 0xb0, 0xde, 0xb2, 0xe0, 0x7c, 0xcb, 0x82, 0xd7, 0xfe, 0x81, 0x57, 0x63, 0xb7, 0xce, 0x27, + 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x70, 0xb7, 0x00, 0xbd, 0x7a, 0x03, 0x00, 0x00, } func (this *RuleUpdateDesc) Equal(that interface{}) bool { @@ -352,14 +352,14 @@ func (this *RuleGroupDesc) Equal(that interface{}) bool { } return true } -func (this *Rule) Equal(that interface{}) bool { +func (this *RuleDesc) Equal(that interface{}) bool { if that == nil { return this == nil } - that1, ok := that.(*Rule) + that1, ok := that.(*RuleDesc) if !ok { - that2, ok := that.(Rule) + that2, ok := that.(RuleDesc) if ok { that1 = &that2 } else { @@ -412,7 +412,7 @@ func (this *RuleUpdateDesc) GoString() string { return "nil" } s := make([]string, 0, 6) - s = append(s, "&store.RuleUpdateDesc{") + s = append(s, "&rules.RuleUpdateDesc{") s = append(s, "User: "+fmt.Sprintf("%#v", this.User)+",\n") s = append(s, "UpdatedAt: "+fmt.Sprintf("%#v", this.UpdatedAt)+",\n") s = append(s, "}") @@ -423,7 +423,7 @@ func (this *RuleGroupDesc) GoString() string { return "nil" } s := make([]string, 0, 10) - s = append(s, "&store.RuleGroupDesc{") + s = append(s, "&rules.RuleGroupDesc{") s = append(s, "Name: "+fmt.Sprintf("%#v", this.Name)+",\n") s = append(s, "Namespace: "+fmt.Sprintf("%#v", this.Namespace)+",\n") s = append(s, "Interval: "+fmt.Sprintf("%#v", this.Interval)+",\n") @@ -435,12 +435,12 @@ func (this *RuleGroupDesc) GoString() string { s = append(s, "}") return strings.Join(s, "") } -func (this *Rule) GoString() string { +func (this *RuleDesc) GoString() string { if this == nil { return "nil" } s := make([]string, 0, 10) - s = append(s, "&store.Rule{") + s = append(s, "&rules.RuleDesc{") s = append(s, "Expr: "+fmt.Sprintf("%#v", this.Expr)+",\n") s = append(s, "Record: "+fmt.Sprintf("%#v", this.Record)+",\n") s = append(s, "Alert: "+fmt.Sprintf("%#v", this.Alert)+",\n") @@ -450,7 +450,7 @@ func (this *Rule) GoString() string { s = append(s, "}") return strings.Join(s, "") } -func valueToGoStringStore(v interface{}, typ string) string { +func valueToGoStringRules(v interface{}, typ string) string { rv := reflect.ValueOf(v) if rv.IsNil() { return "nil" @@ -476,13 +476,13 @@ func (m *RuleUpdateDesc) MarshalTo(dAtA []byte) (int, error) { if len(m.User) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintStore(dAtA, i, uint64(len(m.User))) + i = encodeVarintRules(dAtA, i, uint64(len(m.User))) i += copy(dAtA[i:], m.User) } if m.UpdatedAt != 0 { dAtA[i] = 0x10 i++ - i = encodeVarintStore(dAtA, i, uint64(m.UpdatedAt)) + i = encodeVarintRules(dAtA, i, uint64(m.UpdatedAt)) } return i, nil } @@ -505,19 +505,19 @@ func (m *RuleGroupDesc) MarshalTo(dAtA []byte) (int, error) { if len(m.Name) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintStore(dAtA, i, uint64(len(m.Name))) + i = encodeVarintRules(dAtA, i, uint64(len(m.Name))) i += copy(dAtA[i:], m.Name) } if len(m.Namespace) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintStore(dAtA, i, uint64(len(m.Namespace))) + i = encodeVarintRules(dAtA, i, uint64(len(m.Namespace))) i += copy(dAtA[i:], m.Namespace) } if m.Interval != nil { dAtA[i] = 0x1a i++ - i = encodeVarintStore(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdDuration(*m.Interval))) + i = encodeVarintRules(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdDuration(*m.Interval))) n1, err := github_com_gogo_protobuf_types.StdDurationMarshalTo(*m.Interval, dAtA[i:]) if err != nil { return 0, err @@ -528,7 +528,7 @@ func (m *RuleGroupDesc) MarshalTo(dAtA []byte) (int, error) { for _, msg := range m.Rules { dAtA[i] = 0x22 i++ - i = encodeVarintStore(dAtA, i, uint64(msg.Size())) + i = encodeVarintRules(dAtA, i, uint64(msg.Size())) n, err := msg.MarshalTo(dAtA[i:]) if err != nil { return 0, err @@ -549,13 +549,13 @@ func (m *RuleGroupDesc) MarshalTo(dAtA []byte) (int, error) { if len(m.User) > 0 { dAtA[i] = 0x32 i++ - i = encodeVarintStore(dAtA, i, uint64(len(m.User))) + i = encodeVarintRules(dAtA, i, uint64(len(m.User))) i += copy(dAtA[i:], m.User) } return i, nil } -func (m *Rule) Marshal() (dAtA []byte, err error) { +func (m *RuleDesc) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -565,7 +565,7 @@ func (m *Rule) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Rule) MarshalTo(dAtA []byte) (int, error) { +func (m *RuleDesc) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -573,25 +573,25 @@ func (m *Rule) MarshalTo(dAtA []byte) (int, error) { if len(m.Expr) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintStore(dAtA, i, uint64(len(m.Expr))) + i = encodeVarintRules(dAtA, i, uint64(len(m.Expr))) i += copy(dAtA[i:], m.Expr) } if len(m.Record) > 0 { dAtA[i] = 0x12 i++ - i = encodeVarintStore(dAtA, i, uint64(len(m.Record))) + i = encodeVarintRules(dAtA, i, uint64(len(m.Record))) i += copy(dAtA[i:], m.Record) } if len(m.Alert) > 0 { dAtA[i] = 0x1a i++ - i = encodeVarintStore(dAtA, i, uint64(len(m.Alert))) + i = encodeVarintRules(dAtA, i, uint64(len(m.Alert))) i += copy(dAtA[i:], m.Alert) } if m.For != nil { dAtA[i] = 0x22 i++ - i = encodeVarintStore(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdDuration(*m.For))) + i = encodeVarintRules(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdDuration(*m.For))) n2, err := github_com_gogo_protobuf_types.StdDurationMarshalTo(*m.For, dAtA[i:]) if err != nil { return 0, err @@ -602,7 +602,7 @@ func (m *Rule) MarshalTo(dAtA []byte) (int, error) { for _, msg := range m.Labels { dAtA[i] = 0x2a i++ - i = encodeVarintStore(dAtA, i, uint64(msg.Size())) + i = encodeVarintRules(dAtA, i, uint64(msg.Size())) n, err := msg.MarshalTo(dAtA[i:]) if err != nil { return 0, err @@ -614,7 +614,7 @@ func (m *Rule) MarshalTo(dAtA []byte) (int, error) { for _, msg := range m.Annotations { dAtA[i] = 0x32 i++ - i = encodeVarintStore(dAtA, i, uint64(msg.Size())) + i = encodeVarintRules(dAtA, i, uint64(msg.Size())) n, err := msg.MarshalTo(dAtA[i:]) if err != nil { return 0, err @@ -625,7 +625,7 @@ func (m *Rule) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeVarintStore(dAtA []byte, offset int, v uint64) int { +func encodeVarintRules(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 @@ -642,10 +642,10 @@ func (m *RuleUpdateDesc) Size() (n int) { _ = l l = len(m.User) if l > 0 { - n += 1 + l + sovStore(uint64(l)) + n += 1 + l + sovRules(uint64(l)) } if m.UpdatedAt != 0 { - n += 1 + sovStore(uint64(m.UpdatedAt)) + n += 1 + sovRules(uint64(m.UpdatedAt)) } return n } @@ -658,20 +658,20 @@ func (m *RuleGroupDesc) Size() (n int) { _ = l l = len(m.Name) if l > 0 { - n += 1 + l + sovStore(uint64(l)) + n += 1 + l + sovRules(uint64(l)) } l = len(m.Namespace) if l > 0 { - n += 1 + l + sovStore(uint64(l)) + n += 1 + l + sovRules(uint64(l)) } if m.Interval != nil { l = github_com_gogo_protobuf_types.SizeOfStdDuration(*m.Interval) - n += 1 + l + sovStore(uint64(l)) + n += 1 + l + sovRules(uint64(l)) } if len(m.Rules) > 0 { for _, e := range m.Rules { l = e.Size() - n += 1 + l + sovStore(uint64(l)) + n += 1 + l + sovRules(uint64(l)) } } if m.Deleted { @@ -679,12 +679,12 @@ func (m *RuleGroupDesc) Size() (n int) { } l = len(m.User) if l > 0 { - n += 1 + l + sovStore(uint64(l)) + n += 1 + l + sovRules(uint64(l)) } return n } -func (m *Rule) Size() (n int) { +func (m *RuleDesc) Size() (n int) { if m == nil { return 0 } @@ -692,36 +692,36 @@ func (m *Rule) Size() (n int) { _ = l l = len(m.Expr) if l > 0 { - n += 1 + l + sovStore(uint64(l)) + n += 1 + l + sovRules(uint64(l)) } l = len(m.Record) if l > 0 { - n += 1 + l + sovStore(uint64(l)) + n += 1 + l + sovRules(uint64(l)) } l = len(m.Alert) if l > 0 { - n += 1 + l + sovStore(uint64(l)) + n += 1 + l + sovRules(uint64(l)) } if m.For != nil { l = github_com_gogo_protobuf_types.SizeOfStdDuration(*m.For) - n += 1 + l + sovStore(uint64(l)) + n += 1 + l + sovRules(uint64(l)) } if len(m.Labels) > 0 { for _, e := range m.Labels { l = e.Size() - n += 1 + l + sovStore(uint64(l)) + n += 1 + l + sovRules(uint64(l)) } } if len(m.Annotations) > 0 { for _, e := range m.Annotations { l = e.Size() - n += 1 + l + sovStore(uint64(l)) + n += 1 + l + sovRules(uint64(l)) } } return n } -func sovStore(x uint64) (n int) { +func sovRules(x uint64) (n int) { for { n++ x >>= 7 @@ -731,8 +731,8 @@ func sovStore(x uint64) (n int) { } return n } -func sozStore(x uint64) (n int) { - return sovStore(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +func sozRules(x uint64) (n int) { + return sovRules(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } func (this *RuleUpdateDesc) String() string { if this == nil { @@ -753,18 +753,18 @@ func (this *RuleGroupDesc) String() string { `Name:` + fmt.Sprintf("%v", this.Name) + `,`, `Namespace:` + fmt.Sprintf("%v", this.Namespace) + `,`, `Interval:` + strings.Replace(fmt.Sprintf("%v", this.Interval), "Duration", "duration.Duration", 1) + `,`, - `Rules:` + strings.Replace(fmt.Sprintf("%v", this.Rules), "Rule", "Rule", 1) + `,`, + `Rules:` + strings.Replace(fmt.Sprintf("%v", this.Rules), "RuleDesc", "RuleDesc", 1) + `,`, `Deleted:` + fmt.Sprintf("%v", this.Deleted) + `,`, `User:` + fmt.Sprintf("%v", this.User) + `,`, `}`, }, "") return s } -func (this *Rule) String() string { +func (this *RuleDesc) String() string { if this == nil { return "nil" } - s := strings.Join([]string{`&Rule{`, + s := strings.Join([]string{`&RuleDesc{`, `Expr:` + fmt.Sprintf("%v", this.Expr) + `,`, `Record:` + fmt.Sprintf("%v", this.Record) + `,`, `Alert:` + fmt.Sprintf("%v", this.Alert) + `,`, @@ -775,7 +775,7 @@ func (this *Rule) String() string { }, "") return s } -func valueToStringStore(v interface{}) string { +func valueToStringRules(v interface{}) string { rv := reflect.ValueOf(v) if rv.IsNil() { return "nil" @@ -791,7 +791,7 @@ func (m *RuleUpdateDesc) Unmarshal(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -819,7 +819,7 @@ func (m *RuleUpdateDesc) Unmarshal(dAtA []byte) error { var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -833,11 +833,11 @@ func (m *RuleUpdateDesc) Unmarshal(dAtA []byte) error { } intStringLen := int(stringLen) if intStringLen < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } postIndex := iNdEx + intStringLen if postIndex < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if postIndex > l { return io.ErrUnexpectedEOF @@ -851,7 +851,7 @@ func (m *RuleUpdateDesc) Unmarshal(dAtA []byte) error { m.UpdatedAt = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -865,15 +865,15 @@ func (m *RuleUpdateDesc) Unmarshal(dAtA []byte) error { } default: iNdEx = preIndex - skippy, err := skipStore(dAtA[iNdEx:]) + skippy, err := skipRules(dAtA[iNdEx:]) if err != nil { return err } if skippy < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if (iNdEx + skippy) < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -895,7 +895,7 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -923,7 +923,7 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -937,11 +937,11 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { } intStringLen := int(stringLen) if intStringLen < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } postIndex := iNdEx + intStringLen if postIndex < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if postIndex > l { return io.ErrUnexpectedEOF @@ -955,7 +955,7 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -969,11 +969,11 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { } intStringLen := int(stringLen) if intStringLen < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } postIndex := iNdEx + intStringLen if postIndex < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if postIndex > l { return io.ErrUnexpectedEOF @@ -987,7 +987,7 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1000,11 +1000,11 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { } } if msglen < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } postIndex := iNdEx + msglen if postIndex < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if postIndex > l { return io.ErrUnexpectedEOF @@ -1023,7 +1023,7 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1036,16 +1036,16 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { } } if msglen < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } postIndex := iNdEx + msglen if postIndex < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if postIndex > l { return io.ErrUnexpectedEOF } - m.Rules = append(m.Rules, &Rule{}) + m.Rules = append(m.Rules, &RuleDesc{}) if err := m.Rules[len(m.Rules)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } @@ -1057,7 +1057,7 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1077,7 +1077,7 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1091,11 +1091,11 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { } intStringLen := int(stringLen) if intStringLen < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } postIndex := iNdEx + intStringLen if postIndex < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if postIndex > l { return io.ErrUnexpectedEOF @@ -1104,15 +1104,15 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skipStore(dAtA[iNdEx:]) + skippy, err := skipRules(dAtA[iNdEx:]) if err != nil { return err } if skippy < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if (iNdEx + skippy) < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -1126,7 +1126,7 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { } return nil } -func (m *Rule) Unmarshal(dAtA []byte) error { +func (m *RuleDesc) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1134,7 +1134,7 @@ func (m *Rule) Unmarshal(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1149,10 +1149,10 @@ func (m *Rule) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Rule: wiretype end group for non-group") + return fmt.Errorf("proto: RuleDesc: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Rule: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: RuleDesc: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -1162,7 +1162,7 @@ func (m *Rule) Unmarshal(dAtA []byte) error { var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1176,11 +1176,11 @@ func (m *Rule) Unmarshal(dAtA []byte) error { } intStringLen := int(stringLen) if intStringLen < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } postIndex := iNdEx + intStringLen if postIndex < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if postIndex > l { return io.ErrUnexpectedEOF @@ -1194,7 +1194,7 @@ func (m *Rule) Unmarshal(dAtA []byte) error { var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1208,11 +1208,11 @@ func (m *Rule) Unmarshal(dAtA []byte) error { } intStringLen := int(stringLen) if intStringLen < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } postIndex := iNdEx + intStringLen if postIndex < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if postIndex > l { return io.ErrUnexpectedEOF @@ -1226,7 +1226,7 @@ func (m *Rule) Unmarshal(dAtA []byte) error { var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1240,11 +1240,11 @@ func (m *Rule) Unmarshal(dAtA []byte) error { } intStringLen := int(stringLen) if intStringLen < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } postIndex := iNdEx + intStringLen if postIndex < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if postIndex > l { return io.ErrUnexpectedEOF @@ -1258,7 +1258,7 @@ func (m *Rule) Unmarshal(dAtA []byte) error { var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1271,11 +1271,11 @@ func (m *Rule) Unmarshal(dAtA []byte) error { } } if msglen < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } postIndex := iNdEx + msglen if postIndex < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if postIndex > l { return io.ErrUnexpectedEOF @@ -1294,7 +1294,7 @@ func (m *Rule) Unmarshal(dAtA []byte) error { var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1307,11 +1307,11 @@ func (m *Rule) Unmarshal(dAtA []byte) error { } } if msglen < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } postIndex := iNdEx + msglen if postIndex < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if postIndex > l { return io.ErrUnexpectedEOF @@ -1328,7 +1328,7 @@ func (m *Rule) Unmarshal(dAtA []byte) error { var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowStore + return ErrIntOverflowRules } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -1341,11 +1341,11 @@ func (m *Rule) Unmarshal(dAtA []byte) error { } } if msglen < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } postIndex := iNdEx + msglen if postIndex < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if postIndex > l { return io.ErrUnexpectedEOF @@ -1357,15 +1357,15 @@ func (m *Rule) Unmarshal(dAtA []byte) error { iNdEx = postIndex default: iNdEx = preIndex - skippy, err := skipStore(dAtA[iNdEx:]) + skippy, err := skipRules(dAtA[iNdEx:]) if err != nil { return err } if skippy < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if (iNdEx + skippy) < 0 { - return ErrInvalidLengthStore + return ErrInvalidLengthRules } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -1379,14 +1379,14 @@ func (m *Rule) Unmarshal(dAtA []byte) error { } return nil } -func skipStore(dAtA []byte) (n int, err error) { +func skipRules(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return 0, ErrIntOverflowStore + return 0, ErrIntOverflowRules } if iNdEx >= l { return 0, io.ErrUnexpectedEOF @@ -1403,7 +1403,7 @@ func skipStore(dAtA []byte) (n int, err error) { case 0: for shift := uint(0); ; shift += 7 { if shift >= 64 { - return 0, ErrIntOverflowStore + return 0, ErrIntOverflowRules } if iNdEx >= l { return 0, io.ErrUnexpectedEOF @@ -1421,7 +1421,7 @@ func skipStore(dAtA []byte) (n int, err error) { var length int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return 0, ErrIntOverflowStore + return 0, ErrIntOverflowRules } if iNdEx >= l { return 0, io.ErrUnexpectedEOF @@ -1434,11 +1434,11 @@ func skipStore(dAtA []byte) (n int, err error) { } } if length < 0 { - return 0, ErrInvalidLengthStore + return 0, ErrInvalidLengthRules } iNdEx += length if iNdEx < 0 { - return 0, ErrInvalidLengthStore + return 0, ErrInvalidLengthRules } return iNdEx, nil case 3: @@ -1447,7 +1447,7 @@ func skipStore(dAtA []byte) (n int, err error) { var start int = iNdEx for shift := uint(0); ; shift += 7 { if shift >= 64 { - return 0, ErrIntOverflowStore + return 0, ErrIntOverflowRules } if iNdEx >= l { return 0, io.ErrUnexpectedEOF @@ -1463,13 +1463,13 @@ func skipStore(dAtA []byte) (n int, err error) { if innerWireType == 4 { break } - next, err := skipStore(dAtA[start:]) + next, err := skipRules(dAtA[start:]) if err != nil { return 0, err } iNdEx = start + next if iNdEx < 0 { - return 0, ErrInvalidLengthStore + return 0, ErrInvalidLengthRules } } return iNdEx, nil @@ -1486,6 +1486,6 @@ func skipStore(dAtA []byte) (n int, err error) { } var ( - ErrInvalidLengthStore = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowStore = fmt.Errorf("proto: integer overflow") + ErrInvalidLengthRules = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowRules = fmt.Errorf("proto: integer overflow") ) diff --git a/pkg/ruler/store/store.proto b/pkg/storage/rules/rules.proto similarity index 91% rename from pkg/ruler/store/store.proto rename to pkg/storage/rules/rules.proto index 69faeb9e9ef..7f13695d659 100644 --- a/pkg/ruler/store/store.proto +++ b/pkg/storage/rules/rules.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package store; +package rules; -option go_package = "store"; +option go_package = "rules"; import "github.com/gogo/protobuf/gogoproto/gogo.proto"; import "google/protobuf/duration.proto"; @@ -21,14 +21,14 @@ message RuleGroupDesc { string namespace = 2; google.protobuf.Duration interval = 3 [(gogoproto.stdduration) = true]; - repeated Rule rules = 4; + repeated RuleDesc rules = 4; bool deleted = 5; string user = 6; } - message Rule { + message RuleDesc { string expr = 1; string record = 2; string alert = 3; diff --git a/pkg/ruler/store/store.go b/pkg/storage/rules/store.go similarity index 99% rename from pkg/ruler/store/store.go rename to pkg/storage/rules/store.go index 5dfc2d0ada1..90723d773ec 100644 --- a/pkg/ruler/store/store.go +++ b/pkg/storage/rules/store.go @@ -1,4 +1,4 @@ -package store +package rules import ( "context" diff --git a/pkg/storage/testutils/testutils.go b/pkg/storage/testutils/testutils.go index b2d9ef1b505..f74b59705e0 100644 --- a/pkg/storage/testutils/testutils.go +++ b/pkg/storage/testutils/testutils.go @@ -1,13 +1,13 @@ package testutils import ( - alertStore "github.com/cortexproject/cortex/pkg/alertmanager/storage" - "github.com/cortexproject/cortex/pkg/ruler/store" + "github.com/cortexproject/cortex/pkg/storage/alerts" + "github.com/cortexproject/cortex/pkg/storage/rules" ) // Fixture type for per-backend testing. type Fixture interface { Name() string - Clients() (alertStore.AlertStore, store.RuleStore, error) + Clients() (alerts.AlertStore, rules.RuleStore, error) Teardown() error } From 4e0c50cfa5c634285bc2e8d0933fa8bd84594899 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Tue, 23 Jul 2019 16:22:29 -0400 Subject: [PATCH 58/60] ensure alertmanager config flags are registered Signed-off-by: Jacob Lisi --- pkg/alertmanager/multitenant.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/alertmanager/multitenant.go b/pkg/alertmanager/multitenant.go index b10ade1c393..5c4e251a02c 100644 --- a/pkg/alertmanager/multitenant.go +++ b/pkg/alertmanager/multitenant.go @@ -190,6 +190,7 @@ type MultitenantAlertmanagerConfig struct { // RegisterFlags adds the flags required to config this to the given FlagSet. func (cfg *MultitenantAlertmanagerConfig) RegisterFlags(f *flag.FlagSet) { + cfg.AlertStore.RegisterFlags(f) flag.StringVar(&cfg.DataDir, "alertmanager.storage.path", "data/", "Base path for data storage.") flag.DurationVar(&cfg.Retention, "alertmanager.storage.retention", 5*24*time.Hour, "How long to keep data for.") From 559771fd339c87ba3be788568557b1760422d055 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Tue, 23 Jul 2019 16:38:51 -0400 Subject: [PATCH 59/60] fix rebase error Signed-off-by: Jacob Lisi --- pkg/chunk/gcp/gcs_object_client.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/chunk/gcp/gcs_object_client.go b/pkg/chunk/gcp/gcs_object_client.go index 5e93fa8393c..62ebe70db83 100644 --- a/pkg/chunk/gcp/gcs_object_client.go +++ b/pkg/chunk/gcp/gcs_object_client.go @@ -36,11 +36,7 @@ func (cfg *GCSConfig) RegisterFlags(f *flag.FlagSet) { // NewGCSObjectClient makes a new chunk.ObjectClient that writes chunks to GCS. func NewGCSObjectClient(ctx context.Context, cfg GCSConfig, schemaCfg chunk.SchemaConfig) (chunk.ObjectClient, error) { -<<<<<<< HEAD - option, err := gcsInstrumentation(ctx, storage.ScopeReadWrite) -======= - option, err := gcsInstrumentation(ctx, "chunk") ->>>>>>> c48b40de... basic version of gcs backed ruler + option, err := gcsInstrumentation(ctx, "chunk", storage.ScopeReadWrite) if err != nil { return nil, err } From 0fff90b8b7d0c2fb098244aad17ad76ead1ec670 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Tue, 23 Jul 2019 17:08:46 -0400 Subject: [PATCH 60/60] add lateness to work item debug log line Signed-off-by: Jacob Lisi --- pkg/ruler/scheduler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go index fa89711924d..8466a8d8e1f 100644 --- a/pkg/ruler/scheduler.go +++ b/pkg/ruler/scheduler.go @@ -279,7 +279,7 @@ func (s *scheduler) workItemDone(i workItem) { // caused by the overall workload, not the result of latency within a single rule group. missed := (time.Since(i.scheduled) / s.evaluationInterval) - 1 if missed > 0 { - level.Warn(util.Logger).Log("msg", "scheduler: work item missed evaluation", "item", i) + level.Warn(util.Logger).Log("msg", "scheduler: work item missed evaluation", "item", i, "late_by", missed.String) iterationsMissed.WithLabelValues(i.userID).Add(float64(missed)) }