From c951b43847e37fef8a316b21b00a03e1b3b2e35f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Mon, 14 Oct 2019 10:21:53 +0200 Subject: [PATCH 01/31] Added new KV Store client, MultiClient. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This client is configured with multiple stores, one of them is designated as primary. All client operations are forwarded to the primary store. MultiClient also does "mirroring" of values from primary to secondary store. MultiClient can listen on changes in runtime configuration (via overrides mechanism), and switch primary store and enable/disable mirroring. Signed-off-by: Peter Štibraný --- go.mod | 1 + go.sum | 2 + pkg/chunk/chunk_store_test.go | 2 +- pkg/chunk/storage/caching_fixtures.go | 2 +- pkg/chunk/storage/factory_test.go | 2 +- pkg/cortex/cortex.go | 27 +- pkg/cortex/modules.go | 41 +- pkg/cortex/runtime_config.go | 69 ++++ pkg/distributor/distributor_test.go | 2 +- pkg/ingester/ingester_test.go | 2 +- pkg/ingester/lifecycle_test.go | 6 +- pkg/ingester/limiter_test.go | 8 +- pkg/querier/frontend/frontend_test.go | 2 +- pkg/ring/kv/client.go | 66 +++- pkg/ring/kv/client_test.go | 31 ++ pkg/ring/kv/multi.go | 337 +++++++++++++++++ pkg/util/{validation => }/override.go | 73 ++-- pkg/util/{validation => }/override_test.go | 107 +++--- pkg/util/validation/limits.go | 122 +++--- pkg/util/validation/limits_test.go | 33 ++ vendor/github.com/uber-go/atomic/.codecov.yml | 15 + vendor/github.com/uber-go/atomic/.gitignore | 11 + vendor/github.com/uber-go/atomic/.travis.yml | 27 ++ vendor/github.com/uber-go/atomic/LICENSE.txt | 19 + vendor/github.com/uber-go/atomic/Makefile | 51 +++ vendor/github.com/uber-go/atomic/README.md | 36 ++ vendor/github.com/uber-go/atomic/atomic.go | 351 ++++++++++++++++++ vendor/github.com/uber-go/atomic/error.go | 55 +++ vendor/github.com/uber-go/atomic/glide.lock | 17 + vendor/github.com/uber-go/atomic/glide.yaml | 6 + vendor/github.com/uber-go/atomic/string.go | 49 +++ vendor/modules.txt | 2 + 32 files changed, 1382 insertions(+), 192 deletions(-) create mode 100644 pkg/cortex/runtime_config.go create mode 100644 pkg/ring/kv/client_test.go create mode 100644 pkg/ring/kv/multi.go rename pkg/util/{validation => }/override.go (55%) rename pkg/util/{validation => }/override_test.go (55%) create mode 100644 vendor/github.com/uber-go/atomic/.codecov.yml create mode 100644 vendor/github.com/uber-go/atomic/.gitignore create mode 100644 vendor/github.com/uber-go/atomic/.travis.yml create mode 100644 vendor/github.com/uber-go/atomic/LICENSE.txt create mode 100644 vendor/github.com/uber-go/atomic/Makefile create mode 100644 vendor/github.com/uber-go/atomic/README.md create mode 100644 vendor/github.com/uber-go/atomic/atomic.go create mode 100644 vendor/github.com/uber-go/atomic/error.go create mode 100644 vendor/github.com/uber-go/atomic/glide.lock create mode 100644 vendor/github.com/uber-go/atomic/glide.yaml create mode 100644 vendor/github.com/uber-go/atomic/string.go diff --git a/go.mod b/go.mod index 5359f73f094..ee8afc3bd45 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/thanos-io/thanos v0.8.1-0.20200102143048-a37ac093a67a github.com/tinylib/msgp v0.0.0-20161221055906-38a6f61a768d // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect + github.com/uber-go/atomic v1.4.0 github.com/uber/jaeger-client-go v2.20.1+incompatible github.com/weaveworks/billing-client v0.0.0-20171006123215-be0d55e547b1 github.com/weaveworks/common v0.0.0-20190822150010-afb9996716e4 diff --git a/go.sum b/go.sum index f9d70c269dd..c834e13df9d 100644 --- a/go.sum +++ b/go.sum @@ -750,6 +750,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= +github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.20.1+incompatible h1:HgqpYBng0n7tLJIlyT4kPCIv5XgCsF+kai1NnnrJzEU= github.com/uber/jaeger-client-go v2.20.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= diff --git a/pkg/chunk/chunk_store_test.go b/pkg/chunk/chunk_store_test.go index 80632775a26..14f4fae8d83 100644 --- a/pkg/chunk/chunk_store_test.go +++ b/pkg/chunk/chunk_store_test.go @@ -88,7 +88,7 @@ func newTestChunkStoreConfig(t require.TestingT, schemaName string, storeCfg Sto var limits validation.Limits flagext.DefaultValues(&limits) limits.MaxQueryLength = 30 * 24 * time.Hour - overrides, err := validation.NewOverrides(limits) + overrides, err := validation.NewOverrides(limits, nil) require.NoError(t, err) store := NewCompositeStore() diff --git a/pkg/chunk/storage/caching_fixtures.go b/pkg/chunk/storage/caching_fixtures.go index 47e90fe44e5..ece80430986 100644 --- a/pkg/chunk/storage/caching_fixtures.go +++ b/pkg/chunk/storage/caching_fixtures.go @@ -41,5 +41,5 @@ func defaultLimits() (*validation.Overrides, error) { var defaults validation.Limits flagext.DefaultValues(&defaults) defaults.CardinalityLimit = 5 - return validation.NewOverrides(defaults) + return validation.NewOverrides(defaults, nil) } diff --git a/pkg/chunk/storage/factory_test.go b/pkg/chunk/storage/factory_test.go index 0b6737d94d0..2d680250cba 100644 --- a/pkg/chunk/storage/factory_test.go +++ b/pkg/chunk/storage/factory_test.go @@ -30,7 +30,7 @@ func TestFactoryStop(t *testing.T) { }, } - limits, err := validation.NewOverrides(defaults) + limits, err := validation.NewOverrides(defaults, nil) require.NoError(t, err) store, err := NewStore(cfg, storeConfig, schemaConfig, limits) diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index 1fe46fa5271..37a122837b3 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "os" + "time" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" @@ -79,6 +80,9 @@ type Config struct { ConfigDB db.Config `yaml:"configdb,omitempty"` ConfigStore config_client.Config `yaml:"config_store,omitempty"` Alertmanager alertmanager.MultitenantAlertmanagerConfig `yaml:"alertmanager,omitempty"` + + RuntimeConfigFile string `yaml:"runtime_config_file"` + RuntimeConfigLoadPeriod time.Duration `yaml:"runtime_config_load_period"` } // RegisterFlags registers flag. @@ -90,6 +94,8 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&c.AuthEnabled, "auth.enabled", true, "Set to false to disable auth.") f.BoolVar(&c.PrintConfig, "print.config", false, "Print the config and exit.") f.StringVar(&c.HTTPPrefix, "http.prefix", "/api/prom", "HTTP path prefix for Cortex API.") + f.StringVar(&c.RuntimeConfigFile, "runtime-config.file", "", "File with configuration that can be updated in runtime.") + f.DurationVar(&c.RuntimeConfigLoadPeriod, "runtime-config.reload-period", 10*time.Second, "How often to check runtime config file.") c.Server.RegisterFlags(f) c.Distributor.RegisterFlags(f) @@ -146,16 +152,17 @@ type Cortex struct { target moduleName httpAuthMiddleware middleware.Interface - server *server.Server - ring *ring.Ring - overrides *validation.Overrides - distributor *distributor.Distributor - ingester *ingester.Ingester - store chunk.Store - worker frontend.Worker - frontend *frontend.Frontend - tableManager *chunk.TableManager - cache cache.Cache + server *server.Server + ring *ring.Ring + overrides *validation.Overrides + distributor *distributor.Distributor + ingester *ingester.Ingester + store chunk.Store + worker frontend.Worker + frontend *frontend.Frontend + tableManager *chunk.TableManager + cache cache.Cache + runtimeConfig *util.OverridesManager ruler *ruler.Ruler configAPI *api.API diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 36288b4d373..bf7891f42c3 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -40,6 +40,7 @@ type moduleName int // The various modules that make up Cortex. const ( Ring moduleName = iota + RuntimeConfig Overrides Server Distributor @@ -58,6 +59,8 @@ func (m moduleName) String() string { switch m { case Ring: return "ring" + case RuntimeConfig: + return "runtime-config" case Overrides: return "overrides" case Server: @@ -152,6 +155,7 @@ func (t *Cortex) stopServer() (err error) { } func (t *Cortex) initRing(cfg *Config) (err error) { + cfg.Ingester.LifecyclerConfig.RingConfig.KVStore.Multi.ConfigProvider = multiClientRuntimeConfigChannel(t.runtimeConfig) t.ring, err = ring.New(cfg.Ingester.LifecyclerConfig.RingConfig, "ingester", ring.IngesterRingKey) if err != nil { return @@ -161,16 +165,33 @@ func (t *Cortex) initRing(cfg *Config) (err error) { return } -func (t *Cortex) initOverrides(cfg *Config) (err error) { - t.overrides, err = validation.NewOverrides(cfg.LimitsConfig) +func (t *Cortex) initRuntimeConfig(cfg *Config) (err error) { + configFile := cfg.RuntimeConfigFile + reloadPeriod := cfg.RuntimeConfigLoadPeriod + if configFile == "" { + configFile = cfg.LimitsConfig.PerTenantOverrideConfig + reloadPeriod = cfg.LimitsConfig.PerTenantOverridePeriod + } + + c := util.OverridesManagerConfig{ + OverridesReloadPeriod: reloadPeriod, + OverridesLoadPath: configFile, + OverridesLoader: loadRuntimeConfig, + } + t.runtimeConfig, err = util.NewOverridesManager(c) return err } -func (t *Cortex) stopOverrides() error { - t.overrides.Stop() +func (t *Cortex) stopRuntimeConfig() (err error) { + t.runtimeConfig.Stop() return nil } +func (t *Cortex) initOverrides(cfg *Config) (err error) { + t.overrides, err = validation.NewOverrides(cfg.LimitsConfig, tenantLimitsFromRuntimeConfig(t.runtimeConfig)) + return err +} + func (t *Cortex) initDistributor(cfg *Config) (err error) { cfg.Distributor.DistributorRing.ListenPort = cfg.Server.GRPCListenPort @@ -257,6 +278,7 @@ func (t *Cortex) stopQuerier() error { } func (t *Cortex) initIngester(cfg *Config) (err error) { + cfg.Ingester.LifecyclerConfig.RingConfig.KVStore.Multi.ConfigProvider = multiClientRuntimeConfigChannel(t.runtimeConfig) cfg.Ingester.LifecyclerConfig.ListenPort = &cfg.Server.GRPCListenPort cfg.Ingester.TSDBEnabled = cfg.Storage.Engine == storage.StorageEngineTSDB cfg.Ingester.TSDBConfig = cfg.TSDB @@ -446,14 +468,19 @@ var modules = map[moduleName]module{ stop: (*Cortex).stopServer, }, + RuntimeConfig: { + init: (*Cortex).initRuntimeConfig, + stop: (*Cortex).stopRuntimeConfig, + }, + Ring: { - deps: []moduleName{Server}, + deps: []moduleName{Server, RuntimeConfig}, init: (*Cortex).initRing, }, Overrides: { + deps: []moduleName{RuntimeConfig}, init: (*Cortex).initOverrides, - stop: (*Cortex).stopOverrides, }, Distributor: { @@ -469,7 +496,7 @@ var modules = map[moduleName]module{ }, Ingester: { - deps: []moduleName{Overrides, Store, Server}, + deps: []moduleName{Overrides, Store, Server, RuntimeConfig}, init: (*Cortex).initIngester, stop: (*Cortex).stopIngester, }, diff --git a/pkg/cortex/runtime_config.go b/pkg/cortex/runtime_config.go new file mode 100644 index 00000000000..12ede69351c --- /dev/null +++ b/pkg/cortex/runtime_config.go @@ -0,0 +1,69 @@ +package cortex + +import ( + "os" + + "gopkg.in/yaml.v2" + + "github.com/cortexproject/cortex/pkg/ring/kv" + "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/validation" +) + +// runtimeConfigValues are values that can be reloaded from configuration file while Cortex is running. +// Reloading is done by OverridesManager, which also keeps currently loaded config. +// These values are then pushed to the components that are interested in them. +type runtimeConfigValues struct { + TenantLimits map[string]*validation.Limits `yaml:"overrides"` + + Multi kv.MultiRuntimeConfig `yaml:"multi"` +} + +func loadRuntimeConfig(filename string) (interface{}, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + + var overrides = &runtimeConfigValues{} + + decoder := yaml.NewDecoder(f) + decoder.SetStrict(true) + if err := decoder.Decode(&overrides); err != nil { + return nil, err + } + + return overrides, nil +} + +func tenantLimitsFromRuntimeConfig(c *util.OverridesManager) validation.TenantLimits { + return func(userID string) *validation.Limits { + cfg, ok := c.GetOverrides().(*runtimeConfigValues) + if !ok || cfg == nil { + return nil + } + + return cfg.TenantLimits[userID] + } +} + +func multiClientRuntimeConfigChannel(manager *util.OverridesManager) func() <-chan kv.MultiRuntimeConfig { + // returns function that can be used in MultiConfig.ConfigProvider + return func() <-chan kv.MultiRuntimeConfig { + ch := make(chan kv.MultiRuntimeConfig, 1) + + listener := func(newOverrides interface{}) { + cfg, ok := newOverrides.(*runtimeConfigValues) + if !ok || cfg == nil { + return + } + + ch <- cfg.Multi + } + + // push initial config to the channel + listener(manager.GetOverrides()) + manager.AddListener(listener) + return ch + } +} diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index dc40c603817..0de0548c878 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -756,7 +756,7 @@ func prepare(t *testing.T, numIngesters, happyIngesters int, queryDelay time.Dur cfg.DistributorRing.KVStore.Mock = kvStore cfg.DistributorRing.InstanceAddr = "127.0.0.1" - overrides, err := validation.NewOverrides(*limits) + overrides, err := validation.NewOverrides(*limits, nil) require.NoError(t, err) d, err := New(cfg, clientConfig, overrides, ingestersRing, true) diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index f9574405bd8..93f8897aebb 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -41,7 +41,7 @@ func newTestStore(t require.TestingT, cfg Config, clientConfig client.Config, li store := &testStore{ chunks: map[string][]chunk.Chunk{}, } - overrides, err := validation.NewOverrides(limits) + overrides, err := validation.NewOverrides(limits, nil) require.NoError(t, err) ing, err := New(cfg, clientConfig, overrides, store, nil) diff --git a/pkg/ingester/lifecycle_test.go b/pkg/ingester/lifecycle_test.go index 7595d5d3450..1b0e97954b1 100644 --- a/pkg/ingester/lifecycle_test.go +++ b/pkg/ingester/lifecycle_test.go @@ -93,7 +93,7 @@ func TestIngesterRestart(t *testing.T) { } func TestIngesterTransfer(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig()) + limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) // Start the first ingester, and get it into ACTIVE state. @@ -158,7 +158,7 @@ func TestIngesterTransfer(t *testing.T) { } func TestIngesterBadTransfer(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig()) + limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) // Start ingester in PENDING. @@ -410,7 +410,7 @@ func TestV2IngesterTransfer(t *testing.T) { // We run the same under different scenarios for name, scenario := range scenarios { t.Run(name, func(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig()) + limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) require.NoError(t, err) dir1, err := ioutil.TempDir("", "tsdb") diff --git a/pkg/ingester/limiter_test.go b/pkg/ingester/limiter_test.go index 6400f3bdfc7..5af41e9b340 100644 --- a/pkg/ingester/limiter_test.go +++ b/pkg/ingester/limiter_test.go @@ -90,7 +90,7 @@ func TestSeriesLimit_maxSeriesPerMetric(t *testing.T) { limits, err := validation.NewOverrides(validation.Limits{ MaxLocalSeriesPerMetric: testData.maxLocalSeriesPerMetric, MaxGlobalSeriesPerMetric: testData.maxGlobalSeriesPerMetric, - }) + }, nil) require.NoError(t, err) limiter := NewSeriesLimiter(limits, ring, testData.ringReplicationFactor, testData.shardByAllLabels) @@ -180,7 +180,7 @@ func TestSeriesLimit_maxSeriesPerUser(t *testing.T) { limits, err := validation.NewOverrides(validation.Limits{ MaxLocalSeriesPerUser: testData.maxLocalSeriesPerUser, MaxGlobalSeriesPerUser: testData.maxGlobalSeriesPerUser, - }) + }, nil) require.NoError(t, err) limiter := NewSeriesLimiter(limits, ring, testData.ringReplicationFactor, testData.shardByAllLabels) @@ -242,7 +242,7 @@ func TestSeriesLimiter_AssertMaxSeriesPerMetric(t *testing.T) { limits, err := validation.NewOverrides(validation.Limits{ MaxLocalSeriesPerMetric: testData.maxLocalSeriesPerMetric, MaxGlobalSeriesPerMetric: testData.maxGlobalSeriesPerMetric, - }) + }, nil) require.NoError(t, err) limiter := NewSeriesLimiter(limits, ring, testData.ringReplicationFactor, testData.shardByAllLabels) @@ -304,7 +304,7 @@ func TestSeriesLimiter_AssertMaxSeriesPerUser(t *testing.T) { limits, err := validation.NewOverrides(validation.Limits{ MaxLocalSeriesPerUser: testData.maxLocalSeriesPerUser, MaxGlobalSeriesPerUser: testData.maxGlobalSeriesPerUser, - }) + }, nil) require.NoError(t, err) limiter := NewSeriesLimiter(limits, ring, testData.ringReplicationFactor, testData.shardByAllLabels) diff --git a/pkg/querier/frontend/frontend_test.go b/pkg/querier/frontend/frontend_test.go index cb18146d235..b1d1ff65448 100644 --- a/pkg/querier/frontend/frontend_test.go +++ b/pkg/querier/frontend/frontend_test.go @@ -136,7 +136,7 @@ func TestFrontendCancel(t *testing.T) { func defaultOverrides(t *testing.T) *validation.Overrides { var limits validation.Limits flagext.DefaultValues(&limits) - overrides, err := validation.NewOverrides(limits) + overrides, err := validation.NewOverrides(limits, nil) require.NoError(t, err) return overrides } diff --git a/pkg/ring/kv/client.go b/pkg/ring/kv/client.go index ba3317365cc..85fe4a26c77 100644 --- a/pkg/ring/kv/client.go +++ b/pkg/ring/kv/client.go @@ -20,14 +20,22 @@ import ( var inmemoryStoreInit sync.Once var inmemoryStore Client -// Config is config for a KVStore currently used by ring and HA tracker, -// where store can be consul or inmemory. -type Config struct { - Store string `yaml:"store,omitempty"` +// StoreConfig is a configuration used for building single store client, either +// Consul, Etcd, Memberlist or MultiClient. It was extracted from Config to keep +// single-client config separate from final client-config (with all the wrappers) +type StoreConfig struct { Consul consul.Config `yaml:"consul,omitempty"` Etcd etcd.Config `yaml:"etcd,omitempty"` Memberlist memberlist.Config `yaml:"memberlist,omitempty"` - Prefix string `yaml:"prefix,omitempty"` + Multi MultiConfig `yaml:"multi,omitempty"` +} + +// Config is config for a KVStore currently used by ring and HA tracker, +// where store can be consul or inmemory. +type Config struct { + Store string `yaml:"store,omitempty"` + Prefix string `yaml:"prefix,omitempty"` + StoreConfig `yaml:",inline"` Mock Client } @@ -44,12 +52,14 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { // be easier to have everything under ring, so ring.consul. cfg.Consul.RegisterFlags(f, prefix) cfg.Etcd.RegisterFlagsWithPrefix(f, prefix) + cfg.Multi.RegisterFlagsWithPrefix(f, prefix) cfg.Memberlist.RegisterFlags(f, prefix) + if prefix == "" { prefix = "ring." } f.StringVar(&cfg.Prefix, prefix+"prefix", "collectors/", "The prefix for the keys in the store. Should end with a /.") - f.StringVar(&cfg.Store, prefix+"store", "consul", "Backend storage to use for the ring (consul, etcd, inmemory, memberlist [experimental]).") + f.StringVar(&cfg.Store, prefix+"store", "consul", "Backend storage to use for the ring (consul, etcd, inmemory, multi, memberlist [experimental]).") } // Client is a high-level client for key-value stores (such as Etcd and @@ -86,10 +96,14 @@ func NewClient(cfg Config, codec codec.Codec) (Client, error) { return cfg.Mock, nil } + return createClient(cfg.Store, cfg.Prefix, cfg.StoreConfig, codec) +} + +func createClient(name string, prefix string, cfg StoreConfig, codec codec.Codec) (Client, error) { var client Client var err error - switch cfg.Store { + switch name { case "consul": client, err = consul.NewClient(cfg.Consul, codec) @@ -108,17 +122,49 @@ func NewClient(cfg Config, codec codec.Codec) (Client, error) { cfg.Memberlist.MetricsRegisterer = prometheus.DefaultRegisterer client, err = memberlist.NewMemberlistClient(cfg.Memberlist, codec) + case "multi": + client, err = buildMultiClient(cfg, codec) + default: - return nil, fmt.Errorf("invalid KV store type: %s", cfg.Store) + return nil, fmt.Errorf("invalid KV store type: %s", name) } if err != nil { return nil, err } - if cfg.Prefix != "" { - client = PrefixClient(client, cfg.Prefix) + if prefix != "" { + client = PrefixClient(client, prefix) } return metrics{client}, nil } + +func buildMultiClient(cfg StoreConfig, codec codec.Codec) (Client, error) { + if cfg.Multi.Primary == "" || cfg.Multi.Secondary == "" { + return nil, fmt.Errorf("primary or secondary store not set") + } + if cfg.Multi.Primary == "multi" || cfg.Multi.Secondary == "multi" { + return nil, fmt.Errorf("primary and secondary stores cannot be multi-stores") + } + if cfg.Multi.Primary == cfg.Multi.Secondary { + return nil, fmt.Errorf("primary and secondary stores must be different") + } + + primary, err := createClient(cfg.Multi.Primary, "", cfg, codec) + if err != nil { + return nil, err + } + + secondary, err := createClient(cfg.Multi.Secondary, "", cfg, codec) + if err != nil { + return nil, err + } + + clients := []kvclient{ + {client: primary, name: cfg.Multi.Primary}, + {client: secondary, name: cfg.Multi.Secondary}, + } + + return NewMultiClient(cfg.Multi, clients), nil +} diff --git a/pkg/ring/kv/client_test.go b/pkg/ring/kv/client_test.go new file mode 100644 index 00000000000..9800ea3c1c0 --- /dev/null +++ b/pkg/ring/kv/client_test.go @@ -0,0 +1,31 @@ +package kv + +import ( + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" +) + +func TestParseConfig(t *testing.T) { + conf := ` +store: consul +consul: + host: "consul:8500" + consistentreads: true +prefix: "test/" +multi: + primary: consul + secondary: etcd +` + + cfg := Config{} + + err := yaml.Unmarshal([]byte(conf), &cfg) + require.NoError(t, err) + require.Equal(t, "consul", cfg.Store) + require.Equal(t, "test/", cfg.Prefix) + require.Equal(t, "consul:8500", cfg.Consul.Host) + require.Equal(t, "consul", cfg.Multi.Primary) + require.Equal(t, "etcd", cfg.Multi.Secondary) +} diff --git a/pkg/ring/kv/multi.go b/pkg/ring/kv/multi.go new file mode 100644 index 00000000000..ad32017d0a5 --- /dev/null +++ b/pkg/ring/kv/multi.go @@ -0,0 +1,337 @@ +package kv + +import ( + "context" + "flag" + "fmt" + "sync" + + "github.com/go-kit/kit/log/level" + "github.com/uber-go/atomic" + "golang.org/x/time/rate" + + "github.com/cortexproject/cortex/pkg/util" +) + +// MultiConfig is a configuration for MultiClient. +type MultiConfig struct { + Primary string `yaml:"primary"` + Secondary string `yaml:"secondary"` + + MirrorRateLimit float64 `yaml:"mirror-rate-limit"` + MirrorRateBurst int `yaml:"mirror-rate-burst"` + MirrorEnabled bool `yaml:"mirror-enabled"` + MirrorPrefix string `yaml:"mirror-prefix"` + + // ConfigProvider returns channel with MultiRuntimeConfig updates. + ConfigProvider func() <-chan MultiRuntimeConfig +} + +// RegisterFlagsWithPrefix registers flags with prefix. +func (cfg *MultiConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { + f.StringVar(&cfg.Primary, prefix+"multi.primary", "", "Primary backend storage used by multi-client.") + f.StringVar(&cfg.Secondary, prefix+"multi.secondary", "", "Secondary backend storage used by multi-client.") + f.StringVar(&cfg.MirrorPrefix, prefix+"multi.mirror-prefix", "", "Prefix for keys that should be mirrored") + f.BoolVar(&cfg.MirrorEnabled, prefix+"multi.mirror-enabled", false, "Mirror values to secondary store") + f.Float64Var(&cfg.MirrorRateLimit, prefix+"multi.mirror-rate-limit", 1, "Rate limit used for mirroring. 0 disables rate limit.") + f.IntVar(&cfg.MirrorRateBurst, prefix+"multi.mirror-burst-size", 1, "Burst size used by rate limiter. Values less than 1 are treated as 1.") +} + +// MultiRuntimeConfig has values that can change in runtime (via overrides) +type MultiRuntimeConfig struct { + // Primary store used by MultiClient. Can be updated in runtime to switch to a different store (eg. consul -> etcd, + // or to gossip). Doing this allows nice migration between stores. Empty values are ignored. + PrimaryStore string `yaml:"primary"` + + // Put "enabled", "enable" or "true" here to enable. Any other value = disabled. Empty = ignore. + // We don't use bool here, because it would not be possible to distinguish between false = missing value, + // or false = disabled. + Mirroring string `yaml:"mirroring"` +} + +type kvclient struct { + client Client + name string +} + +type clientInProgress struct { + client int + cancel context.CancelFunc +} + +// MultiClient implements kv.Client by forwarding all API calls to primary client. +// At the same time, MultiClient watches for changes in values in the primary store, +// and forwards them to remaining clients. +type MultiClient struct { + // Available KV clients + clients []kvclient + + rateLimiter *rate.Limiter + mirroringEnabled *atomic.Bool + + // Primary client used for interaction. Values from this KV are copied to all the others. + primaryID *atomic.Int32 + + ctx context.Context + cancel context.CancelFunc + + inProgressMu sync.Mutex + // Cancel functions for ongoing operations. key is a value from inProgressCnt. + // What we really need is a []context.CancelFunc, but functions cannot be compared against each other using ==, + // so we use this map instead. + inProgress map[int]clientInProgress + inProgressCnt int +} + +// NewMultiClient creates new MultiClient with given KV Clients. +// First client in the slice is the primary client. +// Channel is used to get notifications about what store to use as primary. +func NewMultiClient(cfg MultiConfig, clients []kvclient) *MultiClient { + c := &MultiClient{ + clients: clients, + primaryID: atomic.NewInt32(0), + inProgress: map[int]clientInProgress{}, + + rateLimiter: createRateLimiter(cfg.MirrorRateLimit, cfg.MirrorRateBurst), + mirroringEnabled: atomic.NewBool(cfg.MirrorEnabled), + } + + c.ctx, c.cancel = context.WithCancel(context.Background()) + + // Start mirroring. + go c.watchChanges(cfg.MirrorPrefix) + + if cfg.ConfigProvider != nil { + go c.watchConfigChannel(cfg.ConfigProvider()) + } + + return c +} + +func (m *MultiClient) watchConfigChannel(configChannel <-chan MultiRuntimeConfig) { + for cfg := range configChannel { + if cfg.Mirroring != "" { + enable := cfg.Mirroring == "true" || cfg.Mirroring == "enable" || cfg.Mirroring == "enabled" + + old := m.mirroringEnabled.Swap(enable) + if old != enable { + level.Info(util.Logger).Log("msg", "toggled mirroring", "enabled", enable) + } + } + + if cfg.PrimaryStore != "" { + switched, err := m.setNewPrimaryClient(cfg.PrimaryStore) + if switched { + level.Info(util.Logger).Log("msg", "switched primary KV store", "primary", cfg.PrimaryStore) + } + if err != nil { + level.Error(util.Logger).Log("msg", "failed to switch primary KV store", "primary", cfg.PrimaryStore, "err", err) + } + } + } +} + +func (m *MultiClient) getPrimaryClient() (int, kvclient) { + v := m.primaryID.Load() + return int(v), m.clients[v] +} + +func (m *MultiClient) setNewPrimaryClient(store string) (bool, error) { + newPrimaryIx := -1 + for ix, c := range m.clients { + if c.name == store { + newPrimaryIx = ix + break + } + } + + if newPrimaryIx < 0 { + return false, fmt.Errorf("KV store not found") + } + + prev := int(m.primaryID.Swap(int32(newPrimaryIx))) + if prev == newPrimaryIx { + return false, nil + } + + // switching to new primary... cancel clients using previous one + m.inProgressMu.Lock() + defer m.inProgressMu.Unlock() + + for _, inp := range m.inProgress { + if inp.client == prev { + inp.cancel() + } + } + return true, nil +} + +func (m *MultiClient) registerCancelFn(clientID int, fn context.CancelFunc) int { + m.inProgressMu.Lock() + defer m.inProgressMu.Unlock() + + m.inProgressCnt++ + id := m.inProgressCnt + m.inProgress[id] = clientInProgress{client: clientID, cancel: fn} + return id +} + +func (m *MultiClient) unregisterCancelFn(id int) { + m.inProgressMu.Lock() + defer m.inProgressMu.Unlock() + + delete(m.inProgress, id) +} + +// Runs supplied fn with current primary client. If primary client changes, fn is restarted. +// When fn finishes (with or without error), this method returns given error value. +func (m *MultiClient) runWithPrimaryClient(origCtx context.Context, fn func(newCtx context.Context, primary kvclient) error) error { + cancelFn := context.CancelFunc(nil) + cancelFnID := 0 + + cleanup := func() { + if cancelFn != nil { + cancelFn() + } + if cancelFnID > 0 { + m.unregisterCancelFn(cancelFnID) + } + } + + defer cleanup() + + // This only loops if switchover to a new primary backend happens while calling 'fn', which is very rare. + for { + cleanup() + pid, kv := m.getPrimaryClient() + + var cancelCtx context.Context + cancelCtx, cancelFn = context.WithCancel(origCtx) + cancelFnID = m.registerCancelFn(pid, cancelFn) + + err := fn(cancelCtx, kv) + + if err == nil { + return nil + } + + if cancelCtx.Err() == context.Canceled && origCtx.Err() == nil { + // our context was cancelled, but outer context is not done yet. retry + continue + } + + return err + } +} + +// Get is a part of kv.Client interface +func (m *MultiClient) Get(ctx context.Context, key string) (interface{}, error) { + var result interface{} + err := m.runWithPrimaryClient(ctx, func(newCtx context.Context, primary kvclient) error { + var err2 error + result, err2 = primary.client.Get(newCtx, key) + return err2 + }) + return result, err +} + +// CAS is a part of kv.Client interface +func (m *MultiClient) CAS(ctx context.Context, key string, f func(in interface{}) (out interface{}, retry bool, err error)) error { + return m.runWithPrimaryClient(ctx, func(newCtx context.Context, primary kvclient) error { + return primary.client.CAS(newCtx, key, f) + }) +} + +// WatchKey is a part of kv.Client interface +func (m *MultiClient) WatchKey(ctx context.Context, key string, f func(interface{}) bool) { + _ = m.runWithPrimaryClient(ctx, func(newCtx context.Context, primary kvclient) error { + primary.client.WatchKey(newCtx, key, f) + return newCtx.Err() + }) +} + +// WatchPrefix is a part of kv.Client interface +func (m *MultiClient) WatchPrefix(ctx context.Context, prefix string, f func(string, interface{}) bool) { + _ = m.runWithPrimaryClient(ctx, func(newCtx context.Context, primary kvclient) error { + primary.client.WatchPrefix(newCtx, prefix, f) + return newCtx.Err() + }) +} + +// watchChanges performs mirroring -- it watches changes in the primary client, and puts them into secondary. +func (m *MultiClient) watchChanges(prefix string) { + for m.ctx.Err() == nil { + err := m.runWithPrimaryClient(m.ctx, func(newCtx context.Context, primary kvclient) error { + primary.client.WatchPrefix(newCtx, prefix, func(key string, val interface{}) bool { + level.Debug(util.Logger).Log("msg", "value updated", "key", key, "store", primary.name) + + if m.mirroringEnabled.Load() { + // don't pass new context to keyUpdated, we don't want to react on primary client changes here. + m.keyUpdated(primary, key, val) + } + + err := m.rateLimiter.Wait(newCtx) + if err != nil && err != context.Canceled { + level.Error(util.Logger).Log("msg", "error while rate limiting multi-client", "err", err) + } + + return true + }) + + return m.ctx.Err() + }) + + if err != nil { + level.Warn(util.Logger).Log("msg", "watching changes returned error", "err", err) + } + } +} + +func (m *MultiClient) keyUpdated(primary kvclient, key string, newValue interface{}) { + // let's propagate this to all remaining clients + for _, kvc := range m.clients { + if kvc == primary { + continue + } + + stored := true + err := kvc.client.CAS(m.ctx, key, func(in interface{}) (out interface{}, retry bool, err error) { + stored = true + + if eq, ok := in.(withEqual); ok && eq.Equal(newValue) { + stored = false + level.Debug(util.Logger).Log("msg", "no change", "key", key) + return nil, false, nil + } + + // try once + return in, false, nil + }) + + if err != nil { + level.Error(util.Logger).Log("msg", "failed to update value in secondary store", "key", key, "err", err, "primary", primary.name, "secondary", kvc.name) + } else if stored { + level.Info(util.Logger).Log("msg", "stored updated value to secondary store", "key", key, "primary", primary.name, "secondary", kvc.name) + } + } +} + +// Stop the multiClient (mirroring part). +func (m *MultiClient) stop() { + m.cancel() +} + +func createRateLimiter(rateLimit float64, burst int) *rate.Limiter { + if rateLimit <= 0 { + // burst is ignored when limit = rate.Inf + return rate.NewLimiter(rate.Inf, 0) + } + if burst < 1 { + burst = 1 + } + return rate.NewLimiter(rate.Limit(rateLimit), burst) +} + +type withEqual interface { + Equal(that interface{}) bool +} diff --git a/pkg/util/validation/override.go b/pkg/util/override.go similarity index 55% rename from pkg/util/validation/override.go rename to pkg/util/override.go index e6ef2d2a271..95ecceab3da 100644 --- a/pkg/util/validation/override.go +++ b/pkg/util/override.go @@ -1,10 +1,9 @@ -package validation +package util import ( "sync" "time" - "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -15,25 +14,31 @@ var overridesReloadSuccess = promauto.NewGauge(prometheus.GaugeOpts{ Help: "Whether the last overrides reload attempt was successful.", }) -// OverridesLoader loads the overrides -type OverridesLoader func(string) (map[string]interface{}, error) +// OverridesLoader loads the configuration from file. +type OverridesLoader func(filename string) (interface{}, error) + +// OverridesListener gets notified when new overrides is loaded +type OverridesListener func(newConfig interface{}) // OverridesManagerConfig holds the config for an OverridesManager instance. -// It holds config related to loading per-tentant overrides and the default limits +// It holds config related to loading per-tenant overrides. type OverridesManagerConfig struct { OverridesReloadPeriod time.Duration OverridesLoadPath string OverridesLoader OverridesLoader - Defaults interface{} } -// OverridesManager manages default and per user limits i.e overrides. -// It can periodically keep reloading overrides based on config. +// OverridesManager periodically reloads configuration from a file, and keeps this +// configuration available for clients. type OverridesManager struct { - cfg OverridesManagerConfig - overrides map[string]interface{} + cfg OverridesManagerConfig + quit chan struct{} + + listenersMtx sync.Mutex + listeners []OverridesListener + overridesMtx sync.RWMutex - quit chan struct{} + overrides interface{} } // NewOverridesManager creates an instance of OverridesManager and starts reload overrides loop based on config @@ -46,16 +51,29 @@ func NewOverridesManager(cfg OverridesManagerConfig) (*OverridesManager, error) if cfg.OverridesLoadPath != "" { if err := overridesManager.loadOverrides(); err != nil { // Log but don't stop on error - we don't want to halt all ingesters because of a typo - level.Error(util.Logger).Log("msg", "failed to load limit overrides", "err", err) + level.Error(Logger).Log("msg", "failed to load overrides", "err", err) } go overridesManager.loop() } else { - level.Info(util.Logger).Log("msg", "per-tenant overrides disabled") + level.Info(Logger).Log("msg", "overrides disabled") } return &overridesManager, nil } +// AddListener registers new listener function, that will receive updates configuration. +// Listener is called asynchronously to avoid blocking main reloading loop. +func (om *OverridesManager) AddListener(l OverridesListener) { + if l == nil { + panic("nil listener") + } + + om.listenersMtx.Lock() + defer om.listenersMtx.Unlock() + + om.listeners = append(om.listeners, l) +} + func (om *OverridesManager) loop() { ticker := time.NewTicker(om.cfg.OverridesReloadPeriod) defer ticker.Stop() @@ -66,7 +84,7 @@ func (om *OverridesManager) loop() { err := om.loadOverrides() if err != nil { // Log but don't stop on error - we don't want to halt all ingesters because of a typo - level.Error(util.Logger).Log("msg", "failed to load limit overrides", "err", err) + level.Error(Logger).Log("msg", "failed to load overrides", "err", err) } case <-om.quit: return @@ -82,10 +100,24 @@ func (om *OverridesManager) loadOverrides() error { } overridesReloadSuccess.Set(1) + om.setOverrides(overrides) + om.callListeners(overrides) + + return nil +} + +func (om *OverridesManager) setOverrides(overrides interface{}) { om.overridesMtx.Lock() defer om.overridesMtx.Unlock() om.overrides = overrides - return nil +} + +func (om *OverridesManager) callListeners(newValue interface{}) { + om.listenersMtx.Lock() + defer om.listenersMtx.Unlock() + for _, l := range om.listeners { + go l(newValue) + } } // Stop stops the OverridesManager @@ -93,15 +125,10 @@ func (om *OverridesManager) Stop() { close(om.quit) } -// GetLimits returns Limits for a specific userID if its set otherwise the default Limits -func (om *OverridesManager) GetLimits(userID string) interface{} { +// GetOverrides returns last loaded overrides value, possibly nil. +func (om *OverridesManager) GetOverrides() interface{} { om.overridesMtx.RLock() defer om.overridesMtx.RUnlock() - override, ok := om.overrides[userID] - if !ok { - return om.cfg.Defaults - } - - return override + return om.overrides } diff --git a/pkg/util/validation/override_test.go b/pkg/util/override_test.go similarity index 55% rename from pkg/util/validation/override_test.go rename to pkg/util/override_test.go index 9792ecefe07..2c33c7b9864 100644 --- a/pkg/util/validation/override_test.go +++ b/pkg/util/override_test.go @@ -1,4 +1,4 @@ -package validation +package util import ( "io/ioutil" @@ -18,6 +18,10 @@ type TestLimits struct { // WARNING: THIS GLOBAL VARIABLE COULD LEAD TO UNEXPECTED BEHAVIOUR WHEN RUNNING MULTIPLE DIFFERENT TESTS var defaultTestLimits *TestLimits +type testOverrides struct { + Overrides map[string]*TestLimits `yaml:"overrides"` +} + // UnmarshalYAML implements the yaml.Unmarshaler interface. func (l *TestLimits) UnmarshalYAML(unmarshal func(interface{}) error) error { if defaultTestLimits != nil { @@ -27,15 +31,14 @@ func (l *TestLimits) UnmarshalYAML(unmarshal func(interface{}) error) error { return unmarshal((*plain)(l)) } -func testLoadOverrides(filename string) (map[string]interface{}, error) { +func testLoadOverrides(filename string) (interface{}, error) { f, err := os.Open(filename) if err != nil { return nil, err } + defer f.Close() - var overrides struct { - Overrides map[string]*TestLimits `yaml:"overrides"` - } + var overrides = &testOverrides{} decoder := yaml.NewDecoder(f) decoder.SetStrict(true) @@ -43,12 +46,7 @@ func testLoadOverrides(filename string) (map[string]interface{}, error) { return nil, err } - overridesAsInterface := map[string]interface{}{} - for k := range overrides.Overrides { - overridesAsInterface[k] = overrides.Overrides[k] - } - - return overridesAsInterface, nil + return overrides, nil } func TestNewOverridesManager(t *testing.T) { @@ -73,69 +71,76 @@ func TestNewOverridesManager(t *testing.T) { OverridesReloadPeriod: time.Second, OverridesLoadPath: tempFile.Name(), OverridesLoader: testLoadOverrides, - Defaults: defaultTestLimits, } - var overridesManager *OverridesManager - done := make(chan struct{}) - - go func() { - overridesManager, err = NewOverridesManager(overridesManagerConfig) - close(done) - }() - - select { - case <-time.After(time.Second): - t.Fatal("failed to get a response from NewOverridesManager() before timeout") - case <-done: - } + overridesManager, err := NewOverridesManager(overridesManagerConfig) require.NoError(t, err) // Cleaning up overridesManager.Stop() + + // Make sure test limits were loaded. + require.NotNil(t, overridesManager.GetOverrides()) } -func TestOverridesManager_GetLimits(t *testing.T) { +func TestOverridesManager_Listener(t *testing.T) { + tempFile, err := ioutil.TempFile("", "test-validation") + require.NoError(t, err) + require.NoError(t, tempFile.Close()) + + defer func() { + // Clean up + require.NoError(t, os.Remove(tempFile.Name())) + }() + + err = ioutil.WriteFile(tempFile.Name(), []byte(`overrides: + user1: + limit2: 150`), 0600) + require.NoError(t, err) + defaultTestLimits = &TestLimits{Limit1: 100} + + // testing NewOverridesManager with overrides reload config set overridesManagerConfig := OverridesManagerConfig{ - OverridesReloadPeriod: 0, - OverridesLoadPath: "", + OverridesReloadPeriod: time.Second, + OverridesLoadPath: tempFile.Name(), OverridesLoader: testLoadOverrides, - Defaults: defaultTestLimits, } overridesManager, err := NewOverridesManager(overridesManagerConfig) require.NoError(t, err) - require.Equal(t, 100, overridesManager.GetLimits("user1").(*TestLimits).Limit1) - require.Equal(t, 0, overridesManager.GetLimits("user1").(*TestLimits).Limit2) + // listeners are called asynchronously + ch := make(chan interface{}) + overridesManager.AddListener(func(newConfig interface{}) { + ch <- newConfig + }) - // Setting up perTenantOverrides for user user1 - tempFile, err := ioutil.TempFile("", "test-validation") + // rewrite file + err = ioutil.WriteFile(tempFile.Name(), []byte(`overrides: + user2: + limit2: 200`), 0600) require.NoError(t, err) - defer func() { - // Clean up - require.NoError(t, tempFile.Close()) - require.NoError(t, os.Remove(tempFile.Name())) - }() - - _, err = tempFile.WriteString(`overrides: - user1: - limit2: 150`) + // reload + err = overridesManager.loadOverrides() require.NoError(t, err) - overridesManager.cfg.OverridesLoadPath = tempFile.Name() - require.NoError(t, overridesManager.loadOverrides()) - - // Checking whether overrides were enforced - require.Equal(t, 100, overridesManager.GetLimits("user1").(*TestLimits).Limit1) - require.Equal(t, 150, overridesManager.GetLimits("user1").(*TestLimits).Limit2) + var newValue interface{} + select { + case newValue = <-ch: + // ok + case <-time.After(time.Second): + t.Fatal("listener was not called") + } - // Verifying user2 limits are not impacted by overrides - require.Equal(t, 100, overridesManager.GetLimits("user2").(*TestLimits).Limit1) - require.Equal(t, 0, overridesManager.GetLimits("user2").(*TestLimits).Limit2) + to := newValue.(*testOverrides) + require.Equal(t, 200, to.Overrides["user2"].Limit2) // new overrides + require.Equal(t, 100, to.Overrides["user2"].Limit1) // from defaults // Cleaning up overridesManager.Stop() + + // Make sure test limits were loaded. + require.NotNil(t, overridesManager.GetOverrides()) } diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 2eb60d9a163..5a508c7880f 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -3,11 +3,8 @@ package validation import ( "errors" "flag" - "os" "time" - "gopkg.in/yaml.v2" - "github.com/cortexproject/cortex/pkg/util/flagext" ) @@ -55,7 +52,7 @@ type Limits struct { MaxQueryParallelism int `yaml:"max_query_parallelism"` CardinalityLimit int `yaml:"cardinality_limit"` - // Config for overrides, convenient if it goes here. + // Config for overrides, convenient if it goes here. [Deprecated in favor of RuntimeConfig flag in cortex.Config] PerTenantOverrideConfig string `yaml:"per_tenant_override_config"` PerTenantOverridePeriod time.Duration `yaml:"per_tenant_override_period"` } @@ -90,8 +87,8 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxQueryParallelism, "querier.max-query-parallelism", 14, "Maximum number of queries will be scheduled in parallel by the frontend.") f.IntVar(&l.CardinalityLimit, "store.cardinality-limit", 1e5, "Cardinality limit for index queries.") - f.StringVar(&l.PerTenantOverrideConfig, "limits.per-user-override-config", "", "File name of per-user overrides.") - f.DurationVar(&l.PerTenantOverridePeriod, "limits.per-user-override-period", 10*time.Second, "Period with which to reload the overrides.") + f.StringVar(&l.PerTenantOverrideConfig, "limits.per-user-override-config", "", "File name of per-user overrides. [deprecated, use -runtime-config.file instead]") + f.DurationVar(&l.PerTenantOverridePeriod, "limits.per-user-override-period", 10*time.Second, "Period with which to reload the overrides. [deprecated, use -runtime-config.reload-period instead]") } // Validate the limits config and returns an error if the validation @@ -126,43 +123,31 @@ func (l *Limits) UnmarshalYAML(unmarshal func(interface{}) error) error { // find a nicer way I'm afraid. var defaultLimits *Limits +// TenantLimits is a function that returns limits for given tenant, or +// nil, if there are no tenant-specific limits. +type TenantLimits func(userID string) *Limits + // Overrides periodically fetch a set of per-user overrides, and provides convenience // functions for fetching the correct value. type Overrides struct { - overridesManager *OverridesManager + defaultLimits *Limits + tenantLimits TenantLimits } // NewOverrides makes a new Overrides. // We store the supplied limits in a global variable to ensure per-tenant limits // are defaulted to those values. As such, the last call to NewOverrides will // become the new global defaults. -func NewOverrides(defaults Limits) (*Overrides, error) { - defaultLimits = &defaults - overridesManagerConfig := OverridesManagerConfig{ - OverridesReloadPeriod: defaults.PerTenantOverridePeriod, - OverridesLoadPath: defaults.PerTenantOverrideConfig, - OverridesLoader: loadOverrides, - Defaults: &defaults, - } - - overridesManager, err := NewOverridesManager(overridesManagerConfig) - if err != nil { - return nil, err - } - +func NewOverrides(defaults Limits, tenantLimits TenantLimits) (*Overrides, error) { return &Overrides{ - overridesManager: overridesManager, + tenantLimits: tenantLimits, + defaultLimits: &defaults, }, nil } -// Stop background reloading of overrides. -func (o *Overrides) Stop() { - o.overridesManager.Stop() -} - // IngestionRate returns the limit on ingester rate (samples per second). func (o *Overrides) IngestionRate(userID string) float64 { - return o.overridesManager.GetLimits(userID).(*Limits).IngestionRate + return o.getOverridesForUser(userID).IngestionRate } // IngestionRateStrategy returns whether the ingestion rate limit should be individually applied @@ -175,148 +160,129 @@ func (o *Overrides) IngestionRateStrategy() string { // IngestionBurstSize returns the burst size for ingestion rate. func (o *Overrides) IngestionBurstSize(userID string) int { - return o.overridesManager.GetLimits(userID).(*Limits).IngestionBurstSize + return o.getOverridesForUser(userID).IngestionBurstSize } // AcceptHASamples returns whether the distributor should track and accept samples from HA replicas for this user. func (o *Overrides) AcceptHASamples(userID string) bool { - return o.overridesManager.GetLimits(userID).(*Limits).AcceptHASamples + return o.getOverridesForUser(userID).AcceptHASamples } // HAClusterLabel returns the cluster label to look for when deciding whether to accept a sample from a Prometheus HA replica. func (o *Overrides) HAClusterLabel(userID string) string { - return o.overridesManager.GetLimits(userID).(*Limits).HAClusterLabel + return o.getOverridesForUser(userID).HAClusterLabel } // HAReplicaLabel returns the replica label to look for when deciding whether to accept a sample from a Prometheus HA replica. func (o *Overrides) HAReplicaLabel(userID string) string { - return o.overridesManager.GetLimits(userID).(*Limits).HAReplicaLabel + return o.getOverridesForUser(userID).HAReplicaLabel } // DropLabels returns the list of labels to be dropped when ingesting HA samples for the user. func (o *Overrides) DropLabels(userID string) flagext.StringSlice { - return o.overridesManager.GetLimits(userID).(*Limits).DropLabels + return o.getOverridesForUser(userID).DropLabels } // MaxLabelNameLength returns maximum length a label name can be. func (o *Overrides) MaxLabelNameLength(userID string) int { - return o.overridesManager.GetLimits(userID).(*Limits).MaxLabelNameLength + return o.getOverridesForUser(userID).MaxLabelNameLength } // MaxLabelValueLength returns maximum length a label value can be. This also is // the maximum length of a metric name. func (o *Overrides) MaxLabelValueLength(userID string) int { - return o.overridesManager.GetLimits(userID).(*Limits).MaxLabelValueLength + return o.getOverridesForUser(userID).MaxLabelValueLength } // MaxLabelNamesPerSeries returns maximum number of label/value pairs timeseries. func (o *Overrides) MaxLabelNamesPerSeries(userID string) int { - return o.overridesManager.GetLimits(userID).(*Limits).MaxLabelNamesPerSeries + return o.getOverridesForUser(userID).MaxLabelNamesPerSeries } // RejectOldSamples returns true when we should reject samples older than certain // age. func (o *Overrides) RejectOldSamples(userID string) bool { - return o.overridesManager.GetLimits(userID).(*Limits).RejectOldSamples + return o.getOverridesForUser(userID).RejectOldSamples } // RejectOldSamplesMaxAge returns the age at which samples should be rejected. func (o *Overrides) RejectOldSamplesMaxAge(userID string) time.Duration { - return o.overridesManager.GetLimits(userID).(*Limits).RejectOldSamplesMaxAge + return o.getOverridesForUser(userID).RejectOldSamplesMaxAge } // CreationGracePeriod is misnamed, and actually returns how far into the future // we should accept samples. func (o *Overrides) CreationGracePeriod(userID string) time.Duration { - return o.overridesManager.GetLimits(userID).(*Limits).CreationGracePeriod + return o.getOverridesForUser(userID).CreationGracePeriod } // MaxSeriesPerQuery returns the maximum number of series a query is allowed to hit. func (o *Overrides) MaxSeriesPerQuery(userID string) int { - return o.overridesManager.GetLimits(userID).(*Limits).MaxSeriesPerQuery + return o.getOverridesForUser(userID).MaxSeriesPerQuery } // MaxSamplesPerQuery returns the maximum number of samples in a query (from the ingester). func (o *Overrides) MaxSamplesPerQuery(userID string) int { - return o.overridesManager.GetLimits(userID).(*Limits).MaxSamplesPerQuery + return o.getOverridesForUser(userID).MaxSamplesPerQuery } // MaxLocalSeriesPerUser returns the maximum number of series a user is allowed to store in a single ingester. func (o *Overrides) MaxLocalSeriesPerUser(userID string) int { - return o.overridesManager.GetLimits(userID).(*Limits).MaxLocalSeriesPerUser + return o.getOverridesForUser(userID).MaxLocalSeriesPerUser } // MaxLocalSeriesPerMetric returns the maximum number of series allowed per metric in a single ingester. func (o *Overrides) MaxLocalSeriesPerMetric(userID string) int { - return o.overridesManager.GetLimits(userID).(*Limits).MaxLocalSeriesPerMetric + return o.getOverridesForUser(userID).MaxLocalSeriesPerMetric } // MaxGlobalSeriesPerUser returns the maximum number of series a user is allowed to store across the cluster. func (o *Overrides) MaxGlobalSeriesPerUser(userID string) int { - return o.overridesManager.GetLimits(userID).(*Limits).MaxGlobalSeriesPerUser + return o.getOverridesForUser(userID).MaxGlobalSeriesPerUser } // MaxGlobalSeriesPerMetric returns the maximum number of series allowed per metric across the cluster. func (o *Overrides) MaxGlobalSeriesPerMetric(userID string) int { - return o.overridesManager.GetLimits(userID).(*Limits).MaxGlobalSeriesPerMetric + return o.getOverridesForUser(userID).MaxGlobalSeriesPerMetric } // MaxChunksPerQuery returns the maximum number of chunks allowed per query. func (o *Overrides) MaxChunksPerQuery(userID string) int { - return o.overridesManager.GetLimits(userID).(*Limits).MaxChunksPerQuery + return o.getOverridesForUser(userID).MaxChunksPerQuery } // MaxQueryLength returns the limit of the length (in time) of a query. func (o *Overrides) MaxQueryLength(userID string) time.Duration { - return o.overridesManager.GetLimits(userID).(*Limits).MaxQueryLength + return o.getOverridesForUser(userID).MaxQueryLength } // MaxQueryParallelism returns the limit to the number of sub-queries the // frontend will process in parallel. func (o *Overrides) MaxQueryParallelism(userID string) int { - return o.overridesManager.GetLimits(userID).(*Limits).MaxQueryParallelism + return o.getOverridesForUser(userID).MaxQueryParallelism } // EnforceMetricName whether to enforce the presence of a metric name. func (o *Overrides) EnforceMetricName(userID string) bool { - return o.overridesManager.GetLimits(userID).(*Limits).EnforceMetricName + return o.getOverridesForUser(userID).EnforceMetricName } // CardinalityLimit returns the maximum number of timeseries allowed in a query. func (o *Overrides) CardinalityLimit(userID string) int { - return o.overridesManager.GetLimits(userID).(*Limits).CardinalityLimit + return o.getOverridesForUser(userID).CardinalityLimit } // MinChunkLength returns the minimum size of chunk that will be saved by ingesters func (o *Overrides) MinChunkLength(userID string) int { - return o.overridesManager.GetLimits(userID).(*Limits).MinChunkLength -} - -// Loads overrides and returns the limits as an interface to store them in OverridesManager. -// We need to implement it here since OverridesManager must store type Limits in an interface but -// it doesn't know its definition to initialize it. -// We could have used yamlv3.Node for this but there is no way to enforce strict decoding due to a bug in it -// TODO: Use yamlv3.Node to move this to OverridesManager after https://github.com/go-yaml/yaml/issues/460 is fixed -func loadOverrides(filename string) (map[string]interface{}, error) { - f, err := os.Open(filename) - if err != nil { - return nil, err - } - - var overrides struct { - Overrides map[string]*Limits `yaml:"overrides"` - } - - decoder := yaml.NewDecoder(f) - decoder.SetStrict(true) - if err := decoder.Decode(&overrides); err != nil { - return nil, err - } + return o.getOverridesForUser(userID).MinChunkLength +} - overridesAsInterface := map[string]interface{}{} - for userID := range overrides.Overrides { - overridesAsInterface[userID] = overrides.Overrides[userID] +func (o *Overrides) getOverridesForUser(userID string) *Limits { + if o.tenantLimits != nil { + l := o.tenantLimits(userID) + if l != nil { + return l + } } - - return overridesAsInterface, nil + return o.defaultLimits } diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index dcb4b15313b..fba3f71d350 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestLimits_Validate(t *testing.T) { @@ -39,3 +40,35 @@ func TestLimits_Validate(t *testing.T) { }) } } + +func TestOverridesManager_GetOverrides(t *testing.T) { + tenantLimits := map[string]*Limits{} + + defaults := Limits{ + MaxLabelNamesPerSeries: 100, + } + ov, err := NewOverrides(defaults, func(userID string) *Limits { + return tenantLimits[userID] + }) + + require.NoError(t, err) + + require.Equal(t, 100, ov.MaxLabelNamesPerSeries("user1")) + require.Equal(t, 0, ov.MaxLabelValueLength("user1")) + + // Update limits for tenant user1. We only update single field, the rest is copied from defaults. + // (That is how limits work when loaded from YAML) + l := Limits{} + l = defaults + l.MaxLabelValueLength = 150 + + tenantLimits["user1"] = &l + + // Checking whether overrides were enforced + require.Equal(t, 100, ov.MaxLabelNamesPerSeries("user1")) + require.Equal(t, 150, ov.MaxLabelValueLength("user1")) + + // Verifying user2 limits are not impacted by overrides + require.Equal(t, 100, ov.MaxLabelNamesPerSeries("user2")) + require.Equal(t, 0, ov.MaxLabelValueLength("user2")) +} diff --git a/vendor/github.com/uber-go/atomic/.codecov.yml b/vendor/github.com/uber-go/atomic/.codecov.yml new file mode 100644 index 00000000000..6d4d1be7b57 --- /dev/null +++ b/vendor/github.com/uber-go/atomic/.codecov.yml @@ -0,0 +1,15 @@ +coverage: + range: 80..100 + round: down + precision: 2 + + status: + project: # measuring the overall project coverage + default: # context, you can create multiple ones with custom titles + enabled: yes # must be yes|true to enable this status + target: 100 # specify the target coverage for each commit status + # option: "auto" (must increase from parent commit or pull request base) + # option: "X%" a static target percentage to hit + if_not_found: success # if parent is not found report status as success, error, or failure + if_ci_failed: error # if ci fails report status as success, error, or failure + diff --git a/vendor/github.com/uber-go/atomic/.gitignore b/vendor/github.com/uber-go/atomic/.gitignore new file mode 100644 index 00000000000..0a4504f1109 --- /dev/null +++ b/vendor/github.com/uber-go/atomic/.gitignore @@ -0,0 +1,11 @@ +.DS_Store +/vendor +/cover +cover.out +lint.log + +# Binaries +*.test + +# Profiling output +*.prof diff --git a/vendor/github.com/uber-go/atomic/.travis.yml b/vendor/github.com/uber-go/atomic/.travis.yml new file mode 100644 index 00000000000..0f3769e5fa6 --- /dev/null +++ b/vendor/github.com/uber-go/atomic/.travis.yml @@ -0,0 +1,27 @@ +sudo: false +language: go +go_import_path: go.uber.org/atomic + +go: + - 1.11.x + - 1.12.x + +matrix: + include: + - go: 1.12.x + env: NO_TEST=yes LINT=yes + +cache: + directories: + - vendor + +install: + - make install_ci + +script: + - test -n "$NO_TEST" || make test_ci + - test -n "$NO_TEST" || scripts/test-ubergo.sh + - test -z "$LINT" || make install_lint lint + +after_success: + - bash <(curl -s https://codecov.io/bash) diff --git a/vendor/github.com/uber-go/atomic/LICENSE.txt b/vendor/github.com/uber-go/atomic/LICENSE.txt new file mode 100644 index 00000000000..8765c9fbc61 --- /dev/null +++ b/vendor/github.com/uber-go/atomic/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2016 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/uber-go/atomic/Makefile b/vendor/github.com/uber-go/atomic/Makefile new file mode 100644 index 00000000000..1ef263075d7 --- /dev/null +++ b/vendor/github.com/uber-go/atomic/Makefile @@ -0,0 +1,51 @@ +# Many Go tools take file globs or directories as arguments instead of packages. +PACKAGE_FILES ?= *.go + +# For pre go1.6 +export GO15VENDOREXPERIMENT=1 + + +.PHONY: build +build: + go build -i ./... + + +.PHONY: install +install: + glide --version || go get github.com/Masterminds/glide + glide install + + +.PHONY: test +test: + go test -cover -race ./... + + +.PHONY: install_ci +install_ci: install + go get github.com/wadey/gocovmerge + go get github.com/mattn/goveralls + go get golang.org/x/tools/cmd/cover + +.PHONY: install_lint +install_lint: + go get golang.org/x/lint/golint + + +.PHONY: lint +lint: + @rm -rf lint.log + @echo "Checking formatting..." + @gofmt -d -s $(PACKAGE_FILES) 2>&1 | tee lint.log + @echo "Checking vet..." + @go vet ./... 2>&1 | tee -a lint.log;) + @echo "Checking lint..." + @golint $$(go list ./...) 2>&1 | tee -a lint.log + @echo "Checking for unresolved FIXMEs..." + @git grep -i fixme | grep -v -e vendor -e Makefile | tee -a lint.log + @[ ! -s lint.log ] + + +.PHONY: test_ci +test_ci: install_ci build + ./scripts/cover.sh $(shell go list $(PACKAGES)) diff --git a/vendor/github.com/uber-go/atomic/README.md b/vendor/github.com/uber-go/atomic/README.md new file mode 100644 index 00000000000..62eb8e57609 --- /dev/null +++ b/vendor/github.com/uber-go/atomic/README.md @@ -0,0 +1,36 @@ +# atomic [![GoDoc][doc-img]][doc] [![Build Status][ci-img]][ci] [![Coverage Status][cov-img]][cov] [![Go Report Card][reportcard-img]][reportcard] + +Simple wrappers for primitive types to enforce atomic access. + +## Installation +`go get -u go.uber.org/atomic` + +## Usage +The standard library's `sync/atomic` is powerful, but it's easy to forget which +variables must be accessed atomically. `go.uber.org/atomic` preserves all the +functionality of the standard library, but wraps the primitive types to +provide a safer, more convenient API. + +```go +var atom atomic.Uint32 +atom.Store(42) +atom.Sub(2) +atom.CAS(40, 11) +``` + +See the [documentation][doc] for a complete API specification. + +## Development Status +Stable. + +___ +Released under the [MIT License](LICENSE.txt). + +[doc-img]: https://godoc.org/github.com/uber-go/atomic?status.svg +[doc]: https://godoc.org/go.uber.org/atomic +[ci-img]: https://travis-ci.com/uber-go/atomic.svg?branch=master +[ci]: https://travis-ci.com/uber-go/atomic +[cov-img]: https://codecov.io/gh/uber-go/atomic/branch/master/graph/badge.svg +[cov]: https://codecov.io/gh/uber-go/atomic +[reportcard-img]: https://goreportcard.com/badge/go.uber.org/atomic +[reportcard]: https://goreportcard.com/report/go.uber.org/atomic diff --git a/vendor/github.com/uber-go/atomic/atomic.go b/vendor/github.com/uber-go/atomic/atomic.go new file mode 100644 index 00000000000..1db6849fca0 --- /dev/null +++ b/vendor/github.com/uber-go/atomic/atomic.go @@ -0,0 +1,351 @@ +// Copyright (c) 2016 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Package atomic provides simple wrappers around numerics to enforce atomic +// access. +package atomic + +import ( + "math" + "sync/atomic" + "time" +) + +// Int32 is an atomic wrapper around an int32. +type Int32 struct{ v int32 } + +// NewInt32 creates an Int32. +func NewInt32(i int32) *Int32 { + return &Int32{i} +} + +// Load atomically loads the wrapped value. +func (i *Int32) Load() int32 { + return atomic.LoadInt32(&i.v) +} + +// Add atomically adds to the wrapped int32 and returns the new value. +func (i *Int32) Add(n int32) int32 { + return atomic.AddInt32(&i.v, n) +} + +// Sub atomically subtracts from the wrapped int32 and returns the new value. +func (i *Int32) Sub(n int32) int32 { + return atomic.AddInt32(&i.v, -n) +} + +// Inc atomically increments the wrapped int32 and returns the new value. +func (i *Int32) Inc() int32 { + return i.Add(1) +} + +// Dec atomically decrements the wrapped int32 and returns the new value. +func (i *Int32) Dec() int32 { + return i.Sub(1) +} + +// CAS is an atomic compare-and-swap. +func (i *Int32) CAS(old, new int32) bool { + return atomic.CompareAndSwapInt32(&i.v, old, new) +} + +// Store atomically stores the passed value. +func (i *Int32) Store(n int32) { + atomic.StoreInt32(&i.v, n) +} + +// Swap atomically swaps the wrapped int32 and returns the old value. +func (i *Int32) Swap(n int32) int32 { + return atomic.SwapInt32(&i.v, n) +} + +// Int64 is an atomic wrapper around an int64. +type Int64 struct{ v int64 } + +// NewInt64 creates an Int64. +func NewInt64(i int64) *Int64 { + return &Int64{i} +} + +// Load atomically loads the wrapped value. +func (i *Int64) Load() int64 { + return atomic.LoadInt64(&i.v) +} + +// Add atomically adds to the wrapped int64 and returns the new value. +func (i *Int64) Add(n int64) int64 { + return atomic.AddInt64(&i.v, n) +} + +// Sub atomically subtracts from the wrapped int64 and returns the new value. +func (i *Int64) Sub(n int64) int64 { + return atomic.AddInt64(&i.v, -n) +} + +// Inc atomically increments the wrapped int64 and returns the new value. +func (i *Int64) Inc() int64 { + return i.Add(1) +} + +// Dec atomically decrements the wrapped int64 and returns the new value. +func (i *Int64) Dec() int64 { + return i.Sub(1) +} + +// CAS is an atomic compare-and-swap. +func (i *Int64) CAS(old, new int64) bool { + return atomic.CompareAndSwapInt64(&i.v, old, new) +} + +// Store atomically stores the passed value. +func (i *Int64) Store(n int64) { + atomic.StoreInt64(&i.v, n) +} + +// Swap atomically swaps the wrapped int64 and returns the old value. +func (i *Int64) Swap(n int64) int64 { + return atomic.SwapInt64(&i.v, n) +} + +// Uint32 is an atomic wrapper around an uint32. +type Uint32 struct{ v uint32 } + +// NewUint32 creates a Uint32. +func NewUint32(i uint32) *Uint32 { + return &Uint32{i} +} + +// Load atomically loads the wrapped value. +func (i *Uint32) Load() uint32 { + return atomic.LoadUint32(&i.v) +} + +// Add atomically adds to the wrapped uint32 and returns the new value. +func (i *Uint32) Add(n uint32) uint32 { + return atomic.AddUint32(&i.v, n) +} + +// Sub atomically subtracts from the wrapped uint32 and returns the new value. +func (i *Uint32) Sub(n uint32) uint32 { + return atomic.AddUint32(&i.v, ^(n - 1)) +} + +// Inc atomically increments the wrapped uint32 and returns the new value. +func (i *Uint32) Inc() uint32 { + return i.Add(1) +} + +// Dec atomically decrements the wrapped int32 and returns the new value. +func (i *Uint32) Dec() uint32 { + return i.Sub(1) +} + +// CAS is an atomic compare-and-swap. +func (i *Uint32) CAS(old, new uint32) bool { + return atomic.CompareAndSwapUint32(&i.v, old, new) +} + +// Store atomically stores the passed value. +func (i *Uint32) Store(n uint32) { + atomic.StoreUint32(&i.v, n) +} + +// Swap atomically swaps the wrapped uint32 and returns the old value. +func (i *Uint32) Swap(n uint32) uint32 { + return atomic.SwapUint32(&i.v, n) +} + +// Uint64 is an atomic wrapper around a uint64. +type Uint64 struct{ v uint64 } + +// NewUint64 creates a Uint64. +func NewUint64(i uint64) *Uint64 { + return &Uint64{i} +} + +// Load atomically loads the wrapped value. +func (i *Uint64) Load() uint64 { + return atomic.LoadUint64(&i.v) +} + +// Add atomically adds to the wrapped uint64 and returns the new value. +func (i *Uint64) Add(n uint64) uint64 { + return atomic.AddUint64(&i.v, n) +} + +// Sub atomically subtracts from the wrapped uint64 and returns the new value. +func (i *Uint64) Sub(n uint64) uint64 { + return atomic.AddUint64(&i.v, ^(n - 1)) +} + +// Inc atomically increments the wrapped uint64 and returns the new value. +func (i *Uint64) Inc() uint64 { + return i.Add(1) +} + +// Dec atomically decrements the wrapped uint64 and returns the new value. +func (i *Uint64) Dec() uint64 { + return i.Sub(1) +} + +// CAS is an atomic compare-and-swap. +func (i *Uint64) CAS(old, new uint64) bool { + return atomic.CompareAndSwapUint64(&i.v, old, new) +} + +// Store atomically stores the passed value. +func (i *Uint64) Store(n uint64) { + atomic.StoreUint64(&i.v, n) +} + +// Swap atomically swaps the wrapped uint64 and returns the old value. +func (i *Uint64) Swap(n uint64) uint64 { + return atomic.SwapUint64(&i.v, n) +} + +// Bool is an atomic Boolean. +type Bool struct{ v uint32 } + +// NewBool creates a Bool. +func NewBool(initial bool) *Bool { + return &Bool{boolToInt(initial)} +} + +// Load atomically loads the Boolean. +func (b *Bool) Load() bool { + return truthy(atomic.LoadUint32(&b.v)) +} + +// CAS is an atomic compare-and-swap. +func (b *Bool) CAS(old, new bool) bool { + return atomic.CompareAndSwapUint32(&b.v, boolToInt(old), boolToInt(new)) +} + +// Store atomically stores the passed value. +func (b *Bool) Store(new bool) { + atomic.StoreUint32(&b.v, boolToInt(new)) +} + +// Swap sets the given value and returns the previous value. +func (b *Bool) Swap(new bool) bool { + return truthy(atomic.SwapUint32(&b.v, boolToInt(new))) +} + +// Toggle atomically negates the Boolean and returns the previous value. +func (b *Bool) Toggle() bool { + return truthy(atomic.AddUint32(&b.v, 1) - 1) +} + +func truthy(n uint32) bool { + return n&1 == 1 +} + +func boolToInt(b bool) uint32 { + if b { + return 1 + } + return 0 +} + +// Float64 is an atomic wrapper around float64. +type Float64 struct { + v uint64 +} + +// NewFloat64 creates a Float64. +func NewFloat64(f float64) *Float64 { + return &Float64{math.Float64bits(f)} +} + +// Load atomically loads the wrapped value. +func (f *Float64) Load() float64 { + return math.Float64frombits(atomic.LoadUint64(&f.v)) +} + +// Store atomically stores the passed value. +func (f *Float64) Store(s float64) { + atomic.StoreUint64(&f.v, math.Float64bits(s)) +} + +// Add atomically adds to the wrapped float64 and returns the new value. +func (f *Float64) Add(s float64) float64 { + for { + old := f.Load() + new := old + s + if f.CAS(old, new) { + return new + } + } +} + +// Sub atomically subtracts from the wrapped float64 and returns the new value. +func (f *Float64) Sub(s float64) float64 { + return f.Add(-s) +} + +// CAS is an atomic compare-and-swap. +func (f *Float64) CAS(old, new float64) bool { + return atomic.CompareAndSwapUint64(&f.v, math.Float64bits(old), math.Float64bits(new)) +} + +// Duration is an atomic wrapper around time.Duration +// https://godoc.org/time#Duration +type Duration struct { + v Int64 +} + +// NewDuration creates a Duration. +func NewDuration(d time.Duration) *Duration { + return &Duration{v: *NewInt64(int64(d))} +} + +// Load atomically loads the wrapped value. +func (d *Duration) Load() time.Duration { + return time.Duration(d.v.Load()) +} + +// Store atomically stores the passed value. +func (d *Duration) Store(n time.Duration) { + d.v.Store(int64(n)) +} + +// Add atomically adds to the wrapped time.Duration and returns the new value. +func (d *Duration) Add(n time.Duration) time.Duration { + return time.Duration(d.v.Add(int64(n))) +} + +// Sub atomically subtracts from the wrapped time.Duration and returns the new value. +func (d *Duration) Sub(n time.Duration) time.Duration { + return time.Duration(d.v.Sub(int64(n))) +} + +// Swap atomically swaps the wrapped time.Duration and returns the old value. +func (d *Duration) Swap(n time.Duration) time.Duration { + return time.Duration(d.v.Swap(int64(n))) +} + +// CAS is an atomic compare-and-swap. +func (d *Duration) CAS(old, new time.Duration) bool { + return d.v.CAS(int64(old), int64(new)) +} + +// Value shadows the type of the same name from sync/atomic +// https://godoc.org/sync/atomic#Value +type Value struct{ atomic.Value } diff --git a/vendor/github.com/uber-go/atomic/error.go b/vendor/github.com/uber-go/atomic/error.go new file mode 100644 index 00000000000..0489d19badb --- /dev/null +++ b/vendor/github.com/uber-go/atomic/error.go @@ -0,0 +1,55 @@ +// Copyright (c) 2016 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package atomic + +// Error is an atomic type-safe wrapper around Value for errors +type Error struct{ v Value } + +// errorHolder is non-nil holder for error object. +// atomic.Value panics on saving nil object, so err object needs to be +// wrapped with valid object first. +type errorHolder struct{ err error } + +// NewError creates new atomic error object +func NewError(err error) *Error { + e := &Error{} + if err != nil { + e.Store(err) + } + return e +} + +// Load atomically loads the wrapped error +func (e *Error) Load() error { + v := e.v.Load() + if v == nil { + return nil + } + + eh := v.(errorHolder) + return eh.err +} + +// Store atomically stores error. +// NOTE: a holder object is allocated on each Store call. +func (e *Error) Store(err error) { + e.v.Store(errorHolder{err: err}) +} diff --git a/vendor/github.com/uber-go/atomic/glide.lock b/vendor/github.com/uber-go/atomic/glide.lock new file mode 100644 index 00000000000..3c72c59976d --- /dev/null +++ b/vendor/github.com/uber-go/atomic/glide.lock @@ -0,0 +1,17 @@ +hash: f14d51408e3e0e4f73b34e4039484c78059cd7fc5f4996fdd73db20dc8d24f53 +updated: 2016-10-27T00:10:51.16960137-07:00 +imports: [] +testImports: +- name: github.com/davecgh/go-spew + version: 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d + subpackages: + - spew +- name: github.com/pmezard/go-difflib + version: d8ed2627bdf02c080bf22230dbb337003b7aba2d + subpackages: + - difflib +- name: github.com/stretchr/testify + version: d77da356e56a7428ad25149ca77381849a6a5232 + subpackages: + - assert + - require diff --git a/vendor/github.com/uber-go/atomic/glide.yaml b/vendor/github.com/uber-go/atomic/glide.yaml new file mode 100644 index 00000000000..4cf608ec0f8 --- /dev/null +++ b/vendor/github.com/uber-go/atomic/glide.yaml @@ -0,0 +1,6 @@ +package: go.uber.org/atomic +testImport: +- package: github.com/stretchr/testify + subpackages: + - assert + - require diff --git a/vendor/github.com/uber-go/atomic/string.go b/vendor/github.com/uber-go/atomic/string.go new file mode 100644 index 00000000000..ede8136face --- /dev/null +++ b/vendor/github.com/uber-go/atomic/string.go @@ -0,0 +1,49 @@ +// Copyright (c) 2016 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package atomic + +// String is an atomic type-safe wrapper around Value for strings. +type String struct{ v Value } + +// NewString creates a String. +func NewString(str string) *String { + s := &String{} + if str != "" { + s.Store(str) + } + return s +} + +// Load atomically loads the wrapped string. +func (s *String) Load() string { + v := s.v.Load() + if v == nil { + return "" + } + return v.(string) +} + +// Store atomically stores the passed string. +// Note: Converting the string to an interface{} to store in the Value +// requires an allocation. +func (s *String) Store(str string) { + s.v.Store(str) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index d0809d84782..228205ec00c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -538,6 +538,8 @@ github.com/thanos-io/thanos/pkg/tracing github.com/tinylib/msgp/msgp # github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 github.com/tmc/grpc-websocket-proxy/wsproxy +# github.com/uber-go/atomic v1.4.0 +github.com/uber-go/atomic # github.com/uber/jaeger-client-go v2.20.1+incompatible github.com/uber/jaeger-client-go github.com/uber/jaeger-client-go/config From 1b2f37a0c698bff522d4550f2f8f9f8811ae0ce3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Fri, 1 Nov 2019 10:46:50 +0100 Subject: [PATCH 02/31] Use Stop, which is now part of kv.Client. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/ring/kv/multi.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/ring/kv/multi.go b/pkg/ring/kv/multi.go index ad32017d0a5..2033a0d7804 100644 --- a/pkg/ring/kv/multi.go +++ b/pkg/ring/kv/multi.go @@ -316,9 +316,13 @@ func (m *MultiClient) keyUpdated(primary kvclient, key string, newValue interfac } } -// Stop the multiClient (mirroring part). -func (m *MultiClient) stop() { +// Stop the multiClient (mirroring part), and all configured clients. +func (m *MultiClient) Stop() { m.cancel() + + for _, kv := range m.clients { + kv.client.Stop() + } } func createRateLimiter(rateLimit float64, burst int) *rate.Limiter { From 6dc84fc07882888fd929322297f93229b53fd828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Fri, 1 Nov 2019 11:07:49 +0100 Subject: [PATCH 03/31] Put back setting of defaultLimits -- used when loading YAML files. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/util/validation/limits.go | 1 + pkg/util/validation/limits_test.go | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 5a508c7880f..57d257f19aa 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -139,6 +139,7 @@ type Overrides struct { // are defaulted to those values. As such, the last call to NewOverrides will // become the new global defaults. func NewOverrides(defaults Limits, tenantLimits TenantLimits) (*Overrides, error) { + defaultLimits = &defaults return &Overrides{ tenantLimits: tenantLimits, defaultLimits: &defaults, diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index fba3f71d350..a543bf2b7f9 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -5,6 +5,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" ) func TestLimits_Validate(t *testing.T) { @@ -72,3 +73,22 @@ func TestOverridesManager_GetOverrides(t *testing.T) { require.Equal(t, 100, ov.MaxLabelNamesPerSeries("user2")) require.Equal(t, 0, ov.MaxLabelValueLength("user2")) } + +func TestLimitsLoadingFromYaml(t *testing.T) { + defaults := Limits{ + MaxLabelNameLength: 100, + } + + // we call this for its side-effect: setting of defaults + _, err := NewOverrides(defaults, nil) + require.NoError(t, err) + + inp := `ingestion_rate: 0.5` + + l := Limits{} + err = yaml.Unmarshal([]byte(inp), &l) + require.NoError(t, err) + + assert.Equal(t, 0.5, l.IngestionRate, "from yaml") + assert.Equal(t, 100, l.MaxLabelNameLength, "from defaults") +} From 50c013c834797ff9d744fb8a63cbd59d85ffc022 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Fri, 1 Nov 2019 11:15:23 +0100 Subject: [PATCH 04/31] Moved setting of default limits for YAML unmarshal to separate function. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/cortex/modules.go | 1 + pkg/util/validation/limits.go | 11 +++++++---- pkg/util/validation/limits_test.go | 10 +++------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index bf7891f42c3..20494e68d02 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -188,6 +188,7 @@ func (t *Cortex) stopRuntimeConfig() (err error) { } func (t *Cortex) initOverrides(cfg *Config) (err error) { + validation.SetDefaultLimitsForYAMLUnmarshalling(cfg.LimitsConfig) t.overrides, err = validation.NewOverrides(cfg.LimitsConfig, tenantLimitsFromRuntimeConfig(t.runtimeConfig)) return err } diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 57d257f19aa..64278882cae 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -123,6 +123,13 @@ func (l *Limits) UnmarshalYAML(unmarshal func(interface{}) error) error { // find a nicer way I'm afraid. var defaultLimits *Limits +// SetDefaultLimitsForYAMLUnmarshalling sets global default limits, used when loading +// Limits from YAML files. This is used to ensure per-tenant limits are defaulted to +// those values. +func SetDefaultLimitsForYAMLUnmarshalling(defaults Limits) { + defaultLimits = &defaults +} + // TenantLimits is a function that returns limits for given tenant, or // nil, if there are no tenant-specific limits. type TenantLimits func(userID string) *Limits @@ -135,11 +142,7 @@ type Overrides struct { } // NewOverrides makes a new Overrides. -// We store the supplied limits in a global variable to ensure per-tenant limits -// are defaulted to those values. As such, the last call to NewOverrides will -// become the new global defaults. func NewOverrides(defaults Limits, tenantLimits TenantLimits) (*Overrides, error) { - defaultLimits = &defaults return &Overrides{ tenantLimits: tenantLimits, defaultLimits: &defaults, diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index a543bf2b7f9..eb4bbdcf9e0 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -75,18 +75,14 @@ func TestOverridesManager_GetOverrides(t *testing.T) { } func TestLimitsLoadingFromYaml(t *testing.T) { - defaults := Limits{ + SetDefaultLimitsForYAMLUnmarshalling(Limits{ MaxLabelNameLength: 100, - } - - // we call this for its side-effect: setting of defaults - _, err := NewOverrides(defaults, nil) - require.NoError(t, err) + }) inp := `ingestion_rate: 0.5` l := Limits{} - err = yaml.Unmarshal([]byte(inp), &l) + err := yaml.Unmarshal([]byte(inp), &l) require.NoError(t, err) assert.Equal(t, 0.5, l.IngestionRate, "from yaml") From 0fa6cb2d87df9e6dd6ce8ef87b8560007f71c817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 5 Nov 2019 12:58:19 +0100 Subject: [PATCH 05/31] Pass multi-client context as argument. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/ring/kv/multi.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/ring/kv/multi.go b/pkg/ring/kv/multi.go index 2033a0d7804..6beb2e98d71 100644 --- a/pkg/ring/kv/multi.go +++ b/pkg/ring/kv/multi.go @@ -72,7 +72,6 @@ type MultiClient struct { // Primary client used for interaction. Values from this KV are copied to all the others. primaryID *atomic.Int32 - ctx context.Context cancel context.CancelFunc inProgressMu sync.Mutex @@ -96,10 +95,11 @@ func NewMultiClient(cfg MultiConfig, clients []kvclient) *MultiClient { mirroringEnabled: atomic.NewBool(cfg.MirrorEnabled), } - c.ctx, c.cancel = context.WithCancel(context.Background()) + ctx, cancelFn := context.WithCancel(context.Background()) + c.cancel = cancelFn // Start mirroring. - go c.watchChanges(cfg.MirrorPrefix) + go c.mirrorChanges(ctx, cfg.MirrorPrefix) if cfg.ConfigProvider != nil { go c.watchConfigChannel(cfg.ConfigProvider()) @@ -136,6 +136,7 @@ func (m *MultiClient) getPrimaryClient() (int, kvclient) { return int(v), m.clients[v] } +// returns true, if primary client has changed func (m *MultiClient) setNewPrimaryClient(store string) (bool, error) { newPrimaryIx := -1 for ix, c := range m.clients { @@ -258,16 +259,16 @@ func (m *MultiClient) WatchPrefix(ctx context.Context, prefix string, f func(str }) } -// watchChanges performs mirroring -- it watches changes in the primary client, and puts them into secondary. -func (m *MultiClient) watchChanges(prefix string) { - for m.ctx.Err() == nil { - err := m.runWithPrimaryClient(m.ctx, func(newCtx context.Context, primary kvclient) error { +// mirrorChanges performs mirroring -- it watches for changes in the primary client, and puts them into secondary. +func (m *MultiClient) mirrorChanges(ctx context.Context, prefix string) { + for ctx.Err() == nil { + err := m.runWithPrimaryClient(ctx, func(newCtx context.Context, primary kvclient) error { primary.client.WatchPrefix(newCtx, prefix, func(key string, val interface{}) bool { level.Debug(util.Logger).Log("msg", "value updated", "key", key, "store", primary.name) if m.mirroringEnabled.Load() { // don't pass new context to keyUpdated, we don't want to react on primary client changes here. - m.keyUpdated(primary, key, val) + m.keyUpdated(ctx, primary, key, val) } err := m.rateLimiter.Wait(newCtx) @@ -278,7 +279,7 @@ func (m *MultiClient) watchChanges(prefix string) { return true }) - return m.ctx.Err() + return ctx.Err() }) if err != nil { @@ -287,7 +288,7 @@ func (m *MultiClient) watchChanges(prefix string) { } } -func (m *MultiClient) keyUpdated(primary kvclient, key string, newValue interface{}) { +func (m *MultiClient) keyUpdated(ctx context.Context, primary kvclient, key string, newValue interface{}) { // let's propagate this to all remaining clients for _, kvc := range m.clients { if kvc == primary { @@ -295,7 +296,7 @@ func (m *MultiClient) keyUpdated(primary kvclient, key string, newValue interfac } stored := true - err := kvc.client.CAS(m.ctx, key, func(in interface{}) (out interface{}, retry bool, err error) { + err := kvc.client.CAS(ctx, key, func(in interface{}) (out interface{}, retry bool, err error) { stored = true if eq, ok := in.(withEqual); ok && eq.Equal(newValue) { From e2efe9e5e70134ed6dc63c63ebed474340869cb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 5 Nov 2019 13:01:31 +0100 Subject: [PATCH 06/31] watchConfigChannel now reacts on context being done as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/ring/kv/multi.go | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/pkg/ring/kv/multi.go b/pkg/ring/kv/multi.go index 6beb2e98d71..3827538fd85 100644 --- a/pkg/ring/kv/multi.go +++ b/pkg/ring/kv/multi.go @@ -102,31 +102,41 @@ func NewMultiClient(cfg MultiConfig, clients []kvclient) *MultiClient { go c.mirrorChanges(ctx, cfg.MirrorPrefix) if cfg.ConfigProvider != nil { - go c.watchConfigChannel(cfg.ConfigProvider()) + go c.watchConfigChannel(ctx, cfg.ConfigProvider()) } return c } -func (m *MultiClient) watchConfigChannel(configChannel <-chan MultiRuntimeConfig) { - for cfg := range configChannel { - if cfg.Mirroring != "" { - enable := cfg.Mirroring == "true" || cfg.Mirroring == "enable" || cfg.Mirroring == "enabled" - - old := m.mirroringEnabled.Swap(enable) - if old != enable { - level.Info(util.Logger).Log("msg", "toggled mirroring", "enabled", enable) +func (m *MultiClient) watchConfigChannel(ctx context.Context, configChannel <-chan MultiRuntimeConfig) { + for { + select { + case cfg, ok := <-configChannel: + if !ok { + return } - } - if cfg.PrimaryStore != "" { - switched, err := m.setNewPrimaryClient(cfg.PrimaryStore) - if switched { - level.Info(util.Logger).Log("msg", "switched primary KV store", "primary", cfg.PrimaryStore) + if cfg.Mirroring != "" { + enable := cfg.Mirroring == "true" || cfg.Mirroring == "enable" || cfg.Mirroring == "enabled" + + old := m.mirroringEnabled.Swap(enable) + if old != enable { + level.Info(util.Logger).Log("msg", "toggled mirroring", "enabled", enable) + } } - if err != nil { - level.Error(util.Logger).Log("msg", "failed to switch primary KV store", "primary", cfg.PrimaryStore, "err", err) + + if cfg.PrimaryStore != "" { + switched, err := m.setNewPrimaryClient(cfg.PrimaryStore) + if switched { + level.Info(util.Logger).Log("msg", "switched primary KV store", "primary", cfg.PrimaryStore) + } + if err != nil { + level.Error(util.Logger).Log("msg", "failed to switch primary KV store", "primary", cfg.PrimaryStore, "err", err) + } } + + case <-ctx.Done(): + return } } } From 56491b24a07fffc42f575be4ab168cb312362a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 5 Nov 2019 13:12:10 +0100 Subject: [PATCH 07/31] Changed Mirroring to *bool. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/ring/kv/multi.go | 11 ++++------- pkg/ring/kv/multi_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 pkg/ring/kv/multi_test.go diff --git a/pkg/ring/kv/multi.go b/pkg/ring/kv/multi.go index 3827538fd85..ef608f57c4c 100644 --- a/pkg/ring/kv/multi.go +++ b/pkg/ring/kv/multi.go @@ -43,10 +43,8 @@ type MultiRuntimeConfig struct { // or to gossip). Doing this allows nice migration between stores. Empty values are ignored. PrimaryStore string `yaml:"primary"` - // Put "enabled", "enable" or "true" here to enable. Any other value = disabled. Empty = ignore. - // We don't use bool here, because it would not be possible to distinguish between false = missing value, - // or false = disabled. - Mirroring string `yaml:"mirroring"` + // Mirroring enabled or not. Nil = no change. + Mirroring *bool `yaml:"mirroring"` } type kvclient struct { @@ -116,9 +114,8 @@ func (m *MultiClient) watchConfigChannel(ctx context.Context, configChannel <-ch return } - if cfg.Mirroring != "" { - enable := cfg.Mirroring == "true" || cfg.Mirroring == "enable" || cfg.Mirroring == "enabled" - + if cfg.Mirroring != nil { + enable := *cfg.Mirroring old := m.mirroringEnabled.Swap(enable) if old != enable { level.Info(util.Logger).Log("msg", "toggled mirroring", "enabled", enable) diff --git a/pkg/ring/kv/multi_test.go b/pkg/ring/kv/multi_test.go new file mode 100644 index 00000000000..b999cb1d152 --- /dev/null +++ b/pkg/ring/kv/multi_test.go @@ -0,0 +1,32 @@ +package kv + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v2" +) + +func boolPtr(b bool) *bool { + return &b +} + +func TestMultiRuntimeConfigWithVariousEnabledValues(t *testing.T) { + testcases := map[string]struct { + yaml string + expected *bool + }{ + "nil": {"primary: test", nil}, + "true": {"primary: test\nmirroring: true", boolPtr(true)}, + "false": {"mirroring: false", boolPtr(false)}, + } + + for name, tc := range testcases { + t.Run(name, func(t *testing.T) { + c := MultiRuntimeConfig{} + err := yaml.Unmarshal([]byte(tc.yaml), &c) + assert.NoError(t, err, tc.yaml) + assert.Equal(t, tc.expected, c.Mirroring, tc.yaml) + }) + } +} From 364bc9cad330aba952657cedfe59fdddca47efc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 5 Nov 2019 13:14:21 +0100 Subject: [PATCH 08/31] Ignore mock by yaml. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/ring/kv/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ring/kv/client.go b/pkg/ring/kv/client.go index 85fe4a26c77..b31bbc99832 100644 --- a/pkg/ring/kv/client.go +++ b/pkg/ring/kv/client.go @@ -37,7 +37,7 @@ type Config struct { Prefix string `yaml:"prefix,omitempty"` StoreConfig `yaml:",inline"` - Mock Client + Mock Client `yaml:"-"` } // RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet. From 2d4f1c02514ae1af10b123d68c3ff75bf041d5b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Wed, 6 Nov 2019 09:11:15 +0100 Subject: [PATCH 09/31] Renamed mirroring to mirror-enabled to be consistent with MultiConfig. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/ring/kv/multi.go | 2 +- pkg/ring/kv/multi_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/ring/kv/multi.go b/pkg/ring/kv/multi.go index ef608f57c4c..a99ada5352f 100644 --- a/pkg/ring/kv/multi.go +++ b/pkg/ring/kv/multi.go @@ -44,7 +44,7 @@ type MultiRuntimeConfig struct { PrimaryStore string `yaml:"primary"` // Mirroring enabled or not. Nil = no change. - Mirroring *bool `yaml:"mirroring"` + Mirroring *bool `yaml:"mirror-enabled"` } type kvclient struct { diff --git a/pkg/ring/kv/multi_test.go b/pkg/ring/kv/multi_test.go index b999cb1d152..bc7626cd629 100644 --- a/pkg/ring/kv/multi_test.go +++ b/pkg/ring/kv/multi_test.go @@ -17,8 +17,8 @@ func TestMultiRuntimeConfigWithVariousEnabledValues(t *testing.T) { expected *bool }{ "nil": {"primary: test", nil}, - "true": {"primary: test\nmirroring: true", boolPtr(true)}, - "false": {"mirroring: false", boolPtr(false)}, + "true": {"primary: test\nmirror-enabled: true", boolPtr(true)}, + "false": {"mirror-enabled: false", boolPtr(false)}, } for name, tc := range testcases { From 9665933f79980785dbd12fae4cca843541a59dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Wed, 6 Nov 2019 09:17:07 +0100 Subject: [PATCH 10/31] Renamed 'multi' to 'multi_kv_config' in overrides.yaml. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/cortex/runtime_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cortex/runtime_config.go b/pkg/cortex/runtime_config.go index 12ede69351c..3d92505dbb3 100644 --- a/pkg/cortex/runtime_config.go +++ b/pkg/cortex/runtime_config.go @@ -16,7 +16,7 @@ import ( type runtimeConfigValues struct { TenantLimits map[string]*validation.Limits `yaml:"overrides"` - Multi kv.MultiRuntimeConfig `yaml:"multi"` + Multi kv.MultiRuntimeConfig `yaml:"multi_kv_config"` } func loadRuntimeConfig(filename string) (interface{}, error) { From a3de2a81eed2eef201f91689da4159140f875762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Wed, 6 Nov 2019 11:24:18 +0100 Subject: [PATCH 11/31] Forward writes done via CAS function to secondary client. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirroring goroutine was removed, and replaced by forwarding writes done via CAS function to secondary client. Rate limits config was removed, but there is now timeout for secondary write, to avoid blocking CAS function for too long, if secondary write is slow (eg. etcd being down can cause very long writes). Only WatchKey and WatchPrefix functions now react on change of primary client. Signed-off-by: Peter Štibraný --- pkg/ring/kv/multi.go | 113 +++++++++++++++---------------------------- 1 file changed, 39 insertions(+), 74 deletions(-) diff --git a/pkg/ring/kv/multi.go b/pkg/ring/kv/multi.go index a99ada5352f..1d68fdff743 100644 --- a/pkg/ring/kv/multi.go +++ b/pkg/ring/kv/multi.go @@ -5,10 +5,10 @@ import ( "flag" "fmt" "sync" + "time" "github.com/go-kit/kit/log/level" "github.com/uber-go/atomic" - "golang.org/x/time/rate" "github.com/cortexproject/cortex/pkg/util" ) @@ -18,10 +18,8 @@ type MultiConfig struct { Primary string `yaml:"primary"` Secondary string `yaml:"secondary"` - MirrorRateLimit float64 `yaml:"mirror-rate-limit"` - MirrorRateBurst int `yaml:"mirror-rate-burst"` - MirrorEnabled bool `yaml:"mirror-enabled"` - MirrorPrefix string `yaml:"mirror-prefix"` + MirrorEnabled bool `yaml:"mirror-enabled"` + MirrorTimeout time.Duration `yaml:"mirror-timeout"` // ConfigProvider returns channel with MultiRuntimeConfig updates. ConfigProvider func() <-chan MultiRuntimeConfig @@ -31,10 +29,8 @@ type MultiConfig struct { func (cfg *MultiConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { f.StringVar(&cfg.Primary, prefix+"multi.primary", "", "Primary backend storage used by multi-client.") f.StringVar(&cfg.Secondary, prefix+"multi.secondary", "", "Secondary backend storage used by multi-client.") - f.StringVar(&cfg.MirrorPrefix, prefix+"multi.mirror-prefix", "", "Prefix for keys that should be mirrored") - f.BoolVar(&cfg.MirrorEnabled, prefix+"multi.mirror-enabled", false, "Mirror values to secondary store") - f.Float64Var(&cfg.MirrorRateLimit, prefix+"multi.mirror-rate-limit", 1, "Rate limit used for mirroring. 0 disables rate limit.") - f.IntVar(&cfg.MirrorRateBurst, prefix+"multi.mirror-burst-size", 1, "Burst size used by rate limiter. Values less than 1 are treated as 1.") + f.BoolVar(&cfg.MirrorEnabled, prefix+"multi.mirror-enabled", false, "Mirror writes to secondary store.") + f.DurationVar(&cfg.MirrorTimeout, prefix+"multi.mirror-timeout", 2*time.Second, "Timeout for storing value to secondary store.") } // MultiRuntimeConfig has values that can change in runtime (via overrides) @@ -58,16 +54,15 @@ type clientInProgress struct { } // MultiClient implements kv.Client by forwarding all API calls to primary client. -// At the same time, MultiClient watches for changes in values in the primary store, -// and forwards them to remaining clients. +// Writes performed via CAS method are also (optionally) forwarded to secondary clients. type MultiClient struct { // Available KV clients clients []kvclient - rateLimiter *rate.Limiter + mirrorTimeout time.Duration mirroringEnabled *atomic.Bool - // Primary client used for interaction. Values from this KV are copied to all the others. + // The primary client used for interaction. primaryID *atomic.Int32 cancel context.CancelFunc @@ -82,23 +77,19 @@ type MultiClient struct { // NewMultiClient creates new MultiClient with given KV Clients. // First client in the slice is the primary client. -// Channel is used to get notifications about what store to use as primary. func NewMultiClient(cfg MultiConfig, clients []kvclient) *MultiClient { c := &MultiClient{ clients: clients, primaryID: atomic.NewInt32(0), inProgress: map[int]clientInProgress{}, - rateLimiter: createRateLimiter(cfg.MirrorRateLimit, cfg.MirrorRateBurst), + mirrorTimeout: cfg.MirrorTimeout, mirroringEnabled: atomic.NewBool(cfg.MirrorEnabled), } ctx, cancelFn := context.WithCancel(context.Background()) c.cancel = cancelFn - // Start mirroring. - go c.mirrorChanges(ctx, cfg.MirrorPrefix) - if cfg.ConfigProvider != nil { go c.watchConfigChannel(ctx, cfg.ConfigProvider()) } @@ -232,25 +223,33 @@ func (m *MultiClient) runWithPrimaryClient(origCtx context.Context, fn func(newC } } -// Get is a part of kv.Client interface +// Get is a part of kv.Client interface. func (m *MultiClient) Get(ctx context.Context, key string) (interface{}, error) { - var result interface{} - err := m.runWithPrimaryClient(ctx, func(newCtx context.Context, primary kvclient) error { - var err2 error - result, err2 = primary.client.Get(newCtx, key) - return err2 - }) - return result, err + _, kv := m.getPrimaryClient() + val, err := kv.client.Get(ctx, key) + level.Info(util.Logger).Log("key", key, "value", val, "store", kv.name, "err", err) + return val, err } -// CAS is a part of kv.Client interface +// CAS is a part of kv.Client interface. func (m *MultiClient) CAS(ctx context.Context, key string, f func(in interface{}) (out interface{}, retry bool, err error)) error { - return m.runWithPrimaryClient(ctx, func(newCtx context.Context, primary kvclient) error { - return primary.client.CAS(newCtx, key, f) + _, kv := m.getPrimaryClient() + + updatedValue := interface{}(nil) + err := kv.client.CAS(ctx, key, func(in interface{}) (interface{}, bool, error) { + out, retry, err := f(in) + updatedValue = out + return out, retry, err }) + + if err == nil && updatedValue != nil && m.mirroringEnabled.Load() { + m.writeToSecondary(ctx, kv, key, updatedValue) + } + + return err } -// WatchKey is a part of kv.Client interface +// WatchKey is a part of kv.Client interface. func (m *MultiClient) WatchKey(ctx context.Context, key string, f func(interface{}) bool) { _ = m.runWithPrimaryClient(ctx, func(newCtx context.Context, primary kvclient) error { primary.client.WatchKey(newCtx, key, f) @@ -258,7 +257,7 @@ func (m *MultiClient) WatchKey(ctx context.Context, key string, f func(interface }) } -// WatchPrefix is a part of kv.Client interface +// WatchPrefix is a part of kv.Client interface. func (m *MultiClient) WatchPrefix(ctx context.Context, prefix string, f func(string, interface{}) bool) { _ = m.runWithPrimaryClient(ctx, func(newCtx context.Context, primary kvclient) error { primary.client.WatchPrefix(newCtx, prefix, f) @@ -266,43 +265,20 @@ func (m *MultiClient) WatchPrefix(ctx context.Context, prefix string, f func(str }) } -// mirrorChanges performs mirroring -- it watches for changes in the primary client, and puts them into secondary. -func (m *MultiClient) mirrorChanges(ctx context.Context, prefix string) { - for ctx.Err() == nil { - err := m.runWithPrimaryClient(ctx, func(newCtx context.Context, primary kvclient) error { - primary.client.WatchPrefix(newCtx, prefix, func(key string, val interface{}) bool { - level.Debug(util.Logger).Log("msg", "value updated", "key", key, "store", primary.name) - - if m.mirroringEnabled.Load() { - // don't pass new context to keyUpdated, we don't want to react on primary client changes here. - m.keyUpdated(ctx, primary, key, val) - } - - err := m.rateLimiter.Wait(newCtx) - if err != nil && err != context.Canceled { - level.Error(util.Logger).Log("msg", "error while rate limiting multi-client", "err", err) - } - - return true - }) - - return ctx.Err() - }) - - if err != nil { - level.Warn(util.Logger).Log("msg", "watching changes returned error", "err", err) - } +func (m *MultiClient) writeToSecondary(ctx context.Context, primary kvclient, key string, newValue interface{}) { + if m.mirrorTimeout > 0 { + var cfn context.CancelFunc + ctx, cfn = context.WithTimeout(ctx, m.mirrorTimeout) + defer cfn() } -} -func (m *MultiClient) keyUpdated(ctx context.Context, primary kvclient, key string, newValue interface{}) { - // let's propagate this to all remaining clients + // let's propagate new value to all remaining clients for _, kvc := range m.clients { if kvc == primary { continue } - stored := true + stored := false err := kvc.client.CAS(ctx, key, func(in interface{}) (out interface{}, retry bool, err error) { stored = true @@ -313,7 +289,7 @@ func (m *MultiClient) keyUpdated(ctx context.Context, primary kvclient, key stri } // try once - return in, false, nil + return newValue, false, nil }) if err != nil { @@ -324,7 +300,7 @@ func (m *MultiClient) keyUpdated(ctx context.Context, primary kvclient, key stri } } -// Stop the multiClient (mirroring part), and all configured clients. +// Stop the multiClient and all configured clients. func (m *MultiClient) Stop() { m.cancel() @@ -333,17 +309,6 @@ func (m *MultiClient) Stop() { } } -func createRateLimiter(rateLimit float64, burst int) *rate.Limiter { - if rateLimit <= 0 { - // burst is ignored when limit = rate.Inf - return rate.NewLimiter(rate.Inf, 0) - } - if burst < 1 { - burst = 1 - } - return rate.NewLimiter(rate.Limit(rateLimit), burst) -} - type withEqual interface { Equal(that interface{}) bool } From d91a5a726336c4fc6d2e44b8d454b005498c4b10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Wed, 6 Nov 2019 11:48:18 +0100 Subject: [PATCH 12/31] Added metrics to multi client. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/ring/kv/multi.go | 60 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/pkg/ring/kv/multi.go b/pkg/ring/kv/multi.go index 1d68fdff743..c3c0e653e56 100644 --- a/pkg/ring/kv/multi.go +++ b/pkg/ring/kv/multi.go @@ -7,12 +7,40 @@ import ( "sync" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/go-kit/kit/log/level" "github.com/uber-go/atomic" "github.com/cortexproject/cortex/pkg/util" ) +var ( + primaryStoreGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "consul_multikv_primary_store", + Help: "Selected primary KV store", + }, []string{"store"}) + + mirrorEnabled = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "consul_multikv_mirror_enabled", + Help: "Is mirroring to secondary store enabled", + }) + + mirrorWrites = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "consul_multikv_mirror_writes_total", + Help: "Number of mirror-writes to secondary store", + }) + + mirrorFailures = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "consul_multikv_mirror_write_errors_total", + Help: "Number of failures to mirror-write to secondary store", + }) +) + +func init() { + prometheus.MustRegister(primaryStoreGauge, mirrorEnabled, mirrorWrites, mirrorFailures) +} + // MultiConfig is a configuration for MultiClient. type MultiConfig struct { Primary string `yaml:"primary"` @@ -94,6 +122,8 @@ func NewMultiClient(cfg MultiConfig, clients []kvclient) *MultiClient { go c.watchConfigChannel(ctx, cfg.ConfigProvider()) } + c.updatePrimaryStoreGauge() + c.updateMirrorEnabledGauge() return c } @@ -111,6 +141,7 @@ func (m *MultiClient) watchConfigChannel(ctx context.Context, configChannel <-ch if old != enable { level.Info(util.Logger).Log("msg", "toggled mirroring", "enabled", enable) } + m.updateMirrorEnabledGauge() } if cfg.PrimaryStore != "" { @@ -153,6 +184,8 @@ func (m *MultiClient) setNewPrimaryClient(store string) (bool, error) { return false, nil } + defer m.updatePrimaryStoreGauge() // do as the last thing, after releasing the lock + // switching to new primary... cancel clients using previous one m.inProgressMu.Lock() defer m.inProgressMu.Unlock() @@ -165,6 +198,27 @@ func (m *MultiClient) setNewPrimaryClient(store string) (bool, error) { return true, nil } +func (m *MultiClient) updatePrimaryStoreGauge() { + _, pkv := m.getPrimaryClient() + + for _, kv := range m.clients { + value := float64(0) + if pkv == kv { + value = 1 + } + + primaryStoreGauge.WithLabelValues(kv.name).Set(value) + } +} + +func (m *MultiClient) updateMirrorEnabledGauge() { + if m.mirroringEnabled.Load() { + mirrorEnabled.Set(1) + } else { + mirrorEnabled.Set(0) + } +} + func (m *MultiClient) registerCancelFn(clientID int, fn context.CancelFunc) int { m.inProgressMu.Lock() defer m.inProgressMu.Unlock() @@ -278,6 +332,7 @@ func (m *MultiClient) writeToSecondary(ctx context.Context, primary kvclient, ke continue } + mirrorWrites.Inc() stored := false err := kvc.client.CAS(ctx, key, func(in interface{}) (out interface{}, retry bool, err error) { stored = true @@ -293,9 +348,10 @@ func (m *MultiClient) writeToSecondary(ctx context.Context, primary kvclient, ke }) if err != nil { - level.Error(util.Logger).Log("msg", "failed to update value in secondary store", "key", key, "err", err, "primary", primary.name, "secondary", kvc.name) + mirrorFailures.Inc() + level.Warn(util.Logger).Log("msg", "failed to update value in secondary store", "key", key, "err", err, "primary", primary.name, "secondary", kvc.name) } else if stored { - level.Info(util.Logger).Log("msg", "stored updated value to secondary store", "key", key, "primary", primary.name, "secondary", kvc.name) + level.Debug(util.Logger).Log("msg", "stored updated value to secondary store", "key", key, "primary", primary.name, "secondary", kvc.name) } } } From 43ec6061cbb08a8690b60612658549759573f497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Wed, 6 Nov 2019 11:50:30 +0100 Subject: [PATCH 13/31] Removed equality check when writing to secondary store. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without watch-and-mirror functionality, there is no need to check if value is already present in the secondary store. Signed-off-by: Peter Štibraný --- pkg/ring/kv/multi.go | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/pkg/ring/kv/multi.go b/pkg/ring/kv/multi.go index c3c0e653e56..2799d97413e 100644 --- a/pkg/ring/kv/multi.go +++ b/pkg/ring/kv/multi.go @@ -333,16 +333,7 @@ func (m *MultiClient) writeToSecondary(ctx context.Context, primary kvclient, ke } mirrorWrites.Inc() - stored := false err := kvc.client.CAS(ctx, key, func(in interface{}) (out interface{}, retry bool, err error) { - stored = true - - if eq, ok := in.(withEqual); ok && eq.Equal(newValue) { - stored = false - level.Debug(util.Logger).Log("msg", "no change", "key", key) - return nil, false, nil - } - // try once return newValue, false, nil }) @@ -350,7 +341,7 @@ func (m *MultiClient) writeToSecondary(ctx context.Context, primary kvclient, ke if err != nil { mirrorFailures.Inc() level.Warn(util.Logger).Log("msg", "failed to update value in secondary store", "key", key, "err", err, "primary", primary.name, "secondary", kvc.name) - } else if stored { + } else { level.Debug(util.Logger).Log("msg", "stored updated value to secondary store", "key", key, "primary", primary.name, "secondary", kvc.name) } } @@ -364,7 +355,3 @@ func (m *MultiClient) Stop() { kv.client.Stop() } } - -type withEqual interface { - Equal(that interface{}) bool -} From 9bbea9b6dcc385cb970e90eb964c1feceb8daa05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Mon, 18 Nov 2019 11:25:18 +0100 Subject: [PATCH 14/31] Renamed OverridesManager and moved it to its own package. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/cortex/cortex.go | 19 +-- pkg/cortex/modules.go | 17 +-- pkg/cortex/runtime_config.go | 12 +- pkg/util/override.go | 134 ----------------- pkg/util/runtime_config/manager.go | 141 ++++++++++++++++++ .../manager_test.go} | 32 ++-- 6 files changed, 177 insertions(+), 178 deletions(-) delete mode 100644 pkg/util/override.go create mode 100644 pkg/util/runtime_config/manager.go rename pkg/util/{override_test.go => runtime_config/manager_test.go} (77%) diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index 37a122837b3..c825197b048 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -4,7 +4,6 @@ import ( "flag" "fmt" "os" - "time" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" @@ -32,6 +31,7 @@ import ( "github.com/cortexproject/cortex/pkg/ruler" "github.com/cortexproject/cortex/pkg/storage/tsdb" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/runtime_config" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -76,13 +76,11 @@ type Config struct { Encoding encoding.Config `yaml:"-"` // No yaml for this, it only works with flags. TSDB tsdb.Config `yaml:"tsdb"` - Ruler ruler.Config `yaml:"ruler,omitempty"` - ConfigDB db.Config `yaml:"configdb,omitempty"` - ConfigStore config_client.Config `yaml:"config_store,omitempty"` - Alertmanager alertmanager.MultitenantAlertmanagerConfig `yaml:"alertmanager,omitempty"` - - RuntimeConfigFile string `yaml:"runtime_config_file"` - RuntimeConfigLoadPeriod time.Duration `yaml:"runtime_config_load_period"` + Ruler ruler.Config `yaml:"ruler,omitempty"` + ConfigDB db.Config `yaml:"configdb,omitempty"` + ConfigStore config_client.Config `yaml:"config_store,omitempty"` + Alertmanager alertmanager.MultitenantAlertmanagerConfig `yaml:"alertmanager,omitempty"` + RuntimeConfig runtime_config.ManagerConfig `yaml:"runtime_config"` } // RegisterFlags registers flag. @@ -94,8 +92,6 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&c.AuthEnabled, "auth.enabled", true, "Set to false to disable auth.") f.BoolVar(&c.PrintConfig, "print.config", false, "Print the config and exit.") f.StringVar(&c.HTTPPrefix, "http.prefix", "/api/prom", "HTTP path prefix for Cortex API.") - f.StringVar(&c.RuntimeConfigFile, "runtime-config.file", "", "File with configuration that can be updated in runtime.") - f.DurationVar(&c.RuntimeConfigLoadPeriod, "runtime-config.reload-period", 10*time.Second, "How often to check runtime config file.") c.Server.RegisterFlags(f) c.Distributor.RegisterFlags(f) @@ -118,6 +114,7 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { c.ConfigDB.RegisterFlags(f) c.ConfigStore.RegisterFlagsWithPrefix("alertmanager.", f) c.Alertmanager.RegisterFlags(f) + c.RuntimeConfig.RegisterFlags(f) // These don't seem to have a home. flag.IntVar(&chunk_util.QueryParallelism, "querier.query-parallelism", 100, "Max subqueries run in parallel per higher-level query.") @@ -162,7 +159,7 @@ type Cortex struct { frontend *frontend.Frontend tableManager *chunk.TableManager cache cache.Cache - runtimeConfig *util.OverridesManager + runtimeConfig *runtime_config.Manager ruler *ruler.Ruler configAPI *api.API diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 20494e68d02..00ea948533b 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -32,6 +32,7 @@ import ( "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ruler" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/runtime_config" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -166,19 +167,13 @@ func (t *Cortex) initRing(cfg *Config) (err error) { } func (t *Cortex) initRuntimeConfig(cfg *Config) (err error) { - configFile := cfg.RuntimeConfigFile - reloadPeriod := cfg.RuntimeConfigLoadPeriod - if configFile == "" { - configFile = cfg.LimitsConfig.PerTenantOverrideConfig - reloadPeriod = cfg.LimitsConfig.PerTenantOverridePeriod + if cfg.RuntimeConfig.LoadPath == "" { + cfg.RuntimeConfig.LoadPath = cfg.LimitsConfig.PerTenantOverrideConfig + cfg.RuntimeConfig.ReloadPeriod = cfg.LimitsConfig.PerTenantOverridePeriod } + cfg.RuntimeConfig.Loader = loadRuntimeConfig - c := util.OverridesManagerConfig{ - OverridesReloadPeriod: reloadPeriod, - OverridesLoadPath: configFile, - OverridesLoader: loadRuntimeConfig, - } - t.runtimeConfig, err = util.NewOverridesManager(c) + t.runtimeConfig, err = runtime_config.NewRuntimeConfigManager(cfg.RuntimeConfig) return err } diff --git a/pkg/cortex/runtime_config.go b/pkg/cortex/runtime_config.go index 3d92505dbb3..36fc5fa1280 100644 --- a/pkg/cortex/runtime_config.go +++ b/pkg/cortex/runtime_config.go @@ -6,12 +6,12 @@ import ( "gopkg.in/yaml.v2" "github.com/cortexproject/cortex/pkg/ring/kv" - "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/runtime_config" "github.com/cortexproject/cortex/pkg/util/validation" ) // runtimeConfigValues are values that can be reloaded from configuration file while Cortex is running. -// Reloading is done by OverridesManager, which also keeps currently loaded config. +// Reloading is done by runtime_config.Manager, which also keeps the currently loaded config. // These values are then pushed to the components that are interested in them. type runtimeConfigValues struct { TenantLimits map[string]*validation.Limits `yaml:"overrides"` @@ -36,9 +36,9 @@ func loadRuntimeConfig(filename string) (interface{}, error) { return overrides, nil } -func tenantLimitsFromRuntimeConfig(c *util.OverridesManager) validation.TenantLimits { +func tenantLimitsFromRuntimeConfig(c *runtime_config.Manager) validation.TenantLimits { return func(userID string) *validation.Limits { - cfg, ok := c.GetOverrides().(*runtimeConfigValues) + cfg, ok := c.GetConfig().(*runtimeConfigValues) if !ok || cfg == nil { return nil } @@ -47,7 +47,7 @@ func tenantLimitsFromRuntimeConfig(c *util.OverridesManager) validation.TenantLi } } -func multiClientRuntimeConfigChannel(manager *util.OverridesManager) func() <-chan kv.MultiRuntimeConfig { +func multiClientRuntimeConfigChannel(manager *runtime_config.Manager) func() <-chan kv.MultiRuntimeConfig { // returns function that can be used in MultiConfig.ConfigProvider return func() <-chan kv.MultiRuntimeConfig { ch := make(chan kv.MultiRuntimeConfig, 1) @@ -62,7 +62,7 @@ func multiClientRuntimeConfigChannel(manager *util.OverridesManager) func() <-ch } // push initial config to the channel - listener(manager.GetOverrides()) + listener(manager.GetConfig()) manager.AddListener(listener) return ch } diff --git a/pkg/util/override.go b/pkg/util/override.go deleted file mode 100644 index 95ecceab3da..00000000000 --- a/pkg/util/override.go +++ /dev/null @@ -1,134 +0,0 @@ -package util - -import ( - "sync" - "time" - - "github.com/go-kit/kit/log/level" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -var overridesReloadSuccess = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "cortex_overrides_last_reload_successful", - Help: "Whether the last overrides reload attempt was successful.", -}) - -// OverridesLoader loads the configuration from file. -type OverridesLoader func(filename string) (interface{}, error) - -// OverridesListener gets notified when new overrides is loaded -type OverridesListener func(newConfig interface{}) - -// OverridesManagerConfig holds the config for an OverridesManager instance. -// It holds config related to loading per-tenant overrides. -type OverridesManagerConfig struct { - OverridesReloadPeriod time.Duration - OverridesLoadPath string - OverridesLoader OverridesLoader -} - -// OverridesManager periodically reloads configuration from a file, and keeps this -// configuration available for clients. -type OverridesManager struct { - cfg OverridesManagerConfig - quit chan struct{} - - listenersMtx sync.Mutex - listeners []OverridesListener - - overridesMtx sync.RWMutex - overrides interface{} -} - -// NewOverridesManager creates an instance of OverridesManager and starts reload overrides loop based on config -func NewOverridesManager(cfg OverridesManagerConfig) (*OverridesManager, error) { - overridesManager := OverridesManager{ - cfg: cfg, - quit: make(chan struct{}), - } - - if cfg.OverridesLoadPath != "" { - if err := overridesManager.loadOverrides(); err != nil { - // Log but don't stop on error - we don't want to halt all ingesters because of a typo - level.Error(Logger).Log("msg", "failed to load overrides", "err", err) - } - go overridesManager.loop() - } else { - level.Info(Logger).Log("msg", "overrides disabled") - } - - return &overridesManager, nil -} - -// AddListener registers new listener function, that will receive updates configuration. -// Listener is called asynchronously to avoid blocking main reloading loop. -func (om *OverridesManager) AddListener(l OverridesListener) { - if l == nil { - panic("nil listener") - } - - om.listenersMtx.Lock() - defer om.listenersMtx.Unlock() - - om.listeners = append(om.listeners, l) -} - -func (om *OverridesManager) loop() { - ticker := time.NewTicker(om.cfg.OverridesReloadPeriod) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - err := om.loadOverrides() - if err != nil { - // Log but don't stop on error - we don't want to halt all ingesters because of a typo - level.Error(Logger).Log("msg", "failed to load overrides", "err", err) - } - case <-om.quit: - return - } - } -} - -func (om *OverridesManager) loadOverrides() error { - overrides, err := om.cfg.OverridesLoader(om.cfg.OverridesLoadPath) - if err != nil { - overridesReloadSuccess.Set(0) - return err - } - overridesReloadSuccess.Set(1) - - om.setOverrides(overrides) - om.callListeners(overrides) - - return nil -} - -func (om *OverridesManager) setOverrides(overrides interface{}) { - om.overridesMtx.Lock() - defer om.overridesMtx.Unlock() - om.overrides = overrides -} - -func (om *OverridesManager) callListeners(newValue interface{}) { - om.listenersMtx.Lock() - defer om.listenersMtx.Unlock() - for _, l := range om.listeners { - go l(newValue) - } -} - -// Stop stops the OverridesManager -func (om *OverridesManager) Stop() { - close(om.quit) -} - -// GetOverrides returns last loaded overrides value, possibly nil. -func (om *OverridesManager) GetOverrides() interface{} { - om.overridesMtx.RLock() - defer om.overridesMtx.RUnlock() - - return om.overrides -} diff --git a/pkg/util/runtime_config/manager.go b/pkg/util/runtime_config/manager.go new file mode 100644 index 00000000000..5570403ab95 --- /dev/null +++ b/pkg/util/runtime_config/manager.go @@ -0,0 +1,141 @@ +package runtime_config + +import ( + "flag" + "sync" + "time" + + "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log/level" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var overridesReloadSuccess = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "cortex_overrides_last_reload_successful", + Help: "Whether the last config reload attempt was successful.", +}) + +// RuntimeConfigLoader loads the configuration from file. +type RuntimeConfigLoader func(filename string) (interface{}, error) + +// RuntimeConfigListener gets notified when new config is loaded +type RuntimeConfigListener func(newConfig interface{}) + +// ManagerConfig holds the config for an Manager instance. +// It holds config related to loading per-tenant config. +type ManagerConfig struct { + ReloadPeriod time.Duration `yaml:"period"` + LoadPath string `yaml:"file"` + Loader RuntimeConfigLoader +} + +func (omc *ManagerConfig) RegisterFlags(f *flag.FlagSet) { + f.StringVar(&omc.LoadPath, "runtime-config.file", "", "File with the configuration that can be updated in runtime.") + f.DurationVar(&omc.ReloadPeriod, "runtime-config.reload-period", 10*time.Second, "How often to check runtime config file.") +} + +// Manager periodically reloads the configuration from a file, and keeps this +// configuration available for clients. +type Manager struct { + cfg ManagerConfig + quit chan struct{} + + listenersMtx sync.Mutex + listeners []RuntimeConfigListener + + configMtx sync.RWMutex + config interface{} +} + +// NewRuntimeConfigManager creates an instance of Manager and starts reload config loop based on config +func NewRuntimeConfigManager(cfg ManagerConfig) (*Manager, error) { + mgr := Manager{ + cfg: cfg, + quit: make(chan struct{}), + } + + if cfg.LoadPath != "" { + if err := mgr.loadConfig(); err != nil { + // Log but don't stop on error - we don't want to halt all ingesters because of a typo + level.Error(util.Logger).Log("msg", "failed to load config", "err", err) + } + go mgr.loop() + } else { + level.Info(util.Logger).Log("msg", "config disabled") + } + + return &mgr, nil +} + +// AddListener registers new listener function, that will receive updates configuration. +// Listener is called asynchronously to avoid blocking main reloading loop. +func (om *Manager) AddListener(l RuntimeConfigListener) { + if l == nil { + panic("nil listener") + } + + om.listenersMtx.Lock() + defer om.listenersMtx.Unlock() + + om.listeners = append(om.listeners, l) +} + +func (om *Manager) loop() { + ticker := time.NewTicker(om.cfg.ReloadPeriod) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + err := om.loadConfig() + if err != nil { + // Log but don't stop on error - we don't want to halt all ingesters because of a typo + level.Error(util.Logger).Log("msg", "failed to load config", "err", err) + } + case <-om.quit: + return + } + } +} + +func (om *Manager) loadConfig() error { + cfg, err := om.cfg.Loader(om.cfg.LoadPath) + if err != nil { + overridesReloadSuccess.Set(0) + return err + } + overridesReloadSuccess.Set(1) + + om.setConfig(cfg) + om.callListeners(cfg) + + return nil +} + +func (om *Manager) setConfig(config interface{}) { + om.configMtx.Lock() + defer om.configMtx.Unlock() + om.config = config +} + +func (om *Manager) callListeners(newValue interface{}) { + om.listenersMtx.Lock() + defer om.listenersMtx.Unlock() + for _, l := range om.listeners { + go l(newValue) + } +} + +// Stop stops the Manager +func (om *Manager) Stop() { + close(om.quit) +} + +// GetConfig returns last loaded config value, possibly nil. +func (om *Manager) GetConfig() interface{} { + om.configMtx.RLock() + defer om.configMtx.RUnlock() + + return om.config +} diff --git a/pkg/util/override_test.go b/pkg/util/runtime_config/manager_test.go similarity index 77% rename from pkg/util/override_test.go rename to pkg/util/runtime_config/manager_test.go index 2c33c7b9864..6e5146a68af 100644 --- a/pkg/util/override_test.go +++ b/pkg/util/runtime_config/manager_test.go @@ -1,4 +1,4 @@ -package util +package runtime_config import ( "io/ioutil" @@ -66,21 +66,21 @@ func TestNewOverridesManager(t *testing.T) { defaultTestLimits = &TestLimits{Limit1: 100} - // testing NewOverridesManager with overrides reload config set - overridesManagerConfig := OverridesManagerConfig{ - OverridesReloadPeriod: time.Second, - OverridesLoadPath: tempFile.Name(), - OverridesLoader: testLoadOverrides, + // testing NewRuntimeConfigManager with overrides reload config set + overridesManagerConfig := ManagerConfig{ + ReloadPeriod: time.Second, + LoadPath: tempFile.Name(), + Loader: testLoadOverrides, } - overridesManager, err := NewOverridesManager(overridesManagerConfig) + overridesManager, err := NewRuntimeConfigManager(overridesManagerConfig) require.NoError(t, err) // Cleaning up overridesManager.Stop() // Make sure test limits were loaded. - require.NotNil(t, overridesManager.GetOverrides()) + require.NotNil(t, overridesManager.GetConfig()) } func TestOverridesManager_Listener(t *testing.T) { @@ -100,14 +100,14 @@ func TestOverridesManager_Listener(t *testing.T) { defaultTestLimits = &TestLimits{Limit1: 100} - // testing NewOverridesManager with overrides reload config set - overridesManagerConfig := OverridesManagerConfig{ - OverridesReloadPeriod: time.Second, - OverridesLoadPath: tempFile.Name(), - OverridesLoader: testLoadOverrides, + // testing NewRuntimeConfigManager with overrides reload config set + overridesManagerConfig := ManagerConfig{ + ReloadPeriod: time.Second, + LoadPath: tempFile.Name(), + Loader: testLoadOverrides, } - overridesManager, err := NewOverridesManager(overridesManagerConfig) + overridesManager, err := NewRuntimeConfigManager(overridesManagerConfig) require.NoError(t, err) // listeners are called asynchronously @@ -123,7 +123,7 @@ func TestOverridesManager_Listener(t *testing.T) { require.NoError(t, err) // reload - err = overridesManager.loadOverrides() + err = overridesManager.loadConfig() require.NoError(t, err) var newValue interface{} @@ -142,5 +142,5 @@ func TestOverridesManager_Listener(t *testing.T) { overridesManager.Stop() // Make sure test limits were loaded. - require.NotNil(t, overridesManager.GetOverrides()) + require.NotNil(t, overridesManager.GetConfig()) } From 8c2f557927969b05a1e69a62ebe383e71d03fef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 26 Nov 2019 09:53:00 +0100 Subject: [PATCH 15/31] Make lint happy, 3. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/cortex/cortex.go | 6 +++--- pkg/cortex/modules.go | 4 ++-- pkg/cortex/runtime_config.go | 6 +++--- .../{runtime_config => runtimeconfig}/manager.go | 16 ++++++++-------- .../manager_test.go | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) rename pkg/util/{runtime_config => runtimeconfig}/manager.go (89%) rename pkg/util/{runtime_config => runtimeconfig}/manager_test.go (99%) diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index c825197b048..5b2bb26480a 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -31,7 +31,7 @@ import ( "github.com/cortexproject/cortex/pkg/ruler" "github.com/cortexproject/cortex/pkg/storage/tsdb" "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/runtime_config" + "github.com/cortexproject/cortex/pkg/util/runtimeconfig" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -80,7 +80,7 @@ type Config struct { ConfigDB db.Config `yaml:"configdb,omitempty"` ConfigStore config_client.Config `yaml:"config_store,omitempty"` Alertmanager alertmanager.MultitenantAlertmanagerConfig `yaml:"alertmanager,omitempty"` - RuntimeConfig runtime_config.ManagerConfig `yaml:"runtime_config"` + RuntimeConfig runtimeconfig.ManagerConfig `yaml:"runtime_config"` } // RegisterFlags registers flag. @@ -159,7 +159,7 @@ type Cortex struct { frontend *frontend.Frontend tableManager *chunk.TableManager cache cache.Cache - runtimeConfig *runtime_config.Manager + runtimeConfig *runtimeconfig.Manager ruler *ruler.Ruler configAPI *api.API diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 00ea948533b..77a70a89974 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -32,7 +32,7 @@ import ( "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ruler" "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/runtime_config" + "github.com/cortexproject/cortex/pkg/util/runtimeconfig" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -173,7 +173,7 @@ func (t *Cortex) initRuntimeConfig(cfg *Config) (err error) { } cfg.RuntimeConfig.Loader = loadRuntimeConfig - t.runtimeConfig, err = runtime_config.NewRuntimeConfigManager(cfg.RuntimeConfig) + t.runtimeConfig, err = runtimeconfig.NewRuntimeConfigManager(cfg.RuntimeConfig) return err } diff --git a/pkg/cortex/runtime_config.go b/pkg/cortex/runtime_config.go index 36fc5fa1280..14164ed5c55 100644 --- a/pkg/cortex/runtime_config.go +++ b/pkg/cortex/runtime_config.go @@ -6,7 +6,7 @@ import ( "gopkg.in/yaml.v2" "github.com/cortexproject/cortex/pkg/ring/kv" - "github.com/cortexproject/cortex/pkg/util/runtime_config" + "github.com/cortexproject/cortex/pkg/util/runtimeconfig" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -36,7 +36,7 @@ func loadRuntimeConfig(filename string) (interface{}, error) { return overrides, nil } -func tenantLimitsFromRuntimeConfig(c *runtime_config.Manager) validation.TenantLimits { +func tenantLimitsFromRuntimeConfig(c *runtimeconfig.Manager) validation.TenantLimits { return func(userID string) *validation.Limits { cfg, ok := c.GetConfig().(*runtimeConfigValues) if !ok || cfg == nil { @@ -47,7 +47,7 @@ func tenantLimitsFromRuntimeConfig(c *runtime_config.Manager) validation.TenantL } } -func multiClientRuntimeConfigChannel(manager *runtime_config.Manager) func() <-chan kv.MultiRuntimeConfig { +func multiClientRuntimeConfigChannel(manager *runtimeconfig.Manager) func() <-chan kv.MultiRuntimeConfig { // returns function that can be used in MultiConfig.ConfigProvider return func() <-chan kv.MultiRuntimeConfig { ch := make(chan kv.MultiRuntimeConfig, 1) diff --git a/pkg/util/runtime_config/manager.go b/pkg/util/runtimeconfig/manager.go similarity index 89% rename from pkg/util/runtime_config/manager.go rename to pkg/util/runtimeconfig/manager.go index 5570403ab95..eca03e33ea8 100644 --- a/pkg/util/runtime_config/manager.go +++ b/pkg/util/runtimeconfig/manager.go @@ -1,4 +1,4 @@ -package runtime_config +package runtimeconfig import ( "flag" @@ -16,18 +16,18 @@ var overridesReloadSuccess = promauto.NewGauge(prometheus.GaugeOpts{ Help: "Whether the last config reload attempt was successful.", }) -// RuntimeConfigLoader loads the configuration from file. -type RuntimeConfigLoader func(filename string) (interface{}, error) +// Loader loads the configuration from file. +type Loader func(filename string) (interface{}, error) -// RuntimeConfigListener gets notified when new config is loaded -type RuntimeConfigListener func(newConfig interface{}) +// Listener gets notified when new config is loaded +type Listener func(newConfig interface{}) // ManagerConfig holds the config for an Manager instance. // It holds config related to loading per-tenant config. type ManagerConfig struct { ReloadPeriod time.Duration `yaml:"period"` LoadPath string `yaml:"file"` - Loader RuntimeConfigLoader + Loader Loader } func (omc *ManagerConfig) RegisterFlags(f *flag.FlagSet) { @@ -42,7 +42,7 @@ type Manager struct { quit chan struct{} listenersMtx sync.Mutex - listeners []RuntimeConfigListener + listeners []Listener configMtx sync.RWMutex config interface{} @@ -70,7 +70,7 @@ func NewRuntimeConfigManager(cfg ManagerConfig) (*Manager, error) { // AddListener registers new listener function, that will receive updates configuration. // Listener is called asynchronously to avoid blocking main reloading loop. -func (om *Manager) AddListener(l RuntimeConfigListener) { +func (om *Manager) AddListener(l Listener) { if l == nil { panic("nil listener") } diff --git a/pkg/util/runtime_config/manager_test.go b/pkg/util/runtimeconfig/manager_test.go similarity index 99% rename from pkg/util/runtime_config/manager_test.go rename to pkg/util/runtimeconfig/manager_test.go index 6e5146a68af..902418e9b27 100644 --- a/pkg/util/runtime_config/manager_test.go +++ b/pkg/util/runtimeconfig/manager_test.go @@ -1,4 +1,4 @@ -package runtime_config +package runtimeconfig import ( "io/ioutil" From d7f7f9282add1776e9ede7e4a9d5284440ba7346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 26 Nov 2019 10:00:58 +0100 Subject: [PATCH 16/31] Make lint happy, 4. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/util/runtimeconfig/manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/util/runtimeconfig/manager.go b/pkg/util/runtimeconfig/manager.go index eca03e33ea8..bff74d47036 100644 --- a/pkg/util/runtimeconfig/manager.go +++ b/pkg/util/runtimeconfig/manager.go @@ -30,6 +30,7 @@ type ManagerConfig struct { Loader Loader } +// RegisterFlags registers flags. func (omc *ManagerConfig) RegisterFlags(f *flag.FlagSet) { f.StringVar(&omc.LoadPath, "runtime-config.file", "", "File with the configuration that can be updated in runtime.") f.DurationVar(&omc.ReloadPeriod, "runtime-config.reload-period", 10*time.Second, "How often to check runtime config file.") From df3c6e0429082b2ff909f4c5b3a56e99ead13486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 26 Nov 2019 10:29:25 +0100 Subject: [PATCH 17/31] Add metric type to variable names, yaml name changes, fixed metric names, removed forgotten log. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/ring/kv/multi.go | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/pkg/ring/kv/multi.go b/pkg/ring/kv/multi.go index 2799d97413e..6d32a5d468b 100644 --- a/pkg/ring/kv/multi.go +++ b/pkg/ring/kv/multi.go @@ -17,28 +17,28 @@ import ( var ( primaryStoreGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "consul_multikv_primary_store", + Name: "cortex_multikv_primary_store", Help: "Selected primary KV store", }, []string{"store"}) - mirrorEnabled = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "consul_multikv_mirror_enabled", + mirrorEnabledGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "cortex_multikv_mirror_enabled", Help: "Is mirroring to secondary store enabled", }) - mirrorWrites = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "consul_multikv_mirror_writes_total", + mirrorWritesCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "cortex_multikv_mirror_writes_total", Help: "Number of mirror-writes to secondary store", }) - mirrorFailures = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "consul_multikv_mirror_write_errors_total", + mirrorFailuresCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "cortex_multikv_mirror_write_errors_total", Help: "Number of failures to mirror-write to secondary store", }) ) func init() { - prometheus.MustRegister(primaryStoreGauge, mirrorEnabled, mirrorWrites, mirrorFailures) + prometheus.MustRegister(primaryStoreGauge, mirrorEnabledGauge, mirrorWritesCounter, mirrorFailuresCounter) } // MultiConfig is a configuration for MultiClient. @@ -46,8 +46,8 @@ type MultiConfig struct { Primary string `yaml:"primary"` Secondary string `yaml:"secondary"` - MirrorEnabled bool `yaml:"mirror-enabled"` - MirrorTimeout time.Duration `yaml:"mirror-timeout"` + MirrorEnabled bool `yaml:"mirror_enabled"` + MirrorTimeout time.Duration `yaml:"mirror_timeout"` // ConfigProvider returns channel with MultiRuntimeConfig updates. ConfigProvider func() <-chan MultiRuntimeConfig @@ -136,10 +136,10 @@ func (m *MultiClient) watchConfigChannel(ctx context.Context, configChannel <-ch } if cfg.Mirroring != nil { - enable := *cfg.Mirroring - old := m.mirroringEnabled.Swap(enable) - if old != enable { - level.Info(util.Logger).Log("msg", "toggled mirroring", "enabled", enable) + enabled := *cfg.Mirroring + old := m.mirroringEnabled.Swap(enabled) + if old != enabled { + level.Info(util.Logger).Log("msg", "toggled mirroring", "enabled", enabled) } m.updateMirrorEnabledGauge() } @@ -213,9 +213,9 @@ func (m *MultiClient) updatePrimaryStoreGauge() { func (m *MultiClient) updateMirrorEnabledGauge() { if m.mirroringEnabled.Load() { - mirrorEnabled.Set(1) + mirrorEnabledGauge.Set(1) } else { - mirrorEnabled.Set(0) + mirrorEnabledGauge.Set(0) } } @@ -281,7 +281,6 @@ func (m *MultiClient) runWithPrimaryClient(origCtx context.Context, fn func(newC func (m *MultiClient) Get(ctx context.Context, key string) (interface{}, error) { _, kv := m.getPrimaryClient() val, err := kv.client.Get(ctx, key) - level.Info(util.Logger).Log("key", key, "value", val, "store", kv.name, "err", err) return val, err } @@ -332,14 +331,14 @@ func (m *MultiClient) writeToSecondary(ctx context.Context, primary kvclient, ke continue } - mirrorWrites.Inc() + mirrorWritesCounter.Inc() err := kvc.client.CAS(ctx, key, func(in interface{}) (out interface{}, retry bool, err error) { // try once return newValue, false, nil }) if err != nil { - mirrorFailures.Inc() + mirrorFailuresCounter.Inc() level.Warn(util.Logger).Log("msg", "failed to update value in secondary store", "key", key, "err", err, "primary", primary.name, "secondary", kvc.name) } else { level.Debug(util.Logger).Log("msg", "stored updated value to secondary store", "key", key, "primary", primary.name, "secondary", kvc.name) From 103dbaf2ba0f72f23c6caf3d3b1575b0272615ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 26 Nov 2019 10:30:55 +0100 Subject: [PATCH 18/31] Fixed yet one more yaml name. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/ring/kv/multi.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ring/kv/multi.go b/pkg/ring/kv/multi.go index 6d32a5d468b..9e7eea4bbf0 100644 --- a/pkg/ring/kv/multi.go +++ b/pkg/ring/kv/multi.go @@ -68,7 +68,7 @@ type MultiRuntimeConfig struct { PrimaryStore string `yaml:"primary"` // Mirroring enabled or not. Nil = no change. - Mirroring *bool `yaml:"mirror-enabled"` + Mirroring *bool `yaml:"mirror_enabled"` } type kvclient struct { From 7de6ad87f28af45535a572ea04d7ec904e2bfee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 26 Nov 2019 10:55:20 +0100 Subject: [PATCH 19/31] Fixed tests after changing yaml fields. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/ring/kv/multi_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ring/kv/multi_test.go b/pkg/ring/kv/multi_test.go index bc7626cd629..9385ed5ff8e 100644 --- a/pkg/ring/kv/multi_test.go +++ b/pkg/ring/kv/multi_test.go @@ -17,8 +17,8 @@ func TestMultiRuntimeConfigWithVariousEnabledValues(t *testing.T) { expected *bool }{ "nil": {"primary: test", nil}, - "true": {"primary: test\nmirror-enabled: true", boolPtr(true)}, - "false": {"mirror-enabled: false", boolPtr(false)}, + "true": {"primary: test\nmirror_enabled: true", boolPtr(true)}, + "false": {"mirror_enabled: false", boolPtr(false)}, } for name, tc := range testcases { From 63d566f1708716cfb869aeb721082a124c4c402a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Thu, 28 Nov 2019 09:36:25 +0100 Subject: [PATCH 20/31] Fix bug when default limits are not applied until next overrides reload. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/cortex/modules.go | 5 +++++ pkg/util/runtimeconfig/manager.go | 8 +++++--- pkg/util/runtimeconfig/manager_test.go | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 77a70a89974..ec6224e6447 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -184,6 +184,11 @@ func (t *Cortex) stopRuntimeConfig() (err error) { func (t *Cortex) initOverrides(cfg *Config) (err error) { validation.SetDefaultLimitsForYAMLUnmarshalling(cfg.LimitsConfig) + // By this time, config has already been loaded from YAML file, but without default limits. + // We don't want to wait until next reload to default limits to be used, so we ask runtime + // config manager to reload configuration again. Errors are ignored (already logged by previous + // load, if there was any) + _ = t.runtimeConfig.LoadConfig() t.overrides, err = validation.NewOverrides(cfg.LimitsConfig, tenantLimitsFromRuntimeConfig(t.runtimeConfig)) return err } diff --git a/pkg/util/runtimeconfig/manager.go b/pkg/util/runtimeconfig/manager.go index bff74d47036..4ca2795962c 100644 --- a/pkg/util/runtimeconfig/manager.go +++ b/pkg/util/runtimeconfig/manager.go @@ -57,7 +57,7 @@ func NewRuntimeConfigManager(cfg ManagerConfig) (*Manager, error) { } if cfg.LoadPath != "" { - if err := mgr.loadConfig(); err != nil { + if err := mgr.LoadConfig(); err != nil { // Log but don't stop on error - we don't want to halt all ingesters because of a typo level.Error(util.Logger).Log("msg", "failed to load config", "err", err) } @@ -89,7 +89,7 @@ func (om *Manager) loop() { for { select { case <-ticker.C: - err := om.loadConfig() + err := om.LoadConfig() if err != nil { // Log but don't stop on error - we don't want to halt all ingesters because of a typo level.Error(util.Logger).Log("msg", "failed to load config", "err", err) @@ -100,7 +100,9 @@ func (om *Manager) loop() { } } -func (om *Manager) loadConfig() error { +// LoadConfig loads configuration using the loader function, and if successful, +// stores it as current configuration and notifies listeners. +func (om *Manager) LoadConfig() error { cfg, err := om.cfg.Loader(om.cfg.LoadPath) if err != nil { overridesReloadSuccess.Set(0) diff --git a/pkg/util/runtimeconfig/manager_test.go b/pkg/util/runtimeconfig/manager_test.go index 902418e9b27..af69e923a79 100644 --- a/pkg/util/runtimeconfig/manager_test.go +++ b/pkg/util/runtimeconfig/manager_test.go @@ -123,7 +123,7 @@ func TestOverridesManager_Listener(t *testing.T) { require.NoError(t, err) // reload - err = overridesManager.loadConfig() + err = overridesManager.LoadConfig() require.NoError(t, err) var newValue interface{} From 666e17ec2b28fcdb643c725d95bcc830c340f652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Thu, 28 Nov 2019 10:18:50 +0100 Subject: [PATCH 21/31] Ignore LoadConfig if LoadPath is empty. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/util/runtimeconfig/manager.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/util/runtimeconfig/manager.go b/pkg/util/runtimeconfig/manager.go index 4ca2795962c..1b188266ce8 100644 --- a/pkg/util/runtimeconfig/manager.go +++ b/pkg/util/runtimeconfig/manager.go @@ -103,6 +103,10 @@ func (om *Manager) loop() { // LoadConfig loads configuration using the loader function, and if successful, // stores it as current configuration and notifies listeners. func (om *Manager) LoadConfig() error { + if om.cfg.LoadPath == "" { + return nil + } + cfg, err := om.cfg.Loader(om.cfg.LoadPath) if err != nil { overridesReloadSuccess.Set(0) From c83361bcf5e9f1c19701481c886fc5bcfb9f4f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Thu, 28 Nov 2019 10:35:23 +0100 Subject: [PATCH 22/31] Use channels to communicate config updates. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of spawning new goroutine for each config update, we now use channels to communicate config updates. Signed-off-by: Peter Štibraný --- pkg/cortex/runtime_config.go | 27 ++++---- pkg/util/runtimeconfig/manager.go | 54 ++++++++++++---- pkg/util/runtimeconfig/manager_test.go | 88 ++++++++++++++++++++++++-- 3 files changed, 138 insertions(+), 31 deletions(-) diff --git a/pkg/cortex/runtime_config.go b/pkg/cortex/runtime_config.go index 14164ed5c55..8f914839720 100644 --- a/pkg/cortex/runtime_config.go +++ b/pkg/cortex/runtime_config.go @@ -50,20 +50,23 @@ func tenantLimitsFromRuntimeConfig(c *runtimeconfig.Manager) validation.TenantLi func multiClientRuntimeConfigChannel(manager *runtimeconfig.Manager) func() <-chan kv.MultiRuntimeConfig { // returns function that can be used in MultiConfig.ConfigProvider return func() <-chan kv.MultiRuntimeConfig { - ch := make(chan kv.MultiRuntimeConfig, 1) + outCh := make(chan kv.MultiRuntimeConfig, 1) - listener := func(newOverrides interface{}) { - cfg, ok := newOverrides.(*runtimeConfigValues) - if !ok || cfg == nil { - return - } - - ch <- cfg.Multi + // push initial config to the channel + val := manager.GetConfig() + if cfg, ok := val.(*runtimeConfigValues); ok && cfg != nil { + outCh <- cfg.Multi } - // push initial config to the channel - listener(manager.GetConfig()) - manager.AddListener(listener) - return ch + ch := manager.CreateListenerChannel(1) + go func() { + for val := range ch { + if cfg, ok := val.(*runtimeConfigValues); ok && cfg != nil { + outCh <- cfg.Multi + } + } + }() + + return outCh } } diff --git a/pkg/util/runtimeconfig/manager.go b/pkg/util/runtimeconfig/manager.go index 1b188266ce8..3e2e48ee6ca 100644 --- a/pkg/util/runtimeconfig/manager.go +++ b/pkg/util/runtimeconfig/manager.go @@ -19,9 +19,6 @@ var overridesReloadSuccess = promauto.NewGauge(prometheus.GaugeOpts{ // Loader loads the configuration from file. type Loader func(filename string) (interface{}, error) -// Listener gets notified when new config is loaded -type Listener func(newConfig interface{}) - // ManagerConfig holds the config for an Manager instance. // It holds config related to loading per-tenant config. type ManagerConfig struct { @@ -43,7 +40,7 @@ type Manager struct { quit chan struct{} listenersMtx sync.Mutex - listeners []Listener + listeners []chan interface{} configMtx sync.RWMutex config interface{} @@ -69,17 +66,34 @@ func NewRuntimeConfigManager(cfg ManagerConfig) (*Manager, error) { return &mgr, nil } -// AddListener registers new listener function, that will receive updates configuration. -// Listener is called asynchronously to avoid blocking main reloading loop. -func (om *Manager) AddListener(l Listener) { - if l == nil { - panic("nil listener") - } +// CreateListenerChannel creates new channel that can be used to receive new config values. +// If there is no receiver waiting for value when config manager tries to send the update, +// or channel buffer is full, update is discarded. +// +// When config manager is stopped, it closes all channels to notify receivers that they will +// not receive any more updates. +func (om *Manager) CreateListenerChannel(buffer int) <-chan interface{} { + ch := make(chan interface{}, buffer) om.listenersMtx.Lock() defer om.listenersMtx.Unlock() - om.listeners = append(om.listeners, l) + om.listeners = append(om.listeners, ch) + return ch +} + +// CloseListenerChannel removes given channel from list of channels to send notifications to and closes channel. +func (om *Manager) CloseListenerChannel(listener <-chan interface{}) { + om.listenersMtx.Lock() + defer om.listenersMtx.Unlock() + + for ix, ch := range om.listeners { + if ch == listener { + om.listeners = append(om.listeners[:ix], om.listeners[ix+1:]...) + close(ch) + break + } + } } func (om *Manager) loop() { @@ -129,14 +143,28 @@ func (om *Manager) setConfig(config interface{}) { func (om *Manager) callListeners(newValue interface{}) { om.listenersMtx.Lock() defer om.listenersMtx.Unlock() - for _, l := range om.listeners { - go l(newValue) + + for _, ch := range om.listeners { + select { + case ch <- newValue: + // ok + default: + // nobody is listening or buffer full. + } } } // Stop stops the Manager func (om *Manager) Stop() { close(om.quit) + + om.listenersMtx.Lock() + defer om.listenersMtx.Unlock() + + for _, ch := range om.listeners { + close(ch) + } + om.listeners = nil } // GetConfig returns last loaded config value, possibly nil. diff --git a/pkg/util/runtimeconfig/manager_test.go b/pkg/util/runtimeconfig/manager_test.go index af69e923a79..f01102fa5dd 100644 --- a/pkg/util/runtimeconfig/manager_test.go +++ b/pkg/util/runtimeconfig/manager_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/stretchr/testify/require" + "github.com/uber-go/atomic" "gopkg.in/yaml.v2" ) @@ -83,7 +84,7 @@ func TestNewOverridesManager(t *testing.T) { require.NotNil(t, overridesManager.GetConfig()) } -func TestOverridesManager_Listener(t *testing.T) { +func TestOverridesManager_ListenerWithDefaultLimits(t *testing.T) { tempFile, err := ioutil.TempFile("", "test-validation") require.NoError(t, err) require.NoError(t, tempFile.Close()) @@ -110,11 +111,8 @@ func TestOverridesManager_Listener(t *testing.T) { overridesManager, err := NewRuntimeConfigManager(overridesManagerConfig) require.NoError(t, err) - // listeners are called asynchronously - ch := make(chan interface{}) - overridesManager.AddListener(func(newConfig interface{}) { - ch <- newConfig - }) + // need to use buffer, otherwise LoadConfig will throw away update + ch := overridesManager.CreateListenerChannel(1) // rewrite file err = ioutil.WriteFile(tempFile.Name(), []byte(`overrides: @@ -144,3 +142,81 @@ func TestOverridesManager_Listener(t *testing.T) { // Make sure test limits were loaded. require.NotNil(t, overridesManager.GetConfig()) } + +func TestOverridesManager_ListenerChannel(t *testing.T) { + var config = atomic.NewInt32(555) + + // testing NewRuntimeConfigManager with overrides reload config set + overridesManagerConfig := ManagerConfig{ + ReloadPeriod: 5 * time.Second, + LoadPath: "ignored", + Loader: func(filename string) (i interface{}, err error) { + val := int(config.Load()) + return val, nil + }, + } + + overridesManager, err := NewRuntimeConfigManager(overridesManagerConfig) + require.NoError(t, err) + + // need to use buffer, otherwise LoadConfig will throw away update + ch := overridesManager.CreateListenerChannel(1) + + err = overridesManager.LoadConfig() + require.NoError(t, err) + + select { + case newValue := <-ch: + require.Equal(t, 555, newValue) + case <-time.After(time.Second): + t.Fatal("listener was not called") + } + + config.Store(1111) + err = overridesManager.LoadConfig() + require.NoError(t, err) + + select { + case newValue := <-ch: + require.Equal(t, 1111, newValue) + case <-time.After(time.Second): + t.Fatal("listener was not called") + } + + overridesManager.CloseListenerChannel(ch) + select { + case _, ok := <-ch: + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("channel not closed") + } +} + +func TestOverridesManager_StopClosesListenerChannels(t *testing.T) { + var config = atomic.NewInt32(555) + + // testing NewRuntimeConfigManager with overrides reload config set + overridesManagerConfig := ManagerConfig{ + ReloadPeriod: 5 * time.Second, + LoadPath: "ignored", + Loader: func(filename string) (i interface{}, err error) { + val := int(config.Load()) + return val, nil + }, + } + + overridesManager, err := NewRuntimeConfigManager(overridesManagerConfig) + require.NoError(t, err) + + // need to use buffer, otherwise LoadConfig will throw away update + ch := overridesManager.CreateListenerChannel(0) + + overridesManager.Stop() + + select { + case _, ok := <-ch: + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("channel not closed") + } +} From 5714d21e64b3b9dbfb25b264ae094f5fdb086027 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Thu, 28 Nov 2019 10:52:26 +0100 Subject: [PATCH 23/31] Initialize limits before starting runtimeconfig Manager. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/cortex/modules.go | 9 +++------ pkg/util/runtimeconfig/manager.go | 12 ++++-------- pkg/util/runtimeconfig/manager_test.go | 12 ++++++------ 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index ec6224e6447..ccf2735838e 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -173,6 +173,9 @@ func (t *Cortex) initRuntimeConfig(cfg *Config) (err error) { } cfg.RuntimeConfig.Loader = loadRuntimeConfig + // make sure to set default limits before we start loading configuration into memory + validation.SetDefaultLimitsForYAMLUnmarshalling(cfg.LimitsConfig) + t.runtimeConfig, err = runtimeconfig.NewRuntimeConfigManager(cfg.RuntimeConfig) return err } @@ -183,12 +186,6 @@ func (t *Cortex) stopRuntimeConfig() (err error) { } func (t *Cortex) initOverrides(cfg *Config) (err error) { - validation.SetDefaultLimitsForYAMLUnmarshalling(cfg.LimitsConfig) - // By this time, config has already been loaded from YAML file, but without default limits. - // We don't want to wait until next reload to default limits to be used, so we ask runtime - // config manager to reload configuration again. Errors are ignored (already logged by previous - // load, if there was any) - _ = t.runtimeConfig.LoadConfig() t.overrides, err = validation.NewOverrides(cfg.LimitsConfig, tenantLimitsFromRuntimeConfig(t.runtimeConfig)) return err } diff --git a/pkg/util/runtimeconfig/manager.go b/pkg/util/runtimeconfig/manager.go index 3e2e48ee6ca..a0c872bcc8d 100644 --- a/pkg/util/runtimeconfig/manager.go +++ b/pkg/util/runtimeconfig/manager.go @@ -54,7 +54,7 @@ func NewRuntimeConfigManager(cfg ManagerConfig) (*Manager, error) { } if cfg.LoadPath != "" { - if err := mgr.LoadConfig(); err != nil { + if err := mgr.loadConfig(); err != nil { // Log but don't stop on error - we don't want to halt all ingesters because of a typo level.Error(util.Logger).Log("msg", "failed to load config", "err", err) } @@ -103,7 +103,7 @@ func (om *Manager) loop() { for { select { case <-ticker.C: - err := om.LoadConfig() + err := om.loadConfig() if err != nil { // Log but don't stop on error - we don't want to halt all ingesters because of a typo level.Error(util.Logger).Log("msg", "failed to load config", "err", err) @@ -114,13 +114,9 @@ func (om *Manager) loop() { } } -// LoadConfig loads configuration using the loader function, and if successful, +// loadConfig loads configuration using the loader function, and if successful, // stores it as current configuration and notifies listeners. -func (om *Manager) LoadConfig() error { - if om.cfg.LoadPath == "" { - return nil - } - +func (om *Manager) loadConfig() error { cfg, err := om.cfg.Loader(om.cfg.LoadPath) if err != nil { overridesReloadSuccess.Set(0) diff --git a/pkg/util/runtimeconfig/manager_test.go b/pkg/util/runtimeconfig/manager_test.go index f01102fa5dd..c4f590d6d50 100644 --- a/pkg/util/runtimeconfig/manager_test.go +++ b/pkg/util/runtimeconfig/manager_test.go @@ -111,7 +111,7 @@ func TestOverridesManager_ListenerWithDefaultLimits(t *testing.T) { overridesManager, err := NewRuntimeConfigManager(overridesManagerConfig) require.NoError(t, err) - // need to use buffer, otherwise LoadConfig will throw away update + // need to use buffer, otherwise loadConfig will throw away update ch := overridesManager.CreateListenerChannel(1) // rewrite file @@ -121,7 +121,7 @@ func TestOverridesManager_ListenerWithDefaultLimits(t *testing.T) { require.NoError(t, err) // reload - err = overridesManager.LoadConfig() + err = overridesManager.loadConfig() require.NoError(t, err) var newValue interface{} @@ -159,10 +159,10 @@ func TestOverridesManager_ListenerChannel(t *testing.T) { overridesManager, err := NewRuntimeConfigManager(overridesManagerConfig) require.NoError(t, err) - // need to use buffer, otherwise LoadConfig will throw away update + // need to use buffer, otherwise loadConfig will throw away update ch := overridesManager.CreateListenerChannel(1) - err = overridesManager.LoadConfig() + err = overridesManager.loadConfig() require.NoError(t, err) select { @@ -173,7 +173,7 @@ func TestOverridesManager_ListenerChannel(t *testing.T) { } config.Store(1111) - err = overridesManager.LoadConfig() + err = overridesManager.loadConfig() require.NoError(t, err) select { @@ -208,7 +208,7 @@ func TestOverridesManager_StopClosesListenerChannels(t *testing.T) { overridesManager, err := NewRuntimeConfigManager(overridesManagerConfig) require.NoError(t, err) - // need to use buffer, otherwise LoadConfig will throw away update + // need to use buffer, otherwise loadConfig will throw away update ch := overridesManager.CreateListenerChannel(0) overridesManager.Stop() From ecfd16d0bc6b4cae141f4f2c0df3ad80efe5f4a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Thu, 5 Dec 2019 09:32:50 +0100 Subject: [PATCH 24/31] Updated CHANGELOG.md and arguments.md. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- CHANGELOG.md | 2 ++ docs/configuration/arguments.md | 60 ++++++++++++++++++++++++++++--- pkg/cortex/cortex.go | 2 +- pkg/util/runtimeconfig/manager.go | 6 ++-- 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc58a87b22f..84467d7c45c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,11 +12,13 @@ * [CHANGE] Use relative links from /ring page to make it work when used behind reverse proxy. #1896 * [CHANGE] Deprecated `-distributor.limiter-reload-period` flag. #1766 * [CHANGE] Ingesters now write only normalised tokens to the ring, although they can still read denormalised tokens used by other ingesters. `-ingester.normalise-tokens` is now deprecated, and ignored. If you want to switch back to using denormalised tokens, you need to downgrade to Cortex 0.4.0. Previous versions don't handle claiming tokens from normalised ingesters correctly. #1809 +* [CHANGE] Overrides mechanism has been renamed to "runtime config", and is now separate from limits. Runtime config is simply a file that is reloaded by Cortex every couple of seconds. Limits and now also multi KV use this mechanism.
New arguments were introduced: `-runtime-config.file` (defaults to empty) and `-runtime-config.reload-period` (defaults to 10 seconds), which replace previously used `-limits.per-user-override-config` and `-limits.per-user-override-period` options. Old options are still used if `-runtime-config.file` is not specified. This change is also reflected in YAML configuration, where old `limits.per_tenant_override_config` and `limits.per_tenant_override_period` fields are replaced with `runtime_config.file` and `runtime_config.period` respectively. #1749 * [FEATURE] The distributor can now drop labels from samples (similar to the removal of the replica label for HA ingestion) per user via the `distributor.drop-label` flag. #1726 * [FEATURE] Added `global` ingestion rate limiter strategy. Deprecated `-distributor.limiter-reload-period` flag. #1766 * [FEATURE] Added support for Microsoft Azure blob storage to be used for storing chunk data. #1913 * [FEATURE] Added readiness probe endpoint`/ready` to queriers. #1934 * [FEATURE] EXPERIMENTAL: Added `/series` API endpoint support with TSDB blocks storage. #1830 +* [FEATURE] Added "multi" KV store that can interact with two other KV stores, primary one for all reads and writes, and secondary one, which only receives writes. Primary/secondary store can be modified in runtime via runtime-config mechanism (previously "overrides"). #1749 * [ENHANCEMENT] Added `password` and `enable_tls` options to redis cache configuration. Enables usage of Microsoft Azure Cache for Redis service. * [BUGFIX] Fixed unnecessary CAS operations done by the HA tracker when the jitter is enabled. #1861 * [BUGFIX] Fixed #1904 ingesters getting stuck in a LEAVING state after coming up from an ungraceful exit. #1921 diff --git a/docs/configuration/arguments.md b/docs/configuration/arguments.md index 3defae19b22..1d561271105 100644 --- a/docs/configuration/arguments.md +++ b/docs/configuration/arguments.md @@ -122,7 +122,7 @@ The KVStore client is used by both the Ring and HA Tracker. - `{ring,distributor.ha-tracker}.prefix` The prefix for the keys in the store. Should end with a /. For example with a prefix of foo/, the key bar would be stored under foo/bar. - `{ring,distributor.ha-tracker}.store` - Backend storage to use for the ring (consul, etcd, inmemory). + Backend storage to use for the ring (consul, etcd, inmemory, memberlist, multi). #### Consul @@ -182,6 +182,32 @@ Flags for configuring KV store based on memberlist library. This feature is expe Timeout for writing 'packet' data. - `memberlist.transport-debug` Log debug transport messages. Note: global log.level must be at debug level as well. + +#### Multi KV + +This is a special key-value implementation that uses two other KV stores (eg. consul, etcd or memberlist). One of them is always marked as primary, and all reads and writes go to primary store. Other one, secondary, is only used for writes. The idea is that operator can use multi KV store to migrate from primary to secondary store in runtime. For example, migration from Consul to Memberlist would look like this: + +- Set `ring.store` to use `multi` store. Set `-multi.primary=consul` and `-multi.secondary=memberlist`. All consul and memberlist settings must still be specified. +- Start all Cortex microservices. They will still use Consul as primary KV, but they will also write share ring via memberlist. +- Operator can now use "runtime config" mechanism to switch primary store to memberlist. +- After all Cortex microservices have picked up new primary store, and everything looks correct, operator can now shut down Consul, and modify Cortex configuration to use `-ring.store=memberlist` only. + +Multi KV has following parameters: + +- `multi.primary` - name of primary KV store. Same values as in `ring.store` are supported, except `multi`. +- `multi.secondary` - name of secondary KV store. +- `multi.mirror-enabled` - enable mirroring of values to secondary store, defaults to true +- `multi.mirror-timeout` - wait max this time to write to secondary store to finish. Default to 2 seconds. Errors writing to secondary store are not reported to caller, but are logged to the log file and also reported via `cortex_multikv_mirror_write_errors_total` metric. + +Multi KV also reacts on changes done via runtime configuration. It uses this section: + +```yaml +multi_kv_config: + mirror-enabled: false + primary: memberlist +``` + +Note that runtime configuration values take precedence over command line options. ### HA Tracker @@ -276,11 +302,13 @@ It also talks to a KVStore and has it's own copies of the same flags used by the Where you don't want to cache every chunk written by ingesters, but you do want to take advantage of chunk write deduplication, this option will make ingesters write a placeholder to the cache for each chunk. Make sure you configure ingesters with a different cache to queriers, which need the whole value. -## Ingester, Distributor & Querier limits. +## Runtime Configuration file + +Cortex has a concept of "runtime config" file, which is simply a file that is realoded while Cortex is running. It is used by some Cortex components to allow operator to change some aspects of Cortex configuration without restarting it. File is specified by using `-runtime-config.file=` flag and reload period (which defaults to 10 seconds) can be changed by `-runtime-config.reload-period=` flag. Previously this mechanism was only used by limits overrides, and flags were called `-limits.per-user-override-config=` and `-limits.per-user-override-period=10s` respectively. These are still used, if `-runtime-config.file=` is not specified. -Cortex implements various limits on the requests it can process, in order to prevent a single tenant overwhelming the cluster. There are various default global limits which apply to all tenants which can be set on the command line. These limits can also be overridden on a per-tenant basis, using a configuration file. Specify the filename for the override configuration file using the `-limits.per-user-override-config=` flag. The override file will be re-read every 10 seconds by default - this can also be controlled using the `-limits.per-user-override-period=10s` flag. +At the moment, two components use runtime configuration: limits and multi KV store. -The override file should be in YAML format and contain a single `overrides` field, which itself is a map of tenant ID (same values as passed in the `X-Scope-OrgID` header) to the various limits. An example `overrides.yml` could look like: +Example runtime configuration file: ```yaml overrides: @@ -292,11 +320,33 @@ overrides: max_samples_per_query: 1000000 max_series_per_metric: 100000 max_series_per_query: 100000 + +multi_kv_config: + mirror-enabled: false + primary: memberlist ``` When running Cortex on Kubernetes, store this file in a config map and mount it in each services' containers. When changing the values there is no need to restart the services, unless otherwise specified. -Valid fields are (with their corresponding flags for default values): +## Ingester, Distributor & Querier limits. + +Cortex implements various limits on the requests it can process, in order to prevent a single tenant overwhelming the cluster. There are various default global limits which apply to all tenants which can be set on the command line. These limits can also be overridden on a per-tenant basis by using `overrides` field of runtime configuration file. + +The `overrides` field is a map of tenant ID (same values as passed in the `X-Scope-OrgID` header) to the various limits. An example could look like: + +```yaml +overrides: + tenant1: + ingestion_rate: 10000 + max_series_per_metric: 100000 + max_series_per_query: 100000 + tenant2: + max_samples_per_query: 1000000 + max_series_per_metric: 100000 + max_series_per_query: 100000 +``` + +Valid per-tenant limits are (with their corresponding flags for default values): - `ingestion_rate_strategy` / `-distributor.ingestion-rate-limit-strategy` - `ingestion_rate` / `-distributor.ingestion-rate-limit` diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index 5b2bb26480a..87d6f321b1d 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -80,7 +80,7 @@ type Config struct { ConfigDB db.Config `yaml:"configdb,omitempty"` ConfigStore config_client.Config `yaml:"config_store,omitempty"` Alertmanager alertmanager.MultitenantAlertmanagerConfig `yaml:"alertmanager,omitempty"` - RuntimeConfig runtimeconfig.ManagerConfig `yaml:"runtime_config"` + RuntimeConfig runtimeconfig.ManagerConfig `yaml:"runtime_config,omitempty"` } // RegisterFlags registers flag. diff --git a/pkg/util/runtimeconfig/manager.go b/pkg/util/runtimeconfig/manager.go index a0c872bcc8d..78fc5af0744 100644 --- a/pkg/util/runtimeconfig/manager.go +++ b/pkg/util/runtimeconfig/manager.go @@ -28,9 +28,9 @@ type ManagerConfig struct { } // RegisterFlags registers flags. -func (omc *ManagerConfig) RegisterFlags(f *flag.FlagSet) { - f.StringVar(&omc.LoadPath, "runtime-config.file", "", "File with the configuration that can be updated in runtime.") - f.DurationVar(&omc.ReloadPeriod, "runtime-config.reload-period", 10*time.Second, "How often to check runtime config file.") +func (mc *ManagerConfig) RegisterFlags(f *flag.FlagSet) { + f.StringVar(&mc.LoadPath, "runtime-config.file", "", "File with the configuration that can be updated in runtime.") + f.DurationVar(&mc.ReloadPeriod, "runtime-config.reload-period", 10*time.Second, "How often to check runtime config file.") } // Manager periodically reloads the configuration from a file, and keeps this From e5ba1db5d5c564b7afbba28c9735e7ef29c39b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Thu, 5 Dec 2019 09:55:10 +0100 Subject: [PATCH 25/31] Typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- docs/configuration/arguments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/arguments.md b/docs/configuration/arguments.md index 1d561271105..0631da0813d 100644 --- a/docs/configuration/arguments.md +++ b/docs/configuration/arguments.md @@ -304,7 +304,7 @@ It also talks to a KVStore and has it's own copies of the same flags used by the ## Runtime Configuration file -Cortex has a concept of "runtime config" file, which is simply a file that is realoded while Cortex is running. It is used by some Cortex components to allow operator to change some aspects of Cortex configuration without restarting it. File is specified by using `-runtime-config.file=` flag and reload period (which defaults to 10 seconds) can be changed by `-runtime-config.reload-period=` flag. Previously this mechanism was only used by limits overrides, and flags were called `-limits.per-user-override-config=` and `-limits.per-user-override-period=10s` respectively. These are still used, if `-runtime-config.file=` is not specified. +Cortex has a concept of "runtime config" file, which is simply a file that is reloaded while Cortex is running. It is used by some Cortex components to allow operator to change some aspects of Cortex configuration without restarting it. File is specified by using `-runtime-config.file=` flag and reload period (which defaults to 10 seconds) can be changed by `-runtime-config.reload-period=` flag. Previously this mechanism was only used by limits overrides, and flags were called `-limits.per-user-override-config=` and `-limits.per-user-override-period=10s` respectively. These are still used, if `-runtime-config.file=` is not specified. At the moment, two components use runtime configuration: limits and multi KV store. From 7245d4df423535ff174c2ee3fa351039467ac48b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Thu, 5 Dec 2019 10:22:42 +0100 Subject: [PATCH 26/31] Fix compilation error in ingester_v2_test.go. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/ingester/ingester_v2_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ingester/ingester_v2_test.go b/pkg/ingester/ingester_v2_test.go index b62ee10d580..1c04a5811fa 100644 --- a/pkg/ingester/ingester_v2_test.go +++ b/pkg/ingester/ingester_v2_test.go @@ -715,7 +715,7 @@ func newIngesterMockWithTSDBStorage(ingesterCfg Config, registerer prometheus.Re clientCfg := defaultClientTestConfig() limits := defaultLimitsTestConfig() - overrides, err := validation.NewOverrides(limits) + overrides, err := validation.NewOverrides(limits, nil) if err != nil { return nil, nil, err } From b16a010502fce987c03a07ea5cee41d7c94af94d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Fri, 13 Dec 2019 16:35:52 +0100 Subject: [PATCH 27/31] Fixed error after rebase. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/util/validation/limits.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 64278882cae..bae57b70660 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -158,8 +158,7 @@ func (o *Overrides) IngestionRate(userID string) float64 { // to each distributor instance (local) or evenly shared across the cluster (global). func (o *Overrides) IngestionRateStrategy() string { // The ingestion rate strategy can't be overridden on a per-tenant basis - defaultLimits := o.overridesManager.cfg.Defaults - return defaultLimits.(*Limits).IngestionRateStrategy + return o.defaultLimits.IngestionRateStrategy } // IngestionBurstSize returns the burst size for ingestion rate. From 45237f8e703c2b709ce6b8cab8afc44ab9b5725c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Fri, 13 Dec 2019 20:24:10 +0100 Subject: [PATCH 28/31] Fixed error after rebase. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/distributor/ingestion_rate_strategy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/distributor/ingestion_rate_strategy_test.go b/pkg/distributor/ingestion_rate_strategy_test.go index 6004279a305..14bb7445061 100644 --- a/pkg/distributor/ingestion_rate_strategy_test.go +++ b/pkg/distributor/ingestion_rate_strategy_test.go @@ -61,7 +61,7 @@ func TestIngestionRateStrategy(t *testing.T) { var strategy limiter.RateLimiterStrategy // Init limits overrides - overrides, err := validation.NewOverrides(testData.limits) + overrides, err := validation.NewOverrides(testData.limits, nil) require.NoError(t, err) // Instance the strategy From ff24854f2157b2dfdbdc701f61ae0271bf5a1443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 7 Jan 2020 10:46:52 +0100 Subject: [PATCH 29/31] Use logger with component="multikv" to log messages. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/ring/kv/multi.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/pkg/ring/kv/multi.go b/pkg/ring/kv/multi.go index 9e7eea4bbf0..8831540cd0f 100644 --- a/pkg/ring/kv/multi.go +++ b/pkg/ring/kv/multi.go @@ -7,12 +7,12 @@ import ( "sync" "time" + "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/go-kit/kit/log/level" "github.com/uber-go/atomic" - - "github.com/cortexproject/cortex/pkg/util" ) var ( @@ -90,6 +90,9 @@ type MultiClient struct { mirrorTimeout time.Duration mirroringEnabled *atomic.Bool + // logger with "multikv" component + logger log.Logger + // The primary client used for interaction. primaryID *atomic.Int32 @@ -113,6 +116,8 @@ func NewMultiClient(cfg MultiConfig, clients []kvclient) *MultiClient { mirrorTimeout: cfg.MirrorTimeout, mirroringEnabled: atomic.NewBool(cfg.MirrorEnabled), + + logger: log.With(util.Logger, "component", "multikv"), } ctx, cancelFn := context.WithCancel(context.Background()) @@ -139,7 +144,7 @@ func (m *MultiClient) watchConfigChannel(ctx context.Context, configChannel <-ch enabled := *cfg.Mirroring old := m.mirroringEnabled.Swap(enabled) if old != enabled { - level.Info(util.Logger).Log("msg", "toggled mirroring", "enabled", enabled) + level.Info(m.logger).Log("msg", "toggled mirroring", "enabled", enabled) } m.updateMirrorEnabledGauge() } @@ -147,10 +152,10 @@ func (m *MultiClient) watchConfigChannel(ctx context.Context, configChannel <-ch if cfg.PrimaryStore != "" { switched, err := m.setNewPrimaryClient(cfg.PrimaryStore) if switched { - level.Info(util.Logger).Log("msg", "switched primary KV store", "primary", cfg.PrimaryStore) + level.Info(m.logger).Log("msg", "switched primary KV store", "primary", cfg.PrimaryStore) } if err != nil { - level.Error(util.Logger).Log("msg", "failed to switch primary KV store", "primary", cfg.PrimaryStore, "err", err) + level.Error(m.logger).Log("msg", "failed to switch primary KV store", "primary", cfg.PrimaryStore, "err", err) } } @@ -339,9 +344,9 @@ func (m *MultiClient) writeToSecondary(ctx context.Context, primary kvclient, ke if err != nil { mirrorFailuresCounter.Inc() - level.Warn(util.Logger).Log("msg", "failed to update value in secondary store", "key", key, "err", err, "primary", primary.name, "secondary", kvc.name) + level.Warn(m.logger).Log("msg", "failed to update value in secondary store", "key", key, "err", err, "primary", primary.name, "secondary", kvc.name) } else { - level.Debug(util.Logger).Log("msg", "stored updated value to secondary store", "key", key, "primary", primary.name, "secondary", kvc.name) + level.Debug(m.logger).Log("msg", "stored updated value to secondary store", "key", key, "primary", primary.name, "secondary", kvc.name) } } } From 713619a1dd469980ded84c5f5054b7fd8ed01a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 7 Jan 2020 10:48:44 +0100 Subject: [PATCH 30/31] Improve log message when runtime config file is not specified. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/util/runtimeconfig/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/runtimeconfig/manager.go b/pkg/util/runtimeconfig/manager.go index 78fc5af0744..a6b21857ea9 100644 --- a/pkg/util/runtimeconfig/manager.go +++ b/pkg/util/runtimeconfig/manager.go @@ -60,7 +60,7 @@ func NewRuntimeConfigManager(cfg ManagerConfig) (*Manager, error) { } go mgr.loop() } else { - level.Info(util.Logger).Log("msg", "config disabled") + level.Info(util.Logger).Log("msg", "runtime config disabled: file not specified") } return &mgr, nil From cc476c6a9fab629efb2ba3332e39379fc7366bc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Tue, 7 Jan 2020 10:49:57 +0100 Subject: [PATCH 31/31] Don't use memberlist in the example, as it is still experimental. Addressed other review feedback. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- docs/configuration/arguments.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/configuration/arguments.md b/docs/configuration/arguments.md index 0631da0813d..5cff13cd705 100644 --- a/docs/configuration/arguments.md +++ b/docs/configuration/arguments.md @@ -185,19 +185,22 @@ Flags for configuring KV store based on memberlist library. This feature is expe #### Multi KV -This is a special key-value implementation that uses two other KV stores (eg. consul, etcd or memberlist). One of them is always marked as primary, and all reads and writes go to primary store. Other one, secondary, is only used for writes. The idea is that operator can use multi KV store to migrate from primary to secondary store in runtime. For example, migration from Consul to Memberlist would look like this: +This is a special key-value implementation that uses two different KV stores (eg. consul, etcd or memberlist). One of them is always marked as primary, and all reads and writes go to primary store. Other one, secondary, is only used for writes. The idea is that operator can use multi KV store to migrate from primary to secondary store in runtime. -- Set `ring.store` to use `multi` store. Set `-multi.primary=consul` and `-multi.secondary=memberlist`. All consul and memberlist settings must still be specified. -- Start all Cortex microservices. They will still use Consul as primary KV, but they will also write share ring via memberlist. -- Operator can now use "runtime config" mechanism to switch primary store to memberlist. -- After all Cortex microservices have picked up new primary store, and everything looks correct, operator can now shut down Consul, and modify Cortex configuration to use `-ring.store=memberlist` only. +For example, migration from Consul to Etcd would look like this: + +- Set `ring.store` to use `multi` store. Set `-multi.primary=consul` and `-multi.secondary=etcd`. All consul and etcd settings must still be specified. +- Start all Cortex microservices. They will still use Consul as primary KV, but they will also write share ring via etcd. +- Operator can now use "runtime config" mechanism to switch primary store to etcd. +- After all Cortex microservices have picked up new primary store, and everything looks correct, operator can now shut down Consul, and modify Cortex configuration to use `-ring.store=etcd` only. +- At this point, Consul can be shut down. Multi KV has following parameters: - `multi.primary` - name of primary KV store. Same values as in `ring.store` are supported, except `multi`. - `multi.secondary` - name of secondary KV store. - `multi.mirror-enabled` - enable mirroring of values to secondary store, defaults to true -- `multi.mirror-timeout` - wait max this time to write to secondary store to finish. Default to 2 seconds. Errors writing to secondary store are not reported to caller, but are logged to the log file and also reported via `cortex_multikv_mirror_write_errors_total` metric. +- `multi.mirror-timeout` - wait max this time to write to secondary store to finish. Default to 2 seconds. Errors writing to secondary store are not reported to caller, but are logged and also reported via `cortex_multikv_mirror_write_errors_total` metric. Multi KV also reacts on changes done via runtime configuration. It uses this section: