From ebf8caf4f1f29f5f2e0d70e9b1bd4f819bb1eec2 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Thu, 8 Aug 2019 11:44:02 -0400 Subject: [PATCH 1/5] ruler run using prometheus manager Signed-off-by: Jacob Lisi --- Makefile | 1 + go.mod | 2 +- go.sum | 2 + pkg/configs/client/client.go | 171 +- pkg/configs/client/config.go | 70 - pkg/configs/client/configs_test.go | 2 +- pkg/configs/configs.go | 17 + pkg/cortex/cortex.go | 4 +- pkg/cortex/modules.go | 23 +- pkg/ruler/api.go | 118 -- pkg/ruler/api_test.go | 469 ------ pkg/ruler/compat.go | 31 +- pkg/ruler/group.go | 30 - pkg/ruler/lifecycle.go | 24 +- pkg/ruler/mapper.go | 78 + pkg/ruler/mapper_test.go | 130 ++ pkg/ruler/mock_store.go | 41 + pkg/ruler/ruler.go | 372 ++-- pkg/ruler/ruler_test.go | 29 +- pkg/ruler/rules/compat.go | 65 + pkg/ruler/rules/rules.pb.go | 1491 +++++++++++++++++ pkg/ruler/rules/rules.proto | 33 + pkg/ruler/rules/store.go | 40 + pkg/ruler/scheduler.go | 333 ---- pkg/ruler/scheduler_test.go | 97 -- pkg/ruler/scheduling_queue.go | 173 -- pkg/ruler/scheduling_queue_test.go | 236 --- pkg/ruler/storage.go | 37 + pkg/ruler/worker.go | 67 - vendor/github.com/spf13/afero/.travis.yml | 21 + vendor/github.com/spf13/afero/LICENSE.txt | 174 ++ vendor/github.com/spf13/afero/README.md | 452 +++++ vendor/github.com/spf13/afero/afero.go | 108 ++ vendor/github.com/spf13/afero/appveyor.yml | 15 + vendor/github.com/spf13/afero/basepath.go | 180 ++ .../github.com/spf13/afero/cacheOnReadFs.go | 290 ++++ vendor/github.com/spf13/afero/const_bsds.go | 22 + .../github.com/spf13/afero/const_win_unix.go | 25 + .../github.com/spf13/afero/copyOnWriteFs.go | 293 ++++ vendor/github.com/spf13/afero/go.mod | 3 + vendor/github.com/spf13/afero/go.sum | 2 + vendor/github.com/spf13/afero/httpFs.go | 110 ++ vendor/github.com/spf13/afero/ioutil.go | 230 +++ vendor/github.com/spf13/afero/lstater.go | 27 + vendor/github.com/spf13/afero/match.go | 110 ++ vendor/github.com/spf13/afero/mem/dir.go | 37 + vendor/github.com/spf13/afero/mem/dirmap.go | 43 + vendor/github.com/spf13/afero/mem/file.go | 317 ++++ vendor/github.com/spf13/afero/memmap.go | 365 ++++ vendor/github.com/spf13/afero/os.go | 101 ++ vendor/github.com/spf13/afero/path.go | 106 ++ vendor/github.com/spf13/afero/readonlyfs.go | 80 + vendor/github.com/spf13/afero/regexpfs.go | 214 +++ vendor/github.com/spf13/afero/unionFile.go | 320 ++++ vendor/github.com/spf13/afero/util.go | 330 ++++ vendor/modules.txt | 9 +- 56 files changed, 6272 insertions(+), 1898 deletions(-) delete mode 100644 pkg/configs/client/config.go delete mode 100644 pkg/ruler/api.go delete mode 100644 pkg/ruler/api_test.go delete mode 100644 pkg/ruler/group.go create mode 100644 pkg/ruler/mapper.go create mode 100644 pkg/ruler/mapper_test.go create mode 100644 pkg/ruler/mock_store.go create mode 100644 pkg/ruler/rules/compat.go create mode 100644 pkg/ruler/rules/rules.pb.go create mode 100644 pkg/ruler/rules/rules.proto create mode 100644 pkg/ruler/rules/store.go delete mode 100644 pkg/ruler/scheduler.go delete mode 100644 pkg/ruler/scheduler_test.go delete mode 100644 pkg/ruler/scheduling_queue.go delete mode 100644 pkg/ruler/scheduling_queue_test.go create mode 100644 pkg/ruler/storage.go delete mode 100644 pkg/ruler/worker.go create mode 100644 vendor/github.com/spf13/afero/.travis.yml create mode 100644 vendor/github.com/spf13/afero/LICENSE.txt create mode 100644 vendor/github.com/spf13/afero/README.md create mode 100644 vendor/github.com/spf13/afero/afero.go create mode 100644 vendor/github.com/spf13/afero/appveyor.yml create mode 100644 vendor/github.com/spf13/afero/basepath.go create mode 100644 vendor/github.com/spf13/afero/cacheOnReadFs.go create mode 100644 vendor/github.com/spf13/afero/const_bsds.go create mode 100644 vendor/github.com/spf13/afero/const_win_unix.go create mode 100644 vendor/github.com/spf13/afero/copyOnWriteFs.go create mode 100644 vendor/github.com/spf13/afero/go.mod create mode 100644 vendor/github.com/spf13/afero/go.sum create mode 100644 vendor/github.com/spf13/afero/httpFs.go create mode 100644 vendor/github.com/spf13/afero/ioutil.go create mode 100644 vendor/github.com/spf13/afero/lstater.go create mode 100644 vendor/github.com/spf13/afero/match.go create mode 100644 vendor/github.com/spf13/afero/mem/dir.go create mode 100644 vendor/github.com/spf13/afero/mem/dirmap.go create mode 100644 vendor/github.com/spf13/afero/mem/file.go create mode 100644 vendor/github.com/spf13/afero/memmap.go create mode 100644 vendor/github.com/spf13/afero/os.go create mode 100644 vendor/github.com/spf13/afero/path.go create mode 100644 vendor/github.com/spf13/afero/readonlyfs.go create mode 100644 vendor/github.com/spf13/afero/regexpfs.go create mode 100644 vendor/github.com/spf13/afero/unionFile.go create mode 100644 vendor/github.com/spf13/afero/util.go diff --git a/Makefile b/Makefile index 172f088f319..66d6d7dae20 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,7 @@ pkg/querier/frontend/frontend.pb.go: pkg/querier/frontend/frontend.proto pkg/querier/queryrange/queryrange.pb.go: pkg/querier/queryrange/queryrange.proto pkg/chunk/storage/caching_index_client.pb.go: pkg/chunk/storage/caching_index_client.proto pkg/distributor/ha_tracker.pb.go: pkg/distributor/ha_tracker.proto +/pkg/ruler/rules/rules.pb.go: /pkg/ruler/rules/rules.proto all: $(UPTODATE_FILES) test: protos mod-check: protos diff --git a/go.mod b/go.mod index 0ba0cf3382f..11ea06355be 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,6 @@ require ( github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/consul/api v1.1.0 github.com/hashicorp/go-cleanhttp v0.5.1 - github.com/jonboulle/clockwork v0.1.0 github.com/json-iterator/go v1.1.6 github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect @@ -61,6 +60,7 @@ require ( github.com/sercand/kuberesolver v2.1.0+incompatible // indirect github.com/sirupsen/logrus v1.4.2 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/spf13/afero v1.2.2 github.com/stretchr/testify v1.3.0 github.com/tinylib/msgp v0.0.0-20161221055906-38a6f61a768d // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect diff --git a/go.sum b/go.sum index 10aa04d6765..91a2185a752 100644 --- a/go.sum +++ b/go.sum @@ -482,6 +482,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= diff --git a/pkg/configs/client/client.go b/pkg/configs/client/client.go index a5528b29ecc..11e0a88a4d4 100644 --- a/pkg/configs/client/client.go +++ b/pkg/configs/client/client.go @@ -3,97 +3,84 @@ package client import ( "context" "encoding/json" + "flag" "fmt" "net/http" "net/url" + "strings" "time" "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/configs/db" + "github.com/cortexproject/cortex/pkg/ruler/rules" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/go-kit/kit/log/level" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) +// Config says where we can find the ruler configs. +type Config struct { + ConfigsAPIURL flagext.URLValue + ClientTimeout time.Duration // HTTP timeout duration for requests made to the Weave Cloud configs service. +} + +// RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet +func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.Var(&cfg.ConfigsAPIURL, prefix+"configs.url", "URL of configs API server.") + f.DurationVar(&cfg.ClientTimeout, prefix+"configs.client-timeout", 5*time.Second, "Timeout for requests to Weave Cloud configs service.") +} + +var configsRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "cortex", + Name: "configs_request_duration_seconds", + Help: "Time spent requesting configs.", + Buckets: prometheus.DefBuckets, +}, []string{"operation", "status_code"}) + // Client is what the ruler and altermanger needs from a config store to process rules. type Client interface { - // GetRules returns all Cortex configurations from a configs API server - // that have been updated after the given configs.ID was last updated. - GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) - // GetAlerts fetches all the alerts that have changes since since. GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) } // New creates a new ConfigClient. -func New(cfg Config) (Client, error) { - // All of this falderal is to allow for a smooth transition away from - // using the configs server and toward directly connecting to the database. - // See https://github.com/cortexproject/cortex/issues/619 - if cfg.ConfigsAPIURL.URL != nil { - return instrumented{ - next: configsClient{ - URL: cfg.ConfigsAPIURL.URL, - Timeout: cfg.ClientTimeout, - }, - }, nil - } - - db, err := db.New(cfg.DBConfig) - if err != nil { - return nil, err - } - return instrumented{ - next: dbStore{ - db: db, - }, +func New(cfg Config) (*ConfigDBClient, error) { + return &ConfigDBClient{ + URL: cfg.ConfigsAPIURL.URL, + Timeout: cfg.ClientTimeout, }, nil } -// configsClient allows retrieving recording and alerting rules from the configs server. -type configsClient struct { +// ConfigDBClient allows retrieving recording and alerting rules from the configs server. +type ConfigDBClient struct { URL *url.URL Timeout time.Duration } -// GetRules implements ConfigClient. -func (c configsClient) GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { - suffix := "" - if since != 0 { - suffix = fmt.Sprintf("?since=%d", since) - } - endpoint := fmt.Sprintf("%s/private/api/prom/configs/rules%s", c.URL.String(), suffix) - response, err := doRequest(endpoint, c.Timeout, since) - if err != nil { - return nil, err - } - configs := map[string]configs.VersionedRulesConfig{} - for id, view := range response.Configs { - cfg := view.GetVersionedRulesConfig() - if cfg != nil { - configs[id] = *cfg - } - } - return configs, nil -} - // GetAlerts implements ConfigClient. -func (c configsClient) GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) { +func (c ConfigDBClient) GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) { suffix := "" if since != 0 { suffix = fmt.Sprintf("?since=%d", since) } endpoint := fmt.Sprintf("%s/private/api/prom/configs/alertmanager%s", c.URL.String(), suffix) - return doRequest(endpoint, c.Timeout, since) + return doRequest(endpoint, c.Timeout, since, "GetAlerts") } -func doRequest(endpoint string, timeout time.Duration, since configs.ID) (*ConfigsResponse, error) { +func doRequest(endpoint string, timeout time.Duration, since configs.ID, operation string) (*ConfigsResponse, error) { req, err := http.NewRequest("GET", endpoint, nil) if err != nil { return nil, err } client := &http.Client{Timeout: timeout} + + start := time.Now() resp, err := client.Do(req) + if resp != nil { + configsRequestDuration.WithLabelValues(operation, resp.Status).Observe(time.Since(start).Seconds()) + } if err != nil { return nil, err } @@ -113,37 +100,6 @@ func doRequest(endpoint string, timeout time.Duration, since configs.ID) (*Confi return &config, nil } -type dbStore struct { - db db.DB -} - -// GetRules implements ConfigClient. -func (d dbStore) GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { - if since == 0 { - return d.db.GetAllRulesConfigs(ctx) - } - return d.db.GetRulesConfigs(ctx, since) -} - -// GetAlerts implements ConfigClient. -func (d dbStore) GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) { - var resp map[string]configs.View - var err error - if since == 0 { - resp, err = d.db.GetAllConfigs(ctx) - - } - resp, err = d.db.GetConfigs(ctx, since) - if err != nil { - return nil, err - } - - return &ConfigsResponse{ - since: since, - Configs: resp, - }, nil -} - // ConfigsResponse is a response from server for GetConfigs. type ConfigsResponse struct { // The version since which these configs were changed @@ -163,3 +119,52 @@ func (c ConfigsResponse) GetLatestConfigID() configs.ID { } return latest } + +// ListAllRuleGroups polls the configdb server and returns the updated rule groups +func (c *ConfigDBClient) ListAllRuleGroups(ctx context.Context) (map[string]rules.RuleGroupList, error) { + endpoint := fmt.Sprintf("%s/private/api/prom/configs/rules", c.URL.String()) + response, err := doRequest(endpoint, c.Timeout, 0, "ListAllRuleGroups") + if err != nil { + return nil, err + } + + configs := map[string]configs.VersionedRulesConfig{} + for id, view := range response.Configs { + cfg := view.GetVersionedRulesConfig() + if cfg != nil { + configs[id] = *cfg + } + } + + newRules := map[string]rules.RuleGroupList{} + + for user, cfg := range configs { + userRules := rules.RuleGroupList{} + if cfg.IsDeleted() { + newRules[user] = rules.RuleGroupList{} + } + rMap, err := cfg.Config.ParseFormatted() + if err != nil { + return nil, err + } + for file, rgs := range rMap { + for _, rg := range rgs.Groups { + userRules = append(userRules, rules.ToProto(user, file, rg)) + } + } + newRules[user] = userRules + } + + if err != nil { + return nil, err + } + + return newRules, nil +} + +// decomposeGroupSlug breaks the group slug from Parse +// into it's group name and file name +func decomposeGroupSlug(slug string) (string, string) { + components := strings.Split(slug, ";") + return components[0], components[1] +} diff --git a/pkg/configs/client/config.go b/pkg/configs/client/config.go deleted file mode 100644 index 3850b967c68..00000000000 --- a/pkg/configs/client/config.go +++ /dev/null @@ -1,70 +0,0 @@ -package client - -import ( - "context" - "flag" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/weaveworks/common/instrument" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/configs/db" - "github.com/cortexproject/cortex/pkg/util/flagext" -) - -// Config says where we can find the ruler configs. -type Config struct { - DBConfig db.Config - - // DEPRECATED - ConfigsAPIURL flagext.URLValue - - // DEPRECATED. HTTP timeout duration for requests made to the Weave Cloud - // configs service. - ClientTimeout time.Duration -} - -// RegisterFlags adds the flags required to config this to the given FlagSet -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - cfg.DBConfig.RegisterFlags(f) - f.Var(&cfg.ConfigsAPIURL, "ruler.configs.url", "DEPRECATED. URL of configs API server.") - f.DurationVar(&cfg.ClientTimeout, "ruler.client-timeout", 5*time.Second, "DEPRECATED. Timeout for requests to Weave Cloud configs service.") - flag.Var(&cfg.ConfigsAPIURL, "alertmanager.configs.url", "URL of configs API server.") - flag.DurationVar(&cfg.ClientTimeout, "alertmanager.configs.client-timeout", 5*time.Second, "Timeout for requests to Weave Cloud configs service.") -} - -var configsRequestDuration = instrument.NewHistogramCollector(prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "cortex", - Name: "configs_request_duration_seconds", - Help: "Time spent requesting configs.", - Buckets: prometheus.DefBuckets, -}, []string{"operation", "status_code"})) - -func init() { - configsRequestDuration.Register() -} - -type instrumented struct { - next Client -} - -func (i instrumented) GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { - var cfgs map[string]configs.VersionedRulesConfig - err := instrument.CollectedRequest(context.Background(), "Configs.GetConfigs", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - cfgs, err = i.next.GetRules(ctx, since) // Warning: this will produce an incorrect result if the configID ever overflows - return err - }) - return cfgs, err -} - -func (i instrumented) GetAlerts(ctx context.Context, since configs.ID) (*ConfigsResponse, error) { - var cfgs *ConfigsResponse - err := instrument.CollectedRequest(context.Background(), "Configs.GetConfigs", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - cfgs, err = i.next.GetAlerts(ctx, since) // Warning: this will produce an incorrect result if the configID ever overflows - return err - }) - return cfgs, err -} diff --git a/pkg/configs/client/configs_test.go b/pkg/configs/client/configs_test.go index af7acf25b45..ab95a965e18 100644 --- a/pkg/configs/client/configs_test.go +++ b/pkg/configs/client/configs_test.go @@ -34,7 +34,7 @@ func TestDoRequest(t *testing.T) { })) defer server.Close() - resp, err := doRequest(server.URL, 1*time.Second, 0) + resp, err := doRequest(server.URL, 1*time.Second, 0, "TestDoRequest") assert.Nil(t, err) expected := ConfigsResponse{Configs: map[string]configs.View{ diff --git a/pkg/configs/configs.go b/pkg/configs/configs.go index 4cb32268ca5..f77fc5cc554 100644 --- a/pkg/configs/configs.go +++ b/pkg/configs/configs.go @@ -182,6 +182,23 @@ func (c RulesConfig) Parse() (map[string][]rules.Rule, error) { } } +// ParseFormatted returns the rulefmt map of a users rules configs. It allows +// for rules to be mapped to disk and read by the prometheus rules manager. +func (c RulesConfig) ParseFormatted() (map[string]rulefmt.RuleGroups, error) { + ruleMap := map[string]rulefmt.RuleGroups{} + for fn, content := range c.Files { + rgs, errs := rulefmt.Parse([]byte(content)) + if errs != nil { + for _, err := range errs { + return nil, err + } + } + ruleMap[fn] = *rgs + + } + return ruleMap, nil +} + // parseV2 parses and validates the content of the rule files in a RulesConfig // according to the Prometheus 2.x rule format. // diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index 8b135dd33b4..7ce4b0d589b 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -70,6 +70,7 @@ type Config struct { TableManager chunk.TableManagerConfig `yaml:"table_manager,omitempty"` Encoding encoding.Config `yaml:"-"` // No yaml for this, it only works with flags. + ConfigDB db.Config `yaml:"configdb,omitempty"` Ruler ruler.Config `yaml:"ruler,omitempty"` ConfigStore config_client.Config `yaml:"config_store,omitempty"` Alertmanager alertmanager.MultitenantAlertmanagerConfig `yaml:"alertmanager,omitempty"` @@ -101,7 +102,8 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { c.Encoding.RegisterFlags(f) c.Ruler.RegisterFlags(f) - c.ConfigStore.RegisterFlags(f) + c.ConfigDB.RegisterFlags(f) + c.ConfigStore.RegisterFlagsWithPrefix("alertmanager.", f) c.Alertmanager.RegisterFlags(f) // These don't seem to have a home. diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 6c76d4ce7a8..065a18f7ae3 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -21,7 +21,6 @@ import ( "github.com/cortexproject/cortex/pkg/chunk" "github.com/cortexproject/cortex/pkg/chunk/storage" "github.com/cortexproject/cortex/pkg/configs/api" - config_client "github.com/cortexproject/cortex/pkg/configs/client" "github.com/cortexproject/cortex/pkg/configs/db" "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/ingester" @@ -324,32 +323,14 @@ func (t *Cortex) stopTableManager() error { } func (t *Cortex) initRuler(cfg *Config) (err error) { - cfg.Querier.MaxConcurrent = cfg.Ruler.NumWorkers - cfg.Querier.Timeout = cfg.Ruler.GroupTimeout cfg.Ruler.LifecyclerConfig.ListenPort = &cfg.Server.GRPCListenPort queryable, engine := querier.New(cfg.Querier, t.distributor, t.store) - rulesAPI, err := config_client.New(cfg.ConfigStore) - if err != nil { - return err - } - - t.ruler, err = ruler.NewRuler(cfg.Ruler, engine, queryable, t.distributor, rulesAPI) + t.ruler, err = ruler.NewRuler(cfg.Ruler, engine, queryable, t.distributor) if err != nil { return } - // Only serve the API for setting & getting rules configs if we're not - // serving configs from the configs API. Allows for smoother - // migration. See https://github.com/cortexproject/cortex/issues/619 - if cfg.ConfigStore.ConfigsAPIURL.URL == nil { - a, err := ruler.NewAPIFromConfig(cfg.ConfigStore.DBConfig) - if err != nil { - return err - } - a.RegisterRoutes(t.server.HTTP) - } - t.server.HTTP.Handle("/ruler_ring", t.ruler) return } @@ -360,7 +341,7 @@ func (t *Cortex) stopRuler() error { } func (t *Cortex) initConfigs(cfg *Config) (err error) { - t.configDB, err = db.New(cfg.ConfigStore.DBConfig) + t.configDB, err = db.New(cfg.ConfigDB) if err != nil { return } diff --git a/pkg/ruler/api.go b/pkg/ruler/api.go deleted file mode 100644 index aeee8fe5ba2..00000000000 --- a/pkg/ruler/api.go +++ /dev/null @@ -1,118 +0,0 @@ -package ruler - -import ( - "database/sql" - "encoding/json" - "fmt" - "net/http" - - "github.com/go-kit/kit/log/level" - "github.com/gorilla/mux" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/configs/db" - "github.com/cortexproject/cortex/pkg/util" - "github.com/weaveworks/common/user" -) - -// API implements the configs api. -type API struct { - db db.DB - http.Handler -} - -// NewAPIFromConfig makes a new API from our database config. -func NewAPIFromConfig(cfg db.Config) (*API, error) { - db, err := db.New(cfg) - if err != nil { - return nil, err - } - return NewAPI(db), nil -} - -// NewAPI creates a new API. -func NewAPI(db db.DB) *API { - a := &API{db: db} - r := mux.NewRouter() - a.RegisterRoutes(r) - a.Handler = r - return a -} - -// RegisterRoutes registers the configs API HTTP routes with the provided Router. -func (a *API) RegisterRoutes(r *mux.Router) { - for _, route := range []struct { - name, method, path string - handler http.HandlerFunc - }{ - {"get_rules", "GET", "/api/prom/rules", a.getConfig}, - {"cas_rules", "POST", "/api/prom/rules", a.casConfig}, - } { - r.Handle(route.path, route.handler).Methods(route.method).Name(route.name) - } -} - -// getConfig returns the request configuration. -func (a *API) getConfig(w http.ResponseWriter, r *http.Request) { - userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) - if err != nil { - http.Error(w, err.Error(), http.StatusUnauthorized) - return - } - logger := util.WithContext(r.Context(), util.Logger) - - cfg, err := a.db.GetRulesConfig(r.Context(), userID) - if err == sql.ErrNoRows { - http.Error(w, "No configuration", http.StatusNotFound) - return - } else if err != nil { - level.Error(logger).Log("msg", "error getting config", "err", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(cfg); err != nil { - level.Error(logger).Log("msg", "error encoding config", "err", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} - -type configUpdateRequest struct { - OldConfig configs.RulesConfig `json:"old_config"` - NewConfig configs.RulesConfig `json:"new_config"` -} - -func (a *API) casConfig(w http.ResponseWriter, r *http.Request) { - userID, _, err := user.ExtractOrgIDFromHTTPRequest(r) - if err != nil { - http.Error(w, err.Error(), http.StatusUnauthorized) - return - } - logger := util.WithContext(r.Context(), util.Logger) - - var updateReq configUpdateRequest - if err := json.NewDecoder(r.Body).Decode(&updateReq); err != nil { - level.Error(logger).Log("msg", "error decoding json body", "err", err) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if _, err = updateReq.NewConfig.Parse(); err != nil { - level.Error(logger).Log("msg", "invalid rules", "err", err) - http.Error(w, fmt.Sprintf("Invalid rules: %v", err), http.StatusBadRequest) - return - } - - updated, err := a.db.SetRulesConfig(r.Context(), userID, updateReq.OldConfig, updateReq.NewConfig) - if err != nil { - level.Error(logger).Log("msg", "error storing config", "err", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if !updated { - http.Error(w, "Supplied configuration doesn't match current configuration", http.StatusConflict) - } - w.WriteHeader(http.StatusNoContent) -} diff --git a/pkg/ruler/api_test.go b/pkg/ruler/api_test.go deleted file mode 100644 index 3d8e4f51c26..00000000000 --- a/pkg/ruler/api_test.go +++ /dev/null @@ -1,469 +0,0 @@ -package ruler - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/cortexproject/cortex/pkg/configs" - "github.com/cortexproject/cortex/pkg/configs/api" - "github.com/cortexproject/cortex/pkg/configs/client" - "github.com/cortexproject/cortex/pkg/configs/db" - "github.com/cortexproject/cortex/pkg/configs/db/dbtest" - "github.com/weaveworks/common/user" -) - -const ( - endpoint = "/api/prom/rules" -) - -var ( - app *API - database db.DB - counter int - privateAPI client.Client -) - -// setup sets up the environment for the tests. -func setup(t *testing.T) { - database = dbtest.Setup(t) - app = NewAPI(database) - counter = 0 - var err error - privateAPI, err = client.New(client.Config{ - DBConfig: db.Config{ - URI: "mock", // trigger client.NewConfigClient to use the mock DB. - Mock: database, - }, - }) - require.NoError(t, err) -} - -// cleanup cleans up the environment after a test. -func cleanup(t *testing.T) { - dbtest.Cleanup(t, database) -} - -// request makes a request to the configs API. -func request(t *testing.T, handler http.Handler, method, urlStr string, body io.Reader) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - r, err := http.NewRequest(method, urlStr, body) - require.NoError(t, err) - handler.ServeHTTP(w, r) - return w -} - -// requestAsUser makes a request to the configs API as the given user. -func requestAsUser(t *testing.T, handler http.Handler, userID string, method, urlStr string, body io.Reader) *httptest.ResponseRecorder { - w := httptest.NewRecorder() - r, err := http.NewRequest(method, urlStr, body) - require.NoError(t, err) - r = r.WithContext(user.InjectOrgID(r.Context(), userID)) - user.InjectOrgIDIntoHTTPRequest(r.Context(), r) - handler.ServeHTTP(w, r) - return w -} - -// makeString makes a string, guaranteed to be unique within a test. -func makeString(pattern string) string { - counter++ - return fmt.Sprintf(pattern, counter) -} - -// makeUserID makes an arbitrary user ID. Guaranteed to be unique within a test. -func makeUserID() string { - return makeString("user%d") -} - -// makeRulerConfig makes an arbitrary ruler config -func makeRulerConfig(rfv configs.RuleFormatVersion) configs.RulesConfig { - switch rfv { - case configs.RuleFormatV1: - return configs.RulesConfig{ - Files: map[string]string{ - "filename.rules": makeString(` -# Config no. %d. -ALERT ScrapeFailed - IF up != 1 - FOR 10m - LABELS { severity="warning" } - ANNOTATIONS { - summary = "Scrape of {{$labels.job}} (pod: {{$labels.instance}}) failed.", - description = "Prometheus cannot reach the /metrics page on the {{$labels.instance}} pod.", - impact = "We have no monitoring data for {{$labels.job}} - {{$labels.instance}}. At worst, it's completely down. At best, we cannot reliably respond to operational issues.", - dashboardURL = "$${base_url}/admin/prometheus/targets", - } - `), - }, - FormatVersion: configs.RuleFormatV1, - } - case configs.RuleFormatV2: - return configs.RulesConfig{ - Files: map[string]string{ - "filename.rules": makeString(` -# Config no. %d. -groups: -- name: example - rules: - - alert: ScrapeFailed - expr: 'up != 1' - for: 10m - labels: - severity: warning - annotations: - summary: "Scrape of {{$labels.job}} (pod: {{$labels.instance}}) failed." - description: "Prometheus cannot reach the /metrics page on the {{$labels.instance}} pod." - impact: "We have no monitoring data for {{$labels.job}} - {{$labels.instance}}. At worst, it's completely down. At best, we cannot reliably respond to operational issues." - dashboardURL: "$${base_url}/admin/prometheus/targets" - `), - }, - FormatVersion: configs.RuleFormatV2, - } - default: - panic("unknown rule format") - } -} - -// parseVersionedRulesConfig parses a configs.VersionedRulesConfig from JSON. -func parseVersionedRulesConfig(t *testing.T, b []byte) configs.VersionedRulesConfig { - var x configs.VersionedRulesConfig - err := json.Unmarshal(b, &x) - require.NoError(t, err, "Could not unmarshal JSON: %v", string(b)) - return x -} - -// post a config -func post(t *testing.T, userID string, oldConfig configs.RulesConfig, newConfig configs.RulesConfig) configs.VersionedRulesConfig { - updateRequest := configUpdateRequest{ - OldConfig: oldConfig, - NewConfig: newConfig, - } - b, err := json.Marshal(updateRequest) - require.NoError(t, err) - reader := bytes.NewReader(b) - w := requestAsUser(t, app, userID, "POST", endpoint, reader) - require.Equal(t, http.StatusNoContent, w.Code) - return get(t, userID) -} - -// get a config -func get(t *testing.T, userID string) configs.VersionedRulesConfig { - w := requestAsUser(t, app, userID, "GET", endpoint, nil) - return parseVersionedRulesConfig(t, w.Body.Bytes()) -} - -// configs returns 404 if no config has been created yet. -func Test_GetConfig_NotFound(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - w := requestAsUser(t, app, userID, "GET", endpoint, nil) - assert.Equal(t, http.StatusNotFound, w.Code) -} - -// configs returns 401 to requests without authentication. -func Test_PostConfig_Anonymous(t *testing.T) { - setup(t) - defer cleanup(t) - - w := request(t, app, "POST", endpoint, nil) - assert.Equal(t, http.StatusUnauthorized, w.Code) -} - -// Posting to a configuration sets it so that you can get it again. -func Test_PostConfig_CreatesConfig(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - config := makeRulerConfig(configs.RuleFormatV2) - result := post(t, userID, configs.RulesConfig{}, config) - assert.Equal(t, config, result.Config) -} - -// Posting an invalid config when there's none set returns an error and leaves the config unset. -func Test_PostConfig_InvalidNewConfig(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - invalidConfig := configs.RulesConfig{ - Files: map[string]string{ - "some.rules": "invalid config", - }, - FormatVersion: configs.RuleFormatV2, - } - updateRequest := configUpdateRequest{ - OldConfig: configs.RulesConfig{}, - NewConfig: invalidConfig, - } - b, err := json.Marshal(updateRequest) - require.NoError(t, err) - reader := bytes.NewReader(b) - { - w := requestAsUser(t, app, userID, "POST", endpoint, reader) - require.Equal(t, http.StatusBadRequest, w.Code) - } - { - w := requestAsUser(t, app, userID, "GET", endpoint, nil) - require.Equal(t, http.StatusNotFound, w.Code) - } -} - -// Posting a v1 rule format configuration sets it so that you can get it again. -func Test_PostConfig_UpdatesConfig_V1RuleFormat(t *testing.T) { - setup(t) - app = NewAPI(database) - defer cleanup(t) - - userID := makeUserID() - config1 := makeRulerConfig(configs.RuleFormatV1) - view1 := post(t, userID, configs.RulesConfig{}, config1) - config2 := makeRulerConfig(configs.RuleFormatV1) - view2 := post(t, userID, config1, config2) - assert.True(t, view2.ID > view1.ID, "%v > %v", view2.ID, view1.ID) - assert.Equal(t, config2, view2.Config) -} - -// Posting an invalid v1 rule format config when there's one already set returns an error and leaves the config as is. -func Test_PostConfig_InvalidChangedConfig_V1RuleFormat(t *testing.T) { - setup(t) - app = NewAPI(database) - defer cleanup(t) - - userID := makeUserID() - config := makeRulerConfig(configs.RuleFormatV1) - post(t, userID, configs.RulesConfig{}, config) - invalidConfig := configs.RulesConfig{ - Files: map[string]string{ - "some.rules": "invalid config", - }, - FormatVersion: configs.RuleFormatV1, - } - updateRequest := configUpdateRequest{ - OldConfig: configs.RulesConfig{}, - NewConfig: invalidConfig, - } - b, err := json.Marshal(updateRequest) - require.NoError(t, err) - reader := bytes.NewReader(b) - { - w := requestAsUser(t, app, userID, "POST", endpoint, reader) - require.Equal(t, http.StatusBadRequest, w.Code) - } - result := get(t, userID) - assert.Equal(t, config, result.Config) -} - -// Posting a v2 rule format configuration sets it so that you can get it again. -func Test_PostConfig_UpdatesConfig_V2RuleFormat(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - config1 := makeRulerConfig(configs.RuleFormatV2) - view1 := post(t, userID, configs.RulesConfig{}, config1) - config2 := makeRulerConfig(configs.RuleFormatV2) - view2 := post(t, userID, config1, config2) - assert.True(t, view2.ID > view1.ID, "%v > %v", view2.ID, view1.ID) - assert.Equal(t, config2, view2.Config) -} - -// Posting an invalid v2 rule format config when there's one already set returns an error and leaves the config as is. -func Test_PostConfig_InvalidChangedConfig_V2RuleFormat(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - config := makeRulerConfig(configs.RuleFormatV2) - post(t, userID, configs.RulesConfig{}, config) - invalidConfig := configs.RulesConfig{ - Files: map[string]string{ - "some.rules": "invalid config", - }, - } - updateRequest := configUpdateRequest{ - OldConfig: configs.RulesConfig{}, - NewConfig: invalidConfig, - } - b, err := json.Marshal(updateRequest) - require.NoError(t, err) - reader := bytes.NewReader(b) - { - w := requestAsUser(t, app, userID, "POST", endpoint, reader) - require.Equal(t, http.StatusBadRequest, w.Code) - } - result := get(t, userID) - assert.Equal(t, config, result.Config) -} - -// Posting a config with an invalid rule format version returns an error and leaves the config as is. -func Test_PostConfig_InvalidChangedConfig_InvalidRuleFormat(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - config := makeRulerConfig(configs.RuleFormatV2) - post(t, userID, configs.RulesConfig{}, config) - - // We have to provide the marshaled JSON manually here because json.Marshal() would error - // on a bad rule format version. - reader := strings.NewReader(`{"old_config":{"format_version":"1","files":null},"new_config":{"format_version":"","files":{"filename.rules":"# Empty."}}}`) - { - w := requestAsUser(t, app, userID, "POST", endpoint, reader) - require.Equal(t, http.StatusBadRequest, w.Code) - } - result := get(t, userID) - assert.Equal(t, config, result.Config) -} - -// Different users can have different configurations. -func Test_PostConfig_MultipleUsers(t *testing.T) { - setup(t) - defer cleanup(t) - - userID1 := makeUserID() - userID2 := makeUserID() - config1 := post(t, userID1, configs.RulesConfig{}, makeRulerConfig(configs.RuleFormatV2)) - config2 := post(t, userID2, configs.RulesConfig{}, makeRulerConfig(configs.RuleFormatV2)) - foundConfig1 := get(t, userID1) - assert.Equal(t, config1, foundConfig1) - foundConfig2 := get(t, userID2) - assert.Equal(t, config2, foundConfig2) - assert.True(t, config2.ID > config1.ID, "%v > %v", config2.ID, config1.ID) -} - -// GetAllConfigs returns an empty list of configs if there aren't any. -func Test_GetAllConfigs_Empty(t *testing.T) { - setup(t) - defer cleanup(t) - - configs, err := privateAPI.GetRules(context.Background(), 0) - assert.NoError(t, err, "error getting configs") - assert.Equal(t, 0, len(configs)) -} - -// GetAllConfigs returns all created configs. -func Test_GetAllConfigs(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - config := makeRulerConfig(configs.RuleFormatV2) - view := post(t, userID, configs.RulesConfig{}, config) - - found, err := privateAPI.GetRules(context.Background(), 0) - assert.NoError(t, err, "error getting configs") - assert.Equal(t, map[string]configs.VersionedRulesConfig{ - userID: view, - }, found) -} - -// GetAllConfigs returns the *newest* versions of all created configs. -func Test_GetAllConfigs_Newest(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - - config1 := post(t, userID, configs.RulesConfig{}, makeRulerConfig(configs.RuleFormatV2)) - config2 := post(t, userID, config1.Config, makeRulerConfig(configs.RuleFormatV2)) - lastCreated := post(t, userID, config2.Config, makeRulerConfig(configs.RuleFormatV2)) - - found, err := privateAPI.GetRules(context.Background(), 0) - assert.NoError(t, err, "error getting configs") - assert.Equal(t, map[string]configs.VersionedRulesConfig{ - userID: lastCreated, - }, found) -} - -func Test_GetConfigs_IncludesNewerConfigsAndExcludesOlder(t *testing.T) { - setup(t) - defer cleanup(t) - - post(t, makeUserID(), configs.RulesConfig{}, makeRulerConfig(configs.RuleFormatV2)) - config2 := post(t, makeUserID(), configs.RulesConfig{}, makeRulerConfig(configs.RuleFormatV2)) - userID3 := makeUserID() - config3 := post(t, userID3, configs.RulesConfig{}, makeRulerConfig(configs.RuleFormatV2)) - - found, err := privateAPI.GetRules(context.Background(), config2.ID) - assert.NoError(t, err, "error getting configs") - assert.Equal(t, map[string]configs.VersionedRulesConfig{ - userID3: config3, - }, found) -} - -// postAlertmanagerConfig posts an alertmanager config to the alertmanager configs API. -func postAlertmanagerConfig(t *testing.T, userID, configFile string) { - config := configs.Config{ - AlertmanagerConfig: configFile, - RulesConfig: configs.RulesConfig{}, - } - b, err := json.Marshal(config) - require.NoError(t, err) - reader := bytes.NewReader(b) - configsAPI := api.New(database) - w := requestAsUser(t, configsAPI, userID, "POST", "/api/prom/configs/alertmanager", reader) - require.Equal(t, http.StatusNoContent, w.Code) -} - -// getAlertmanagerConfig posts an alertmanager config to the alertmanager configs API. -func getAlertmanagerConfig(t *testing.T, userID string) string { - w := requestAsUser(t, api.New(database), userID, "GET", "/api/prom/configs/alertmanager", nil) - var x configs.View - b := w.Body.Bytes() - err := json.Unmarshal(b, &x) - require.NoError(t, err, "Could not unmarshal JSON: %v", string(b)) - return x.Config.AlertmanagerConfig -} - -// If a user has only got alertmanager config set, then we learn nothing about them via GetConfigs. -func Test_AlertmanagerConfig_NotInAllConfigs(t *testing.T) { - setup(t) - defer cleanup(t) - - config := makeString(` - # Config no. %d. - route: - receiver: noop - - receivers: - - name: noop`) - postAlertmanagerConfig(t, makeUserID(), config) - - found, err := privateAPI.GetRules(context.Background(), 0) - assert.NoError(t, err, "error getting configs") - assert.Equal(t, map[string]configs.VersionedRulesConfig{}, found) -} - -// Setting a ruler config doesn't change alertmanager config. -func Test_AlertmanagerConfig_RulerConfigDoesntChangeIt(t *testing.T) { - setup(t) - defer cleanup(t) - - userID := makeUserID() - alertmanagerConfig := makeString(` - # Config no. %d. - route: - receiver: noop - - receivers: - - name: noop`) - postAlertmanagerConfig(t, userID, alertmanagerConfig) - - rulerConfig := makeRulerConfig(configs.RuleFormatV2) - post(t, userID, configs.RulesConfig{}, rulerConfig) - - newAlertmanagerConfig := getAlertmanagerConfig(t, userID) - assert.Equal(t, alertmanagerConfig, newAlertmanagerConfig) -} diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index b3bc5a4beb7..60f7fca8742 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -5,6 +5,7 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" + "github.com/weaveworks/common/user" "github.com/cortexproject/cortex/pkg/ingester/client" ) @@ -26,6 +27,7 @@ type appendableAppender struct { ctx context.Context labels []labels.Labels samples []client.Sample + userID string } func (a *appendableAppender) Appender() (storage.Appender, error) { @@ -47,7 +49,7 @@ func (a *appendableAppender) AddFast(l labels.Labels, ref uint64, t int64, v flo } func (a *appendableAppender) Commit() error { - _, err := a.pusher.Push(a.ctx, client.ToWriteRequest(a.labels, a.samples, client.RULE)) + _, err := a.pusher.Push(user.InjectOrgID(context.Background(), a.userID), client.ToWriteRequest(a.labels, a.samples, client.RULE)) a.labels = nil a.samples = nil return err @@ -58,3 +60,30 @@ func (a *appendableAppender) Rollback() error { a.samples = nil return nil } + +// TSDB fulfills the storage.Storage interface for prometheus manager +// it allows for alerts to be restored by the manager +type tsdb struct { + appender *appendableAppender + queryable storage.Queryable +} + +// Querier returns a new Querier on the storage. +func (t *tsdb) Querier(ctx context.Context, mint int64, maxt int64) (storage.Querier, error) { + return t.queryable.Querier(ctx, mint, maxt) +} + +// StartTime returns the oldest timestamp stored in the storage. +func (t *tsdb) StartTime() (int64, error) { + return 0, nil +} + +// Appender returns a new appender against the storage. +func (t *tsdb) Appender() (storage.Appender, error) { + return t.appender, nil +} + +// Close closes the storage and all its underlying resources. +func (t *tsdb) Close() error { + return nil +} diff --git a/pkg/ruler/group.go b/pkg/ruler/group.go deleted file mode 100644 index a9a9d34678e..00000000000 --- a/pkg/ruler/group.go +++ /dev/null @@ -1,30 +0,0 @@ -package ruler - -import ( - "context" - "time" - - "github.com/prometheus/prometheus/rules" -) - -// group is a wrapper around a prometheus rules.Group, with a mutable appendable -// appendable stored here will be the same appendable as in promGroup.opts.Appendable -type group struct { - promGroup *rules.Group - appendable *appendableAppender -} - -func newGroup(name string, rls []rules.Rule, appendable *appendableAppender, opts *rules.ManagerOptions) *group { - delay := 0 * time.Second // Unused, so 0 value is fine. - promGroup := rules.NewGroup(name, "none", delay, rls, false, opts) - return &group{promGroup, appendable} -} - -func (g *group) Eval(ctx context.Context, ts time.Time) { - g.appendable.ctx = ctx - g.promGroup.Eval(ctx, ts) -} - -func (g *group) Rules() []rules.Rule { - return g.promGroup.Rules() -} diff --git a/pkg/ruler/lifecycle.go b/pkg/ruler/lifecycle.go index 49dfb0034d8..0b4e775f7d3 100644 --- a/pkg/ruler/lifecycle.go +++ b/pkg/ruler/lifecycle.go @@ -2,10 +2,6 @@ package ruler import ( "context" - "time" - - "github.com/cortexproject/cortex/pkg/util" - "github.com/go-kit/kit/log/level" ) // TransferOut is a noop for the ruler @@ -23,22 +19,4 @@ func (r *Ruler) StopIncomingRequests() {} // scheduled by the ruler, currently every ruler will // query a backend rule store for it's rules so no // flush is required. -func (r *Ruler) Flush() { - flushTicker := time.NewTicker(r.cfg.FlushCheckPeriod) - defer flushTicker.Stop() - - for { - select { - case <-flushTicker.C: - level.Debug(util.Logger).Log("msg", "ruler flush timed out") - return - default: - item := r.scheduler.nextWorkItem() - if item == nil { - level.Debug(util.Logger).Log("msg", "flushing complete") - return - } - level.Debug(util.Logger).Log("msg", "flushing item", "item", item) - } - } -} +func (r *Ruler) Flush() {} diff --git a/pkg/ruler/mapper.go b/pkg/ruler/mapper.go new file mode 100644 index 00000000000..20cd1a828e4 --- /dev/null +++ b/pkg/ruler/mapper.go @@ -0,0 +1,78 @@ +package ruler + +import ( + "crypto/md5" + "sort" + + "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log/level" + "github.com/prometheus/prometheus/pkg/rulefmt" + "github.com/spf13/afero" + "gopkg.in/yaml.v2" +) + +// mapper is designed to enusre the provided rule sets are identical +// to the on-disk rules tracked by the prometheus manager +type mapper struct { + Path string + + FS afero.Fs +} + +func newMapper(path string) *mapper { + return &mapper{ + Path: path, + FS: afero.NewOsFs(), + } +} + +func (m *mapper) MapRules(user string, ruleConfigs map[string][]rulefmt.RuleGroup) (bool, []string, error) { + var update bool + + // user rule files will be stored as `///filename` + path := m.Path + "/" + user + "/" + err := m.FS.MkdirAll(path, 0777) + if err != nil { + return false, nil, err + } + + filenames := []string{} + + // iterate through each rule group and map to disk if updated + for f, groups := range ruleConfigs { + // ensure rule groups are sorted before writing the file to disk + sort.Slice(groups, func(i, j int) bool { + return groups[i].Name > groups[j].Name + }) + + rgs := rulefmt.RuleGroups{Groups: groups} + file := path + f + d, err := yaml.Marshal(&rgs) + if err != nil { + return false, nil, err + } + + // Determine if the file exists and whether it is identical to the current rule file + if _, err := m.FS.Stat(file); err == nil { + current, err := afero.ReadFile(m.FS, file) + if err != nil { + level.Warn(util.Logger).Log("msg", "unable to read rule file on disk", "file", file, "err", err) + continue + } + newHash := md5.New() + currentHash := md5.New() + if string(currentHash.Sum(current)) == string(newHash.Sum(d)) { + continue + } + } + + level.Info(util.Logger).Log("msg", "updating rule file", "file", file) + err = afero.WriteFile(m.FS, file, d, 0777) + if err != nil { + return false, nil, err + } + update = true + filenames = append(filenames, file) + } + return update, filenames, nil +} diff --git a/pkg/ruler/mapper_test.go b/pkg/ruler/mapper_test.go new file mode 100644 index 00000000000..aea838c55b8 --- /dev/null +++ b/pkg/ruler/mapper_test.go @@ -0,0 +1,130 @@ +package ruler + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/pkg/rulefmt" + "github.com/spf13/afero" +) + +var ( + testUser = "user1" + + initialRuleSet = map[string][]rulefmt.RuleGroup{ + "file_one": { + { + Name: "rulegroup_one", + Rules: []rulefmt.Rule{ + { + Record: "example_rule", + Expr: "example_expr", + }, + }, + }, + { + Name: "rulegroup_two", + Rules: []rulefmt.Rule{ + { + Record: "example_rule", + Expr: "example_expr", + }, + }, + }, + }, + } + + outOfOrderRuleSet = map[string][]rulefmt.RuleGroup{ + "file_one": { + { + Name: "rulegroup_two", + Rules: []rulefmt.Rule{ + { + Record: "example_rule", + Expr: "example_expr", + }, + }, + }, + { + Name: "rulegroup_one", + Rules: []rulefmt.Rule{ + { + Record: "example_rule", + Expr: "example_expr", + }, + }, + }, + }, + } + + updatedRuleSet = map[string][]rulefmt.RuleGroup{ + "file_one": { + { + Name: "rulegroup_one", + Rules: []rulefmt.Rule{ + { + Record: "example_rule", + Expr: "example_expr", + }, + }, + }, + { + Name: "rulegroup_two", + Rules: []rulefmt.Rule{ + { + Record: "example_rule", + Expr: "example_expr", + }, + }, + }, + { + Name: "rulegroup_three", + Rules: []rulefmt.Rule{ + { + Record: "example_rule", + Expr: "example_expr", + }, + }, + }, + }, + } +) + +func Test_mapper_MapRules(t *testing.T) { + m := &mapper{ + Path: "/rules", + FS: afero.NewMemMapFs(), + } + + t.Run("basic rulegroup", func(t *testing.T) { + updated, files, err := m.MapRules(testUser, initialRuleSet) + require.True(t, updated) + require.Len(t, files, 1) + require.Equal(t, "/rules/user1/file_one", files[0]) + require.NoError(t, err) + }) + + t.Run("identical rulegroup", func(t *testing.T) { + updated, files, err := m.MapRules(testUser, initialRuleSet) + require.False(t, updated) + require.Len(t, files, 0) + require.NoError(t, err) + }) + + t.Run("out of order identical rulegroup", func(t *testing.T) { + updated, files, err := m.MapRules(testUser, outOfOrderRuleSet) + require.False(t, updated) + require.Len(t, files, 0) + require.NoError(t, err) + + }) + + t.Run("updated rulegroup", func(t *testing.T) { + updated, files, err := m.MapRules(testUser, updatedRuleSet) + require.True(t, updated) + require.Len(t, files, 1) + require.Equal(t, "/rules/user1/file_one", files[0]) + require.NoError(t, err) + }) +} diff --git a/pkg/ruler/mock_store.go b/pkg/ruler/mock_store.go new file mode 100644 index 00000000000..07dab3048d2 --- /dev/null +++ b/pkg/ruler/mock_store.go @@ -0,0 +1,41 @@ +package ruler + +import ( + "context" + "strings" + "sync" + + "github.com/cortexproject/cortex/pkg/ruler/rules" +) + +type mockRuleStore struct { + sync.Mutex + rules map[string]*rules.RuleGroupDesc +} + +// RuleStore returns an RuleStore from the client +func (m *mockRuleStore) RuleStore() rules.RuleStore { + return m +} + +func (m *mockRuleStore) ListAllRuleGroups(ctx context.Context) (map[string]rules.RuleGroupList, error) { + m.Lock() + defer m.Unlock() + + userGroupMap := map[string]rules.RuleGroupList{} + + for id, rg := range m.rules { + components := strings.Split(id, ":") + if len(components) != 3 { + continue + } + user := components[0] + + if _, exists := userGroupMap[user]; !exists { + userGroupMap[user] = rules.RuleGroupList{} + } + userGroupMap[user] = append(userGroupMap[user], rg) + } + + return userGroupMap, nil +} diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 9f99ed12cca..c7c202b6d78 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -3,7 +3,7 @@ package ruler import ( native_ctx "context" "flag" - "fmt" + "hash/fnv" "net/http" "net/url" "sync" @@ -16,72 +16,49 @@ import ( "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/notifier" "github.com/prometheus/prometheus/promql" - "github.com/prometheus/prometheus/rules" - "github.com/prometheus/prometheus/storage" + promRules "github.com/prometheus/prometheus/rules" + promStorage "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/strutil" "golang.org/x/net/context" "golang.org/x/net/context/ctxhttp" - "github.com/cortexproject/cortex/pkg/configs/client" "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/ring" + "github.com/cortexproject/cortex/pkg/ruler/rules" + store "github.com/cortexproject/cortex/pkg/ruler/rules" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" - "github.com/weaveworks/common/instrument" "github.com/weaveworks/common/user" ) var ( - evalDuration = instrument.NewHistogramCollectorFromOpts(prometheus.HistogramOpts{ - Namespace: "cortex", - Name: "group_evaluation_duration_seconds", - Help: "The duration for a rule group to execute.", - Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 25}, - }) - rulesProcessed = promauto.NewCounter(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "rules_processed_total", - Help: "How many rules have been processed.", - }) ringCheckErrors = promauto.NewCounter(prometheus.CounterOpts{ Namespace: "cortex", Name: "ruler_ring_check_errors_total", Help: "Number of errors that have occurred when checking the ring for ownership", }) - ruleMetrics *rules.Metrics + configUpdatesTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "cortex", + Name: "ruler_config_updates_total", + Help: "Total number of config updates triggered by a user", + }, []string{"user"}) ) -func init() { - evalDuration.Register() - ruleMetrics = rules.NewGroupMetrics(prometheus.DefaultRegisterer) -} - // Config is the configuration for the recording rules server. type Config struct { - // This is used for template expansion in alerts; must be a valid URL - ExternalURL flagext.URLValue - - // How frequently to evaluate rules by default. - EvaluationInterval time.Duration - NumWorkers int - - // URL of the Alertmanager to send notifications to. - AlertmanagerURL flagext.URLValue - // Whether to use DNS SRV records to discover alertmanagers. - AlertmanagerDiscovery bool - // How long to wait between refreshing the list of alertmanagers based on - // DNS service discovery. - AlertmanagerRefreshInterval time.Duration - - // Capacity of the queue for notifications to be sent to the Alertmanager. - NotificationQueueCapacity int - // HTTP timeout duration when sending notifications to the Alertmanager. - NotificationTimeout time.Duration - // Timeout for rule group evaluation, including sending result to ingester - GroupTimeout time.Duration - - EnableSharding bool - + ExternalURL flagext.URLValue // This is used for template expansion in alerts; must be a valid URL + EvaluationInterval time.Duration // How frequently to evaluate rules by default. + PollInterval time.Duration // How frequently to poll for updated rules + StoreConfig RuleStoreConfig // Rule Storage and Polling configuration + RulePath string // Path to store rule files for prom manager + + AlertmanagerURL flagext.URLValue // URL of the Alertmanager to send notifications to. + AlertmanagerDiscovery bool // Whether to use DNS SRV records to discover alertmanagers. + AlertmanagerRefreshInterval time.Duration // How long to wait between refreshing the list of alertmanagers based on DNS service discovery. + NotificationQueueCapacity int // Capacity of the queue for notifications to be sent to the Alertmanager. + NotificationTimeout time.Duration // HTTP timeout duration when sending notifications to the Alertmanager. + + EnableSharding bool // Enable sharding rule groups SearchPendingFor time.Duration LifecyclerConfig ring.LifecyclerConfig FlushCheckPeriod time.Duration @@ -90,69 +67,78 @@ type Config struct { // RegisterFlags adds the flags required to config this to the given FlagSet func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.LifecyclerConfig.RegisterFlagsWithPrefix("ruler.", f) + cfg.StoreConfig.RegisterFlags(f) cfg.ExternalURL.URL, _ = url.Parse("") // Must be non-nil f.Var(&cfg.ExternalURL, "ruler.external.url", "URL of alerts return path.") - f.DurationVar(&cfg.EvaluationInterval, "ruler.evaluation-interval", 15*time.Second, "How frequently to evaluate rules") - f.IntVar(&cfg.NumWorkers, "ruler.num-workers", 1, "Number of rule evaluator worker routines in this process") + f.DurationVar(&cfg.EvaluationInterval, "ruler.evaluation-interval", 1*time.Minute, "How frequently to evaluate rules") + f.DurationVar(&cfg.PollInterval, "ruler.poll-interval", 1*time.Minute, "How frequently to poll for rule changes") f.Var(&cfg.AlertmanagerURL, "ruler.alertmanager-url", "URL of the Alertmanager to send notifications to.") f.BoolVar(&cfg.AlertmanagerDiscovery, "ruler.alertmanager-discovery", false, "Use DNS SRV records to discover alertmanager hosts.") f.DurationVar(&cfg.AlertmanagerRefreshInterval, "ruler.alertmanager-refresh-interval", 1*time.Minute, "How long to wait between refreshing alertmanager hosts.") f.IntVar(&cfg.NotificationQueueCapacity, "ruler.notification-queue-capacity", 10000, "Capacity of the queue for notifications to be sent to the Alertmanager.") f.DurationVar(&cfg.NotificationTimeout, "ruler.notification-timeout", 10*time.Second, "HTTP timeout duration when sending notifications to the Alertmanager.") - f.DurationVar(&cfg.GroupTimeout, "ruler.group-timeout", 10*time.Second, "Timeout for rule group evaluation, including sending result to ingester") if flag.Lookup("promql.lookback-delta") == nil { flag.DurationVar(&promql.LookbackDelta, "promql.lookback-delta", promql.LookbackDelta, "Time since the last sample after which a time series is considered stale and ignored by expression evaluations.") } f.DurationVar(&cfg.SearchPendingFor, "ruler.search-pending-for", 5*time.Minute, "Time to spend searching for a pending ruler when shutting down.") f.BoolVar(&cfg.EnableSharding, "ruler.enable-sharding", false, "Distribute rule evaluation using ring backend") f.DurationVar(&cfg.FlushCheckPeriod, "ruler.flush-period", 1*time.Minute, "Period with which to attempt to flush rule groups.") + f.StringVar(&cfg.RulePath, "ruler.rule-path", "/rules", "file path to store temporary rule files for the prometheus rule managers") } // Ruler evaluates rules. type Ruler struct { cfg Config engine *promql.Engine - queryable storage.Queryable + queryable promStorage.Queryable pusher Pusher alertURL *url.URL notifierCfg *config.Config - scheduler *scheduler - workerWG *sync.WaitGroup - lifecycler *ring.Lifecycler ring *ring.Ring + store rules.RuleStore + mapper *mapper + userManagerMtx sync.Mutex + userManagers map[string]*promRules.Manager + // Per-user notifiers with separate queues. notifiersMtx sync.Mutex notifiers map[string]*rulerNotifier + + done chan struct{} + terminated chan struct{} } // NewRuler creates a new ruler from a distributor and chunk store. -func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d *distributor.Distributor, rulesAPI client.Client) (*Ruler, error) { - if cfg.NumWorkers <= 0 { - return nil, fmt.Errorf("must have at least 1 worker, got %d", cfg.NumWorkers) +func NewRuler(cfg Config, engine *promql.Engine, queryable promStorage.Queryable, d *distributor.Distributor) (*Ruler, error) { + ncfg, err := buildNotifierConfig(&cfg) + if err != nil { + return nil, err } - ncfg, err := buildNotifierConfig(&cfg) + ruleStore, err := NewRuleStorage(cfg.StoreConfig) if err != nil { return nil, err } ruler := &Ruler{ - cfg: cfg, - engine: engine, - queryable: queryable, - pusher: d, - alertURL: cfg.ExternalURL.URL, - notifierCfg: ncfg, - notifiers: map[string]*rulerNotifier{}, - workerWG: &sync.WaitGroup{}, + cfg: cfg, + engine: engine, + queryable: queryable, + alertURL: cfg.ExternalURL.URL, + notifierCfg: ncfg, + notifiers: map[string]*rulerNotifier{}, + store: ruleStore, + pusher: d, + mapper: newMapper(cfg.RulePath), + userManagers: map[string]*promRules.Manager{}, + done: make(chan struct{}), + terminated: make(chan struct{}), } - ruler.scheduler = newScheduler(rulesAPI, cfg.EvaluationInterval, cfg.EvaluationInterval, ruler.newGroup, ruler.removeUser) - // If sharding is enabled, create/join a ring to distribute tokens to // the ruler if cfg.EnableSharding { @@ -167,19 +153,7 @@ func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d } } - for i := 0; i < cfg.NumWorkers; i++ { - // initialize each worker in a function that signals when - // the worker has completed - ruler.workerWG.Add(1) - go func() { - w := newWorker(ruler) - w.Run() - ruler.workerWG.Done() - }() - } - - go ruler.scheduler.Run() - + go ruler.run() level.Info(util.Logger).Log("msg", "ruler up and running") return ruler, nil @@ -188,66 +162,50 @@ func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d // Stop stops the Ruler. // Each function of the ruler is terminated before leaving the ring func (r *Ruler) Stop() { + close(r.done) + <-r.terminated + r.notifiersMtx.Lock() for _, n := range r.notifiers { n.stop() } r.notifiersMtx.Unlock() - level.Info(util.Logger).Log("msg", "shutting down rules scheduler") - r.scheduler.Stop() - - level.Info(util.Logger).Log("msg", "waiting for workers to finish") - r.workerWG.Wait() - if r.cfg.EnableSharding { level.Info(util.Logger).Log("msg", "attempting shutdown lifecycle") r.lifecycler.Shutdown() level.Info(util.Logger).Log("msg", "shutting down the ring") r.ring.Stop() } -} - -func (r *Ruler) newGroup(userID string, groupName string, rls []rules.Rule) (*group, error) { - appendable := &appendableAppender{pusher: r.pusher} - notifier, err := r.getOrCreateNotifier(userID) - if err != nil { - return nil, err - } - opts := &rules.ManagerOptions{ - Appendable: appendable, - QueryFunc: rules.EngineQueryFunc(r.engine, r.queryable), - Context: context.Background(), - ExternalURL: r.alertURL, - NotifyFunc: sendAlerts(notifier, r.alertURL.String()), - Logger: util.Logger, - Metrics: ruleMetrics, - } - return newGroup(groupName, rls, appendable, opts), nil -} -func (r *Ruler) removeUser(userID string) error { - r.notifiersMtx.Lock() - defer r.notifiersMtx.Unlock() - - if n, ok := r.notifiers[userID]; ok { - n.stop() + level.Info(util.Logger).Log("msg", "stopping user managers") + wg := sync.WaitGroup{} + r.userManagerMtx.Lock() + for user, manager := range r.userManagers { + level.Debug(util.Logger).Log("msg", "shutting down user manager", "user", user) + go func(manager *promRules.Manager, user string) { + wg.Add(1) + manager.Stop() + wg.Done() + level.Debug(util.Logger).Log("msg", "user manager shut down", "user", user) + }(manager, user) } - delete(r.notifiers, userID) - return nil + wg.Wait() + r.userManagerMtx.Unlock() + level.Info(util.Logger).Log("msg", "all user managers stopped") } // sendAlerts implements a rules.NotifyFunc for a Notifier. // It filters any non-firing alerts from the input. // // Copied from Prometheus's main.go. -func sendAlerts(n *notifier.Manager, externalURL string) rules.NotifyFunc { - return func(ctx native_ctx.Context, expr string, alerts ...*rules.Alert) { +func sendAlerts(n *notifier.Manager, externalURL string) promRules.NotifyFunc { + return func(ctx native_ctx.Context, expr string, alerts ...*promRules.Alert) { var res []*notifier.Alert for _, alert := range alerts { // Only send actually firing alerts. - if alert.State == rules.StatePending { + if alert.State == promRules.StatePending { continue } a := ¬ifier.Alert{ @@ -307,48 +265,19 @@ func (r *Ruler) getOrCreateNotifier(userID string) (*notifier.Manager, error) { return n.notifier, nil } -// Evaluate a list of rules in the given context. -func (r *Ruler) Evaluate(userID string, item *workItem) { - ctx := user.InjectOrgID(context.Background(), userID) - logger := util.WithContext(ctx, util.Logger) - if r.cfg.EnableSharding && !r.ownsRule(item.hash) { - level.Debug(util.Logger).Log("msg", "ruler: skipping evaluation, not owned", "user_id", item.userID, "group", item.groupName) - return - } - level.Debug(logger).Log("msg", "evaluating rules...", "num_rules", len(item.group.Rules())) - ctx, cancelTimeout := context.WithTimeout(ctx, r.cfg.GroupTimeout) - instrument.CollectedRequest(ctx, "Evaluate", evalDuration, nil, func(ctx native_ctx.Context) error { - if span := ot.SpanFromContext(ctx); span != nil { - span.SetTag("instance", userID) - span.SetTag("groupName", item.groupName) - } - item.group.Eval(ctx, time.Now()) - return nil - }) - if err := ctx.Err(); err == nil { - cancelTimeout() // release resources - } else { - level.Warn(logger).Log("msg", "context error", "error", err) - } - - rulesProcessed.Add(float64(len(item.group.Rules()))) -} - -func (r *Ruler) ownsRule(hash uint32) bool { - rlrs, err := r.ring.Get(hash, ring.Read, nil) - // If an error occurs evaluate a rule as if it is owned - // better to have extra datapoints for a rule than none at all - // TODO: add a temporary cache of owned rule values or something to fall back on +func (r *Ruler) ownsRule(hash uint32) (bool, error) { + rlrs, err := r.ring.Get(hash, ring.Read, []ring.IngesterDesc{}) if err != nil { level.Warn(util.Logger).Log("msg", "error reading ring to verify rule group ownership", "err", err) ringCheckErrors.Inc() - return true + return false, err } if rlrs.Ingesters[0].Addr == r.lifecycler.Addr { - return true + level.Debug(util.Logger).Log("msg", "rule group owned", "owner_addr", rlrs.Ingesters[0].Addr, "addr", r.lifecycler.Addr) + return true, nil } - level.Debug(util.Logger).Log("msg", "rule group not owned, address does not match", "owner", rlrs.Ingesters[0].Addr, "current", r.cfg.LifecyclerConfig.Addr) - return false + level.Debug(util.Logger).Log("msg", "rule group not owned, address does not match", "owner_addr", rlrs.Ingesters[0].Addr, "addr", r.lifecycler.Addr) + return false, nil } func (r *Ruler) ServeHTTP(w http.ResponseWriter, req *http.Request) { @@ -368,6 +297,143 @@ func (r *Ruler) ServeHTTP(w http.ResponseWriter, req *http.Request) { ` w.WriteHeader(http.StatusOK) - w.Write([]byte(unshardedPage)) + _, err := w.Write([]byte(unshardedPage)) + if err != nil { + level.Error(util.Logger).Log("msg", "unable to serve status page", "err", err) + } + } +} + +func (r *Ruler) run() { + defer close(r.terminated) + + tick := time.NewTicker(r.cfg.PollInterval) + defer tick.Stop() + + r.loadRules() + for { + select { + case <-r.done: + return + case <-tick.C: + r.loadRules() + } + } +} + +func (r *Ruler) loadRules() { + ringHasher := fnv.New32a() + + configs, err := r.store.ListAllRuleGroups(context.Background()) + if err != nil { + level.Error(util.Logger).Log("msg", "unable to poll for rules", "err", err) + return + } + + // Iterate through each users configuration and determine if the on-disk + // configurations need to be updated + for user, cfg := range configs { + filteredGroups := store.RuleGroupList{} + + // If sharding is enabled, prune the rule group to only contain rules + // this ruler is responsible for. + if r.cfg.EnableSharding { + for _, g := range cfg { + id := g.User + "/" + g.Namespace + "/" + g.Name + ringHasher.Reset() + _, err = ringHasher.Write([]byte(id)) + if err != nil { + level.Error(util.Logger).Log("msg", "failed to create group for user", "user", user, "namespace", g.Namespace, "group", g.Name, "err", err) + continue + } + hash := ringHasher.Sum32() + owned, err := r.ownsRule(hash) + if err != nil { + level.Error(util.Logger).Log("msg", "unable to verify rule group ownership ownership, will retry on the next poll", "err", err) + return + } + if owned { + filteredGroups = append(filteredGroups, g) + } + } + } else { + filteredGroups = cfg + } + + // Map the files to disk and return the file names to be passed to the users manager + update, files, err := r.mapper.MapRules(user, filteredGroups.Formatted()) + if err != nil { + level.Error(util.Logger).Log("msg", "unable to map rule files", "user", user, "err", err) + continue + } + + if update { + configUpdatesTotal.WithLabelValues(user).Inc() + r.userManagerMtx.Lock() + manager, exists := r.userManagers[user] + r.userManagerMtx.Unlock() + if !exists { + manager, err = r.newManager(user) + if err != nil { + level.Error(util.Logger).Log("msg", "unable to create rule manager", "user", user, "err", err) + continue + } + manager.Run() + + r.userManagerMtx.Lock() + r.userManagers[user] = manager + r.userManagerMtx.Unlock() + } + err = manager.Update(r.cfg.EvaluationInterval, files, nil) + if err != nil { + level.Error(util.Logger).Log("msg", "unable to create rule manager", "user", user, "err", err) + continue + } + } + } + + // Check for deleted users and remove them + r.userManagerMtx.Lock() + defer r.userManagerMtx.Unlock() + for user, mngr := range r.userManagers { + if _, exists := configs[user]; !exists { + go mngr.Stop() + delete(r.userManagers, user) + level.Info(util.Logger).Log("msg", "deleting rule manager", "user", user) + } + } + +} + +// newManager creates a prometheus rule manager wrapped with a user id +// configured storage, appendable, notifier, and instrumentation +func (r *Ruler) newManager(userID string) (*promRules.Manager, error) { + db := &tsdb{ + appender: &appendableAppender{ + pusher: r.pusher, + userID: userID, + }, + queryable: r.queryable, + } + + notifier, err := r.getOrCreateNotifier(userID) + if err != nil { + return nil, err + } + + // Wrap registerer with userID and cortex_ prefix + reg := prometheus.WrapRegistererWith(prometheus.Labels{"user": userID}, prometheus.DefaultRegisterer) + reg = prometheus.WrapRegistererWithPrefix("cortex_", reg) + + opts := &promRules.ManagerOptions{ + Appendable: db, + TSDB: db, + QueryFunc: promRules.EngineQueryFunc(r.engine, r.queryable), + Context: user.InjectOrgID(context.Background(), userID), + ExternalURL: r.alertURL, + NotifyFunc: sendAlerts(notifier, r.alertURL.String()), + Logger: util.Logger, + Registerer: reg, } + return promRules.NewManager(opts), nil } diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index 51725da5c7b..f2f57e037b8 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -1,42 +1,34 @@ package ruler import ( - "context" "net/http" "net/http/httptest" "sync" "testing" "time" - "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/promql" + "github.com/stretchr/testify/require" - "github.com/cortexproject/cortex/pkg/configs" - client_config "github.com/cortexproject/cortex/pkg/configs/client" "github.com/cortexproject/cortex/pkg/querier" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ring/kv/codec" "github.com/cortexproject/cortex/pkg/ring/kv/consul" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/prometheus/prometheus/notifier" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/promql" "github.com/stretchr/testify/assert" "github.com/weaveworks/common/user" ) -type mockRuleStore struct{} - -func (m *mockRuleStore) GetRules(ctx context.Context, since configs.ID) (map[string]configs.VersionedRulesConfig, error) { - return map[string]configs.VersionedRulesConfig{}, nil -} - -func (m *mockRuleStore) GetAlerts(ctx context.Context, since configs.ID) (*client_config.ConfigsResponse, error) { - return nil, nil -} - func defaultRulerConfig() Config { codec := codec.Proto{Factory: ring.ProtoDescFactory} consul := consul.NewInMemoryClient(codec) - cfg := Config{} + cfg := Config{ + StoreConfig: RuleStoreConfig{ + mock: &mockRuleStore{}, + }, + } flagext.DefaultValues(&cfg) flagext.DefaultValues(&cfg.LifecyclerConfig) cfg.LifecyclerConfig.RingConfig.ReplicationFactor = 1 @@ -59,7 +51,7 @@ func newTestRuler(t *testing.T, cfg Config) *Ruler { Timeout: 2 * time.Minute, }) queryable := querier.NewQueryable(nil, nil, nil, 0) - ruler, err := NewRuler(cfg, engine, queryable, nil, &mockRuleStore{}) + ruler, err := NewRuler(cfg, engine, queryable, nil) if err != nil { t.Fatal(err) } @@ -78,7 +70,8 @@ func TestNotifierSendsUserIDHeader(t *testing.T) { defer ts.Close() cfg := defaultRulerConfig() - cfg.AlertmanagerURL.Set(ts.URL) + err := cfg.AlertmanagerURL.Set(ts.URL) + require.NoError(t, err) cfg.AlertmanagerDiscovery = false r := newTestRuler(t, cfg) diff --git a/pkg/ruler/rules/compat.go b/pkg/ruler/rules/compat.go new file mode 100644 index 00000000000..5e535c1cffb --- /dev/null +++ b/pkg/ruler/rules/compat.go @@ -0,0 +1,65 @@ +package rules + +import ( + time "time" + + "github.com/cortexproject/cortex/pkg/ingester/client" + + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/rulefmt" +) + +// ToProto transforms a formatted prometheus rulegroup to a rule group protobuf +func ToProto(user string, namespace string, rl rulefmt.RuleGroup) *RuleGroupDesc { + dur := time.Duration(rl.Interval) + rg := RuleGroupDesc{ + Name: rl.Name, + Namespace: namespace, + Interval: &dur, + Rules: formattedRuleToProto(rl.Rules), + User: user, + } + return &rg +} + +func formattedRuleToProto(rls []rulefmt.Rule) []*RuleDesc { + rules := make([]*RuleDesc, len(rls)) + for i := range rls { + f := time.Duration(rls[i].For) + + rules[i] = &RuleDesc{ + Expr: rls[i].Expr, + Record: rls[i].Record, + Alert: rls[i].Alert, + + For: &f, + Labels: client.FromLabelsToLabelAdapaters(labels.FromMap(rls[i].Labels)), + Annotations: client.FromLabelsToLabelAdapaters(labels.FromMap(rls[i].Annotations)), + } + } + + return rules +} + +// FromProto generates a rulefmt RuleGroup +func FromProto(rg *RuleGroupDesc) rulefmt.RuleGroup { + formattedRuleGroup := rulefmt.RuleGroup{ + Name: rg.GetName(), + Interval: model.Duration(*rg.Interval), + Rules: make([]rulefmt.Rule, len(rg.GetRules())), + } + + for i, rl := range rg.GetRules() { + formattedRuleGroup.Rules[i] = rulefmt.Rule{ + Record: rl.GetRecord(), + Alert: rl.GetAlert(), + Expr: rl.GetExpr(), + For: model.Duration(*rl.GetFor()), + Labels: client.FromLabelAdaptersToLabels(rl.Labels).Map(), + Annotations: client.FromLabelAdaptersToLabels(rl.Annotations).Map(), + } + } + + return formattedRuleGroup +} diff --git a/pkg/ruler/rules/rules.pb.go b/pkg/ruler/rules/rules.pb.go new file mode 100644 index 00000000000..d4c15283775 --- /dev/null +++ b/pkg/ruler/rules/rules.pb.go @@ -0,0 +1,1491 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: rules.proto + +package rules + +import ( + fmt "fmt" + _ "github.com/cortexproject/cortex/pkg/ingester/client" + github_com_cortexproject_cortex_pkg_ingester_client "github.com/cortexproject/cortex/pkg/ingester/client" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + github_com_gogo_protobuf_types "github.com/gogo/protobuf/types" + _ "github.com/golang/protobuf/ptypes/duration" + io "io" + math "math" + reflect "reflect" + strings "strings" + time "time" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf +var _ = time.Kitchen + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +type RuleUpdateDesc struct { + User string `protobuf:"bytes,1,opt,name=user,proto3" json:"user,omitempty"` + UpdatedAt int64 `protobuf:"varint,2,opt,name=updatedAt,proto3" json:"updatedAt,omitempty"` +} + +func (m *RuleUpdateDesc) Reset() { *m = RuleUpdateDesc{} } +func (*RuleUpdateDesc) ProtoMessage() {} +func (*RuleUpdateDesc) Descriptor() ([]byte, []int) { + return fileDescriptor_8e722d3e922f0937, []int{0} +} +func (m *RuleUpdateDesc) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RuleUpdateDesc) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_RuleUpdateDesc.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *RuleUpdateDesc) XXX_Merge(src proto.Message) { + xxx_messageInfo_RuleUpdateDesc.Merge(m, src) +} +func (m *RuleUpdateDesc) XXX_Size() int { + return m.Size() +} +func (m *RuleUpdateDesc) XXX_DiscardUnknown() { + xxx_messageInfo_RuleUpdateDesc.DiscardUnknown(m) +} + +var xxx_messageInfo_RuleUpdateDesc proto.InternalMessageInfo + +func (m *RuleUpdateDesc) GetUser() string { + if m != nil { + return m.User + } + return "" +} + +func (m *RuleUpdateDesc) GetUpdatedAt() int64 { + if m != nil { + return m.UpdatedAt + } + return 0 +} + +type RuleGroupDesc struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Namespace string `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` + Interval *time.Duration `protobuf:"bytes,3,opt,name=interval,proto3,stdduration" json:"interval,omitempty"` + Rules []*RuleDesc `protobuf:"bytes,4,rep,name=rules,proto3" json:"rules,omitempty"` + Deleted bool `protobuf:"varint,5,opt,name=deleted,proto3" json:"deleted,omitempty"` + User string `protobuf:"bytes,6,opt,name=user,proto3" json:"user,omitempty"` +} + +func (m *RuleGroupDesc) Reset() { *m = RuleGroupDesc{} } +func (*RuleGroupDesc) ProtoMessage() {} +func (*RuleGroupDesc) Descriptor() ([]byte, []int) { + return fileDescriptor_8e722d3e922f0937, []int{1} +} +func (m *RuleGroupDesc) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RuleGroupDesc) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_RuleGroupDesc.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *RuleGroupDesc) XXX_Merge(src proto.Message) { + xxx_messageInfo_RuleGroupDesc.Merge(m, src) +} +func (m *RuleGroupDesc) XXX_Size() int { + return m.Size() +} +func (m *RuleGroupDesc) XXX_DiscardUnknown() { + xxx_messageInfo_RuleGroupDesc.DiscardUnknown(m) +} + +var xxx_messageInfo_RuleGroupDesc proto.InternalMessageInfo + +func (m *RuleGroupDesc) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *RuleGroupDesc) GetNamespace() string { + if m != nil { + return m.Namespace + } + return "" +} + +func (m *RuleGroupDesc) GetInterval() *time.Duration { + if m != nil { + return m.Interval + } + return nil +} + +func (m *RuleGroupDesc) GetRules() []*RuleDesc { + if m != nil { + return m.Rules + } + return nil +} + +func (m *RuleGroupDesc) GetDeleted() bool { + if m != nil { + return m.Deleted + } + return false +} + +func (m *RuleGroupDesc) GetUser() string { + if m != nil { + return m.User + } + return "" +} + +type RuleDesc struct { + Expr string `protobuf:"bytes,1,opt,name=expr,proto3" json:"expr,omitempty"` + Record string `protobuf:"bytes,2,opt,name=record,proto3" json:"record,omitempty"` + Alert string `protobuf:"bytes,3,opt,name=alert,proto3" json:"alert,omitempty"` + For *time.Duration `protobuf:"bytes,4,opt,name=for,proto3,stdduration" json:"for,omitempty"` + Labels []github_com_cortexproject_cortex_pkg_ingester_client.LabelAdapter `protobuf:"bytes,5,rep,name=labels,proto3,customtype=github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter" json:"labels"` + Annotations []github_com_cortexproject_cortex_pkg_ingester_client.LabelAdapter `protobuf:"bytes,6,rep,name=annotations,proto3,customtype=github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter" json:"annotations"` +} + +func (m *RuleDesc) Reset() { *m = RuleDesc{} } +func (*RuleDesc) ProtoMessage() {} +func (*RuleDesc) Descriptor() ([]byte, []int) { + return fileDescriptor_8e722d3e922f0937, []int{2} +} +func (m *RuleDesc) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RuleDesc) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_RuleDesc.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *RuleDesc) XXX_Merge(src proto.Message) { + xxx_messageInfo_RuleDesc.Merge(m, src) +} +func (m *RuleDesc) XXX_Size() int { + return m.Size() +} +func (m *RuleDesc) XXX_DiscardUnknown() { + xxx_messageInfo_RuleDesc.DiscardUnknown(m) +} + +var xxx_messageInfo_RuleDesc proto.InternalMessageInfo + +func (m *RuleDesc) GetExpr() string { + if m != nil { + return m.Expr + } + return "" +} + +func (m *RuleDesc) GetRecord() string { + if m != nil { + return m.Record + } + return "" +} + +func (m *RuleDesc) GetAlert() string { + if m != nil { + return m.Alert + } + return "" +} + +func (m *RuleDesc) GetFor() *time.Duration { + if m != nil { + return m.For + } + return nil +} + +func init() { + proto.RegisterType((*RuleUpdateDesc)(nil), "rules.RuleUpdateDesc") + proto.RegisterType((*RuleGroupDesc)(nil), "rules.RuleGroupDesc") + proto.RegisterType((*RuleDesc)(nil), "rules.RuleDesc") +} + +func init() { proto.RegisterFile("rules.proto", fileDescriptor_8e722d3e922f0937) } + +var fileDescriptor_8e722d3e922f0937 = []byte{ + // 478 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x92, 0xbf, 0x8e, 0xd4, 0x30, + 0x10, 0xc6, 0xe3, 0xdb, 0x3f, 0xec, 0x7a, 0x05, 0x08, 0x0b, 0x21, 0x73, 0x42, 0xde, 0x68, 0x25, + 0xa4, 0x34, 0x24, 0xe2, 0x28, 0xaf, 0xe1, 0x56, 0x27, 0x41, 0x41, 0x81, 0x2c, 0xd1, 0xd0, 0x79, + 0x93, 0xb9, 0x10, 0xf0, 0xc5, 0x91, 0xe3, 0x20, 0x1a, 0x24, 0x1e, 0x81, 0x92, 0x47, 0xe0, 0x51, + 0xae, 0x5c, 0x89, 0xe6, 0x44, 0x71, 0xb0, 0xd9, 0x86, 0xf2, 0x24, 0x5e, 0x00, 0xd9, 0x4e, 0x6e, + 0xaf, 0x44, 0x48, 0x54, 0x99, 0xcf, 0x33, 0xfe, 0x66, 0x7e, 0xe3, 0xe0, 0x99, 0x6e, 0x24, 0xd4, + 0x71, 0xa5, 0x95, 0x51, 0x64, 0xe4, 0xc4, 0xfe, 0xa3, 0xbc, 0x30, 0x6f, 0x9a, 0x55, 0x9c, 0xaa, + 0xd3, 0x24, 0x57, 0xb9, 0x4a, 0x5c, 0x76, 0xd5, 0x9c, 0x38, 0xe5, 0x84, 0x8b, 0xfc, 0xad, 0x7d, + 0x96, 0x2b, 0x95, 0x4b, 0xd8, 0x55, 0x65, 0x8d, 0x16, 0xa6, 0x50, 0x65, 0x97, 0x7f, 0x7a, 0xcd, + 0x2e, 0x55, 0xda, 0xc0, 0x87, 0x4a, 0xab, 0xb7, 0x90, 0x9a, 0x4e, 0x25, 0xd5, 0xbb, 0x3c, 0x29, + 0xca, 0x1c, 0x6a, 0x03, 0x3a, 0x49, 0x65, 0x01, 0x65, 0x9f, 0xf2, 0x0e, 0x8b, 0x25, 0xbe, 0xc5, + 0x1b, 0x09, 0xaf, 0xaa, 0x4c, 0x18, 0x38, 0x86, 0x3a, 0x25, 0x04, 0x0f, 0x9b, 0x1a, 0x34, 0x45, + 0x21, 0x8a, 0xa6, 0xdc, 0xc5, 0xe4, 0x01, 0x9e, 0x36, 0xae, 0x22, 0x3b, 0x32, 0x74, 0x2f, 0x44, + 0xd1, 0x80, 0xef, 0x0e, 0x16, 0xdf, 0x10, 0xbe, 0x69, 0x4d, 0x9e, 0x69, 0xd5, 0x54, 0xbd, 0x47, + 0x29, 0x4e, 0xa1, 0xf7, 0xb0, 0xb1, 0xf5, 0xb0, 0xdf, 0xba, 0x12, 0x29, 0x38, 0x8f, 0x29, 0xdf, + 0x1d, 0x90, 0x43, 0x3c, 0x29, 0x4a, 0x03, 0xfa, 0xbd, 0x90, 0x74, 0x10, 0xa2, 0x68, 0x76, 0x70, + 0x3f, 0xf6, 0xf0, 0x71, 0x0f, 0x1f, 0x1f, 0x77, 0xf0, 0xcb, 0xe1, 0x97, 0x1f, 0x73, 0xc4, 0xaf, + 0x2e, 0x90, 0x87, 0xd8, 0xaf, 0x97, 0x0e, 0xc3, 0x41, 0x34, 0x3b, 0xb8, 0x1d, 0xfb, 0xcd, 0xdb, + 0x99, 0xec, 0x38, 0xdc, 0x67, 0x09, 0xc5, 0x37, 0x32, 0x90, 0x60, 0x20, 0xa3, 0xa3, 0x10, 0x45, + 0x13, 0xde, 0xcb, 0x2b, 0xe6, 0xf1, 0x8e, 0x79, 0xf1, 0x7b, 0x0f, 0x4f, 0x7a, 0x07, 0x5b, 0x60, + 0x77, 0xdb, 0x03, 0xd9, 0x98, 0xdc, 0xc3, 0x63, 0x0d, 0xa9, 0xd2, 0x59, 0x47, 0xd3, 0x29, 0x72, + 0x17, 0x8f, 0x84, 0x04, 0x6d, 0x1c, 0xc7, 0x94, 0x7b, 0x41, 0x1e, 0xe3, 0xc1, 0x89, 0xd2, 0x74, + 0xf8, 0x77, 0x6c, 0xb6, 0x96, 0xd4, 0x78, 0x2c, 0xc5, 0x0a, 0x64, 0x4d, 0x47, 0x8e, 0xeb, 0x4e, + 0xdc, 0x3d, 0xdd, 0x0b, 0x7b, 0xfa, 0x52, 0x14, 0x7a, 0xf9, 0xfc, 0xec, 0x62, 0x1e, 0x7c, 0xbf, + 0x98, 0xff, 0xcb, 0x8f, 0xe0, 0x6d, 0x8e, 0x32, 0x51, 0x19, 0xd0, 0xbc, 0x6b, 0x45, 0x3e, 0xe2, + 0x99, 0x28, 0x4b, 0x65, 0xdc, 0x34, 0x35, 0x1d, 0xff, 0xff, 0xce, 0xd7, 0xfb, 0x2d, 0x0f, 0xd7, + 0x1b, 0x16, 0x9c, 0x6f, 0x58, 0x70, 0xb9, 0x61, 0xe8, 0x53, 0xcb, 0xd0, 0xd7, 0x96, 0xa1, 0xb3, + 0x96, 0xa1, 0x75, 0xcb, 0xd0, 0xcf, 0x96, 0xa1, 0x5f, 0x2d, 0x0b, 0x2e, 0x5b, 0x86, 0x3e, 0x6f, + 0x59, 0xb0, 0xde, 0xb2, 0xe0, 0x7c, 0xcb, 0x82, 0xd7, 0xfe, 0x81, 0x57, 0x63, 0xb7, 0xce, 0x27, + 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x70, 0xb7, 0x00, 0xbd, 0x7a, 0x03, 0x00, 0x00, +} + +func (this *RuleUpdateDesc) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*RuleUpdateDesc) + if !ok { + that2, ok := that.(RuleUpdateDesc) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.User != that1.User { + return false + } + if this.UpdatedAt != that1.UpdatedAt { + return false + } + return true +} +func (this *RuleGroupDesc) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*RuleGroupDesc) + if !ok { + that2, ok := that.(RuleGroupDesc) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Name != that1.Name { + return false + } + if this.Namespace != that1.Namespace { + return false + } + if this.Interval != nil && that1.Interval != nil { + if *this.Interval != *that1.Interval { + return false + } + } else if this.Interval != nil { + return false + } else if that1.Interval != nil { + return false + } + if len(this.Rules) != len(that1.Rules) { + return false + } + for i := range this.Rules { + if !this.Rules[i].Equal(that1.Rules[i]) { + return false + } + } + if this.Deleted != that1.Deleted { + return false + } + if this.User != that1.User { + return false + } + return true +} +func (this *RuleDesc) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*RuleDesc) + if !ok { + that2, ok := that.(RuleDesc) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Expr != that1.Expr { + return false + } + if this.Record != that1.Record { + return false + } + if this.Alert != that1.Alert { + return false + } + if this.For != nil && that1.For != nil { + if *this.For != *that1.For { + return false + } + } else if this.For != nil { + return false + } else if that1.For != nil { + return false + } + if len(this.Labels) != len(that1.Labels) { + return false + } + for i := range this.Labels { + if !this.Labels[i].Equal(that1.Labels[i]) { + return false + } + } + if len(this.Annotations) != len(that1.Annotations) { + return false + } + for i := range this.Annotations { + if !this.Annotations[i].Equal(that1.Annotations[i]) { + return false + } + } + return true +} +func (this *RuleUpdateDesc) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 6) + s = append(s, "&rules.RuleUpdateDesc{") + s = append(s, "User: "+fmt.Sprintf("%#v", this.User)+",\n") + s = append(s, "UpdatedAt: "+fmt.Sprintf("%#v", this.UpdatedAt)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func (this *RuleGroupDesc) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 10) + s = append(s, "&rules.RuleGroupDesc{") + s = append(s, "Name: "+fmt.Sprintf("%#v", this.Name)+",\n") + s = append(s, "Namespace: "+fmt.Sprintf("%#v", this.Namespace)+",\n") + s = append(s, "Interval: "+fmt.Sprintf("%#v", this.Interval)+",\n") + if this.Rules != nil { + s = append(s, "Rules: "+fmt.Sprintf("%#v", this.Rules)+",\n") + } + s = append(s, "Deleted: "+fmt.Sprintf("%#v", this.Deleted)+",\n") + s = append(s, "User: "+fmt.Sprintf("%#v", this.User)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func (this *RuleDesc) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 10) + s = append(s, "&rules.RuleDesc{") + s = append(s, "Expr: "+fmt.Sprintf("%#v", this.Expr)+",\n") + s = append(s, "Record: "+fmt.Sprintf("%#v", this.Record)+",\n") + s = append(s, "Alert: "+fmt.Sprintf("%#v", this.Alert)+",\n") + s = append(s, "For: "+fmt.Sprintf("%#v", this.For)+",\n") + s = append(s, "Labels: "+fmt.Sprintf("%#v", this.Labels)+",\n") + s = append(s, "Annotations: "+fmt.Sprintf("%#v", this.Annotations)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func valueToGoStringRules(v interface{}, typ string) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) +} +func (m *RuleUpdateDesc) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RuleUpdateDesc) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.User) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintRules(dAtA, i, uint64(len(m.User))) + i += copy(dAtA[i:], m.User) + } + if m.UpdatedAt != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintRules(dAtA, i, uint64(m.UpdatedAt)) + } + return i, nil +} + +func (m *RuleGroupDesc) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RuleGroupDesc) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintRules(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if len(m.Namespace) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintRules(dAtA, i, uint64(len(m.Namespace))) + i += copy(dAtA[i:], m.Namespace) + } + if m.Interval != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintRules(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdDuration(*m.Interval))) + n1, err := github_com_gogo_protobuf_types.StdDurationMarshalTo(*m.Interval, dAtA[i:]) + if err != nil { + return 0, err + } + i += n1 + } + if len(m.Rules) > 0 { + for _, msg := range m.Rules { + dAtA[i] = 0x22 + i++ + i = encodeVarintRules(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.Deleted { + dAtA[i] = 0x28 + i++ + if m.Deleted { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if len(m.User) > 0 { + dAtA[i] = 0x32 + i++ + i = encodeVarintRules(dAtA, i, uint64(len(m.User))) + i += copy(dAtA[i:], m.User) + } + return i, nil +} + +func (m *RuleDesc) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RuleDesc) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Expr) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintRules(dAtA, i, uint64(len(m.Expr))) + i += copy(dAtA[i:], m.Expr) + } + if len(m.Record) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintRules(dAtA, i, uint64(len(m.Record))) + i += copy(dAtA[i:], m.Record) + } + if len(m.Alert) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintRules(dAtA, i, uint64(len(m.Alert))) + i += copy(dAtA[i:], m.Alert) + } + if m.For != nil { + dAtA[i] = 0x22 + i++ + i = encodeVarintRules(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdDuration(*m.For))) + n2, err := github_com_gogo_protobuf_types.StdDurationMarshalTo(*m.For, dAtA[i:]) + if err != nil { + return 0, err + } + i += n2 + } + if len(m.Labels) > 0 { + for _, msg := range m.Labels { + dAtA[i] = 0x2a + i++ + i = encodeVarintRules(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if len(m.Annotations) > 0 { + for _, msg := range m.Annotations { + dAtA[i] = 0x32 + i++ + i = encodeVarintRules(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func encodeVarintRules(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *RuleUpdateDesc) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.User) + if l > 0 { + n += 1 + l + sovRules(uint64(l)) + } + if m.UpdatedAt != 0 { + n += 1 + sovRules(uint64(m.UpdatedAt)) + } + return n +} + +func (m *RuleGroupDesc) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovRules(uint64(l)) + } + l = len(m.Namespace) + if l > 0 { + n += 1 + l + sovRules(uint64(l)) + } + if m.Interval != nil { + l = github_com_gogo_protobuf_types.SizeOfStdDuration(*m.Interval) + n += 1 + l + sovRules(uint64(l)) + } + if len(m.Rules) > 0 { + for _, e := range m.Rules { + l = e.Size() + n += 1 + l + sovRules(uint64(l)) + } + } + if m.Deleted { + n += 2 + } + l = len(m.User) + if l > 0 { + n += 1 + l + sovRules(uint64(l)) + } + return n +} + +func (m *RuleDesc) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Expr) + if l > 0 { + n += 1 + l + sovRules(uint64(l)) + } + l = len(m.Record) + if l > 0 { + n += 1 + l + sovRules(uint64(l)) + } + l = len(m.Alert) + if l > 0 { + n += 1 + l + sovRules(uint64(l)) + } + if m.For != nil { + l = github_com_gogo_protobuf_types.SizeOfStdDuration(*m.For) + n += 1 + l + sovRules(uint64(l)) + } + if len(m.Labels) > 0 { + for _, e := range m.Labels { + l = e.Size() + n += 1 + l + sovRules(uint64(l)) + } + } + if len(m.Annotations) > 0 { + for _, e := range m.Annotations { + l = e.Size() + n += 1 + l + sovRules(uint64(l)) + } + } + return n +} + +func sovRules(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozRules(x uint64) (n int) { + return sovRules(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *RuleUpdateDesc) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&RuleUpdateDesc{`, + `User:` + fmt.Sprintf("%v", this.User) + `,`, + `UpdatedAt:` + fmt.Sprintf("%v", this.UpdatedAt) + `,`, + `}`, + }, "") + return s +} +func (this *RuleGroupDesc) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&RuleGroupDesc{`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `Namespace:` + fmt.Sprintf("%v", this.Namespace) + `,`, + `Interval:` + strings.Replace(fmt.Sprintf("%v", this.Interval), "Duration", "duration.Duration", 1) + `,`, + `Rules:` + strings.Replace(fmt.Sprintf("%v", this.Rules), "RuleDesc", "RuleDesc", 1) + `,`, + `Deleted:` + fmt.Sprintf("%v", this.Deleted) + `,`, + `User:` + fmt.Sprintf("%v", this.User) + `,`, + `}`, + }, "") + return s +} +func (this *RuleDesc) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&RuleDesc{`, + `Expr:` + fmt.Sprintf("%v", this.Expr) + `,`, + `Record:` + fmt.Sprintf("%v", this.Record) + `,`, + `Alert:` + fmt.Sprintf("%v", this.Alert) + `,`, + `For:` + strings.Replace(fmt.Sprintf("%v", this.For), "Duration", "duration.Duration", 1) + `,`, + `Labels:` + fmt.Sprintf("%v", this.Labels) + `,`, + `Annotations:` + fmt.Sprintf("%v", this.Annotations) + `,`, + `}`, + }, "") + return s +} +func valueToStringRules(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *RuleUpdateDesc) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RuleUpdateDesc: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RuleUpdateDesc: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field User", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRules + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRules + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.User = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdatedAt", wireType) + } + m.UpdatedAt = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.UpdatedAt |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipRules(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRules + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthRules + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RuleGroupDesc: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RuleGroupDesc: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRules + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRules + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Namespace", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRules + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRules + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Namespace = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Interval", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRules + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRules + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Interval == nil { + m.Interval = new(time.Duration) + } + if err := github_com_gogo_protobuf_types.StdDurationUnmarshal(m.Interval, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Rules", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRules + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRules + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Rules = append(m.Rules, &RuleDesc{}) + if err := m.Rules[len(m.Rules)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Deleted = bool(v != 0) + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field User", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRules + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRules + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.User = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRules(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRules + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthRules + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RuleDesc) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RuleDesc: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RuleDesc: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Expr", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRules + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRules + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Expr = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Record", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRules + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRules + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Record = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Alert", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRules + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRules + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Alert = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field For", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRules + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRules + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.For == nil { + m.For = new(time.Duration) + } + if err := github_com_gogo_protobuf_types.StdDurationUnmarshal(m.For, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRules + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRules + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Labels = append(m.Labels, github_com_cortexproject_cortex_pkg_ingester_client.LabelAdapter{}) + if err := m.Labels[len(m.Labels)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Annotations", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRules + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRules + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Annotations = append(m.Annotations, github_com_cortexproject_cortex_pkg_ingester_client.LabelAdapter{}) + if err := m.Annotations[len(m.Annotations)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRules(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRules + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthRules + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipRules(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRules + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRules + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRules + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthRules + } + iNdEx += length + if iNdEx < 0 { + return 0, ErrInvalidLengthRules + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowRules + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipRules(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + if iNdEx < 0 { + return 0, ErrInvalidLengthRules + } + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthRules = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowRules = fmt.Errorf("proto: integer overflow") +) diff --git a/pkg/ruler/rules/rules.proto b/pkg/ruler/rules/rules.proto new file mode 100644 index 00000000000..1043668d45b --- /dev/null +++ b/pkg/ruler/rules/rules.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; + +package rules; + +option go_package = "rules"; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; +import "google/protobuf/duration.proto"; +import "github.com/cortexproject/cortex/pkg/ingester/client/cortex.proto"; + +option (gogoproto.marshaler_all) = true; +option (gogoproto.unmarshaler_all) = true; + +message RuleGroupDesc { + string name = 1; + string namespace = 2; + google.protobuf.Duration interval = 3 [(gogoproto.stdduration) = true]; + + repeated RuleDesc rules = 4; + + bool deleted = 5; + + string user = 6; + } + + message RuleDesc { + string expr = 1; + string record = 2; + string alert = 3; + google.protobuf.Duration for = 4 [(gogoproto.stdduration) = true]; + repeated cortex.LabelPair labels = 5 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter"];; + repeated cortex.LabelPair annotations = 6 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter"];; + } \ No newline at end of file diff --git a/pkg/ruler/rules/store.go b/pkg/ruler/rules/store.go new file mode 100644 index 00000000000..c96eee34d0f --- /dev/null +++ b/pkg/ruler/rules/store.go @@ -0,0 +1,40 @@ +package rules + +import ( + "context" + "errors" + + "github.com/prometheus/prometheus/pkg/rulefmt" +) + +var ( + // ErrGroupNotFound is returned if a rule group does not exist + ErrGroupNotFound = errors.New("group does not exist") + // ErrGroupNamespaceNotFound is returned if a namespace does not exist + ErrGroupNamespaceNotFound = errors.New("group namespace does not exist") + // ErrUserNotFound is returned if the user does not currently exist + ErrUserNotFound = errors.New("no rule groups found for user") +) + +// RuleStore is used to store and retrieve rules +type RuleStore interface { + ListAllRuleGroups(ctx context.Context) (map[string]RuleGroupList, error) +} + +// RuleGroupList contains a set of rule groups +type RuleGroupList []*RuleGroupDesc + +// Formatted returns the rule group list as a set of formatted rule groups mapped +// by namespace +func (l RuleGroupList) Formatted() map[string][]rulefmt.RuleGroup { + ruleMap := map[string][]rulefmt.RuleGroup{} + for _, g := range l { + if _, exists := ruleMap[g.Namespace]; !exists { + ruleMap[g.Namespace] = []rulefmt.RuleGroup{FromProto(g)} + continue + } + ruleMap[g.Namespace] = append(ruleMap[g.Namespace], FromProto(g)) + + } + return ruleMap +} diff --git a/pkg/ruler/scheduler.go b/pkg/ruler/scheduler.go deleted file mode 100644 index 7424edea4af..00000000000 --- a/pkg/ruler/scheduler.go +++ /dev/null @@ -1,333 +0,0 @@ -package ruler - -import ( - "context" - "fmt" - "hash" - "hash/fnv" - "math" - "sync" - "time" - - "github.com/go-kit/kit/log/level" - "github.com/jonboulle/clockwork" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/prometheus/rules" - - "github.com/cortexproject/cortex/pkg/configs" - config_client "github.com/cortexproject/cortex/pkg/configs/client" - "github.com/cortexproject/cortex/pkg/util" -) - -var backoffConfig = util.BackoffConfig{ - // Backoff for loading initial configuration set. - MinBackoff: 100 * time.Millisecond, - MaxBackoff: 2 * time.Second, -} - -const ( - timeLogFormat = "2006-01-02T15:04:05" -) - -var ( - totalConfigs = promauto.NewGauge(prometheus.GaugeOpts{ - Namespace: "cortex", - Name: "scheduler_configs_total", - Help: "How many configs the scheduler knows about.", - }) - totalRuleGroups = promauto.NewGauge(prometheus.GaugeOpts{ - Namespace: "cortex", - Name: "scheduler_groups_total", - Help: "How many rule groups the scheduler is currently evaluating", - }) - configUpdates = promauto.NewCounter(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "scheduler_config_updates_total", - Help: "How many config updates the scheduler has made.", - }) -) - -type workItem struct { - userID string - groupName string - hash uint32 - group *group - scheduled time.Time - generation configs.ID // a monotonically increasing number used to spot out of date work items -} - -// Key implements ScheduledItem -func (w workItem) Key() string { - return w.userID + ":" + w.groupName -} - -// Scheduled implements ScheduledItem -func (w workItem) Scheduled() time.Time { - return w.scheduled -} - -// Defer returns a work item with updated rules, rescheduled to a later time. -func (w workItem) Defer(interval time.Duration) workItem { - return workItem{w.userID, w.groupName, w.hash, w.group, w.scheduled.Add(interval), w.generation} -} - -func (w workItem) String() string { - return fmt.Sprintf("%s:%s:%d@%s", w.userID, w.groupName, len(w.group.Rules()), w.scheduled.Format(timeLogFormat)) -} - -type userConfig struct { - rules map[string][]rules.Rule - generation configs.ID // a monotonically increasing number used to spot out of date work items -} - -type groupFactory func(userID string, groupName string, rls []rules.Rule) (*group, error) -type removeFunction func(userID string) error - -type scheduler struct { - ruleStore config_client.Client - evaluationInterval time.Duration // how often we re-evaluate each rule set - q *SchedulingQueue - - pollInterval time.Duration // how often we check for new config - - cfgs map[string]userConfig // all rules for all users - latestConfig configs.ID // # of last update received from config - groupFn groupFactory // function to create a new group - removeFn removeFunction // called when a user is deleted - sync.RWMutex - done chan struct{} -} - -// newScheduler makes a new scheduler. -func newScheduler(ruleStore config_client.Client, evaluationInterval, pollInterval time.Duration, groupFn groupFactory, removeFn removeFunction) *scheduler { - return &scheduler{ - ruleStore: ruleStore, - evaluationInterval: evaluationInterval, - pollInterval: pollInterval, - q: NewSchedulingQueue(clockwork.NewRealClock()), - cfgs: map[string]userConfig{}, - groupFn: groupFn, - removeFn: removeFn, - - done: make(chan struct{}), - } -} - -// Run polls the source of configurations for changes. -func (s *scheduler) Run() { - level.Debug(util.Logger).Log("msg", "scheduler started") - - // Load initial set of all configurations before polling for new ones. - s.addNewConfigs(time.Now(), s.loadAllConfigs()) - ticker := time.NewTicker(s.pollInterval) - for { - select { - case now := <-ticker.C: - err := s.updateConfigs(now) - if err != nil { - level.Warn(util.Logger).Log("msg", "scheduler: error updating configs", "err", err) - } - case <-s.done: - ticker.Stop() - level.Debug(util.Logger).Log("msg", "scheduler config polling stopped") - return - } - } -} - -func (s *scheduler) Stop() { - close(s.done) - s.q.Close() - level.Debug(util.Logger).Log("msg", "scheduler stopped") -} - -// Load the full set of configurations from the server, retrying with backoff -// until we can get them. -func (s *scheduler) loadAllConfigs() map[string]configs.VersionedRulesConfig { - backoff := util.NewBackoff(context.Background(), backoffConfig) - for { - cfgs, err := s.poll() - if err == nil { - level.Debug(util.Logger).Log("msg", "scheduler: initial configuration load", "num_configs", len(cfgs)) - return cfgs - } - level.Warn(util.Logger).Log("msg", "scheduler: error fetching all configurations, backing off", "err", err) - backoff.Wait() - } -} - -func (s *scheduler) updateConfigs(now time.Time) error { - cfgs, err := s.poll() - if err != nil { - return err - } - s.addNewConfigs(now, cfgs) - return nil -} - -// poll the configuration server. Not re-entrant. -func (s *scheduler) poll() (map[string]configs.VersionedRulesConfig, error) { - s.Lock() - configID := s.latestConfig - s.Unlock() - - cfgs, err := s.ruleStore.GetRules(context.Background(), configID) // Warning: this will produce an incorrect result if the configID ever overflows - if err != nil { - level.Warn(util.Logger).Log("msg", "scheduler: configs server poll failed", "err", err) - return nil, err - } - s.Lock() - s.latestConfig = getLatestConfigID(cfgs, configID) - s.Unlock() - return cfgs, nil -} - -// getLatestConfigID gets the latest configs ID. -// max [latest, max (map getID cfgs)] -func getLatestConfigID(cfgs map[string]configs.VersionedRulesConfig, latest configs.ID) configs.ID { - ret := latest - for _, config := range cfgs { - if config.ID > ret { - ret = config.ID - } - } - return ret -} - -// computeNextEvalTime Computes when a user's rules should be next evaluated, based on how far we are through an evaluation cycle -func (s *scheduler) computeNextEvalTime(hasher hash.Hash64, now time.Time, userID string) time.Time { - intervalNanos := float64(s.evaluationInterval.Nanoseconds()) - // Compute how far we are into the current evaluation cycle - currentEvalCyclePoint := math.Mod(float64(now.UnixNano()), intervalNanos) - - hasher.Reset() - hasher.Write([]byte(userID)) - offset := math.Mod( - // We subtract our current point in the cycle to cause the entries - // before 'now' to wrap around to the end. - // We don't want this to come out negative, so we add the interval to it - float64(hasher.Sum64())+intervalNanos-currentEvalCyclePoint, - intervalNanos) - return now.Add(time.Duration(int64(offset))) -} - -func (s *scheduler) addNewConfigs(now time.Time, cfgs map[string]configs.VersionedRulesConfig) { - // TODO: instrument how many configs we have, both valid & invalid. - level.Debug(util.Logger).Log("msg", "adding configurations", "num_configs", len(cfgs)) - hasher := fnv.New64a() - s.Lock() - generation := s.latestConfig - s.Unlock() - - for userID, config := range cfgs { - s.addUserConfig(now, hasher, generation, userID, config) - } - - configUpdates.Add(float64(len(cfgs))) - s.Lock() - lenCfgs := len(s.cfgs) - s.Unlock() - totalConfigs.Set(float64(lenCfgs)) -} - -func (s *scheduler) addUserConfig(now time.Time, hasher hash.Hash64, generation configs.ID, userID string, config configs.VersionedRulesConfig) { - rulesByGroup, err := config.Config.Parse() - if err != nil { - // XXX: This means that if a user has a working configuration and - // they submit a broken one, we'll keep processing the last known - // working configuration, and they'll never know. - // TODO: Provide a way of deleting / cancelling recording rules. - level.Warn(util.Logger).Log("msg", "scheduler: invalid Cortex configuration", "user_id", userID, "err", err) - return - } - - level.Info(util.Logger).Log("msg", "scheduler: updating rules for user", "user_id", userID, "num_groups", len(rulesByGroup), "is_deleted", config.IsDeleted()) - s.Lock() - // if deleted remove from map, otherwise - update map - if config.IsDeleted() { - delete(s.cfgs, userID) - s.Unlock() - if s.removeFn != nil { - s.removeFn(userID) - } - return - } - s.cfgs[userID] = userConfig{rules: rulesByGroup, generation: generation} - s.Unlock() - - ringHasher := fnv.New32a() - evalTime := s.computeNextEvalTime(hasher, now, userID) - workItems := []workItem{} - for group, rules := range rulesByGroup { - level.Debug(util.Logger).Log("msg", "scheduler: updating rules for user and group", "user_id", userID, "group", group, "num_rules", len(rules)) - g, err := s.groupFn(userID, group, rules) - if err != nil { - // XXX: similarly to above if a user has a working configuration and - // for some reason we cannot create a group for the new one we'll use - // the last known working configuration - level.Warn(util.Logger).Log("msg", "scheduler: failed to create group for user", "user_id", userID, "group", group, "err", err) - return - } - ringHasher.Reset() - ringHasher.Write([]byte(userID + ":" + group)) - hash := ringHasher.Sum32() - workItems = append(workItems, workItem{userID, group, hash, g, evalTime, generation}) - } - for _, i := range workItems { - totalRuleGroups.Inc() - s.addWorkItem(i) - } -} - -func (s *scheduler) addWorkItem(i workItem) { - select { - case <-s.done: - level.Debug(util.Logger).Log("msg", "scheduler: work item not added, scheduler stopped", "item", i) - return - default: - // The queue is keyed by userID+groupName, so items for existing userID+groupName will be replaced. - s.q.Enqueue(i) - level.Debug(util.Logger).Log("msg", "scheduler: work item added", "item", i) - } -} - -// Get the next scheduled work item, blocking if none. -// -// Call `workItemDone` on the returned item to indicate that it is ready to be -// rescheduled. -func (s *scheduler) nextWorkItem() *workItem { - level.Debug(util.Logger).Log("msg", "scheduler: work item requested, pending...") - // TODO: We are blocking here on the second Dequeue event. Write more - // tests for the scheduling queue. - op := s.q.Dequeue() - if op == nil { - level.Info(util.Logger).Log("msg", "queue closed; no more work items") - return nil - } - item := op.(workItem) - level.Debug(util.Logger).Log("msg", "scheduler: work item granted", "item", item) - return &item -} - -// workItemDone marks the given item as being ready to be rescheduled. -func (s *scheduler) workItemDone(i workItem) { - s.Lock() - config, found := s.cfgs[i.userID] - var currentRules []rules.Rule - if found { - currentRules = config.rules[i.groupName] - } - s.Unlock() - if !found || len(currentRules) == 0 || i.generation < config.generation { - // Warning: this test will produce an incorrect result if the generation ever overflows - level.Debug(util.Logger).Log("msg", "scheduler: stopping item", "user_id", i.userID, "group", i.groupName, "found", found, "len", len(currentRules)) - totalRuleGroups.Dec() - return - } - - next := i.Defer(s.evaluationInterval) - level.Debug(util.Logger).Log("msg", "scheduler: work item rescheduled", "item", i, "time", next.scheduled.Format(timeLogFormat)) - s.addWorkItem(next) -} diff --git a/pkg/ruler/scheduler_test.go b/pkg/ruler/scheduler_test.go deleted file mode 100644 index e4af6bf97c7..00000000000 --- a/pkg/ruler/scheduler_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package ruler - -import ( - "strconv" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/prometheus/prometheus/rules" -) - -type fakeHasher struct { - data *[]byte -} - -func (h *fakeHasher) Write(data []byte) (int, error) { - h.data = &data - return len(data), nil -} -func (h *fakeHasher) Reset() { - h.data = nil -} -func (h *fakeHasher) Size() int { - return 0 -} -func (h *fakeHasher) BlockSize() int { - return 64 -} -func (h *fakeHasher) Sum([]byte) []byte { - return []byte{} -} -func (h *fakeHasher) Sum64() uint64 { - i, _ := strconv.ParseUint(string(*h.data), 10, 64) - return i -} - -func TestSchedulerComputeNextEvalTime(t *testing.T) { - h := fakeHasher{} - // normal intervals are in seconds; this is nanoseconds for the test - s := scheduler{evaluationInterval: 15} - evalTime := func(now, hashResult int64) int64 { - // We use the fake hasher to give us control over the hash output - // so that we can test the wrap-around behaviour of the modulo - fakeUserID := strconv.FormatInt(hashResult, 10) - return s.computeNextEvalTime(&h, time.Unix(0, now), fakeUserID).UnixNano() - } - { - cycleStartTime := int64(30) - cycleOffset := int64(0) // cycleStartTime % s.evaluationInterval - // Check simple case where hash >= current cycle position - assert.Equal(t, cycleStartTime+0, evalTime(cycleStartTime, cycleOffset+0)) - assert.Equal(t, cycleStartTime+1, evalTime(cycleStartTime, cycleOffset+1)) - assert.Equal(t, cycleStartTime+14, evalTime(cycleStartTime, cycleOffset+14)) - // Check things are cyclic - assert.Equal(t, evalTime(cycleStartTime, 0), evalTime(cycleStartTime, int64(s.evaluationInterval))) - } - { - midCycleTime := int64(35) - cycleOffset := int64(5) // midCycleTime % s.evaluationInterval - // Check case where hash can be either greater or less than current cycle position - assert.Equal(t, midCycleTime+0, evalTime(midCycleTime, cycleOffset+0)) - assert.Equal(t, midCycleTime+1, evalTime(midCycleTime, cycleOffset+1)) - assert.Equal(t, midCycleTime+9, evalTime(midCycleTime, cycleOffset+9)) - assert.Equal(t, midCycleTime+10, evalTime(midCycleTime, cycleOffset-5)) - assert.Equal(t, midCycleTime+14, evalTime(midCycleTime, cycleOffset-1)) - } -} - -func TestSchedulerRulesOverlap(t *testing.T) { - s := newScheduler(nil, 15, 15, nil, nil) - userID := "bob" - groupName := "test" - next := time.Now() - - ruleSet := []rules.Rule{ - nil, - } - ruleSets := map[string][]rules.Rule{} - ruleSets[groupName] = ruleSet - - cfg := userConfig{generation: 1, rules: ruleSets} - s.cfgs[userID] = cfg - w1 := workItem{userID: userID, groupName: groupName, scheduled: next, generation: cfg.generation} - s.workItemDone(w1) - item := s.q.Dequeue().(workItem) - assert.Equal(t, w1.generation, item.generation) - - w0 := workItem{userID: userID, groupName: groupName, scheduled: next, generation: cfg.generation - 1} - s.workItemDone(w1) - s.workItemDone(w0) - item = s.q.Dequeue().(workItem) - assert.Equal(t, w1.generation, item.generation) - - s.q.Close() - assert.Equal(t, nil, s.q.Dequeue()) -} diff --git a/pkg/ruler/scheduling_queue.go b/pkg/ruler/scheduling_queue.go deleted file mode 100644 index d4c0233916d..00000000000 --- a/pkg/ruler/scheduling_queue.go +++ /dev/null @@ -1,173 +0,0 @@ -package ruler - -import ( - "container/heap" - "time" - - "github.com/jonboulle/clockwork" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -var ( - itemEvaluationLatency = promauto.NewHistogram(prometheus.HistogramOpts{ - Namespace: "cortex", - Name: "scheduling_queue_item_latency", - Help: "Difference between the items scheduled evaluation time and actual evaluation time", - Buckets: prometheus.DefBuckets, - }) -) - -// ScheduledItem is an item in a queue of scheduled items. -type ScheduledItem interface { - Key() string - // Scheduled returns the earliest possible time the time is available for - // dequeueing. - Scheduled() time.Time -} - -type queueState struct { - items []ScheduledItem - hit map[string]int -} - -// Less implements heap.Interface -func (q queueState) Less(i, j int) bool { - return q.items[i].Scheduled().Before(q.items[j].Scheduled()) -} - -// Pop implements heap.Interface -func (q *queueState) Pop() interface{} { - old := q.items - n := len(old) - x := old[n-1] - delete(q.hit, x.Key()) - q.items = old[0 : n-1] - return x -} - -// Push implements heap.Interface -func (q *queueState) Push(x interface{}) { - n := len(q.items) - y := x.(ScheduledItem) - q.hit[y.Key()] = n - q.items = append(q.items, y) -} - -func (q *queueState) Swap(i, j int) { - q.hit[q.items[i].Key()] = j - q.hit[q.items[j].Key()] = i - q.items[i], q.items[j] = q.items[j], q.items[i] -} - -func (q *queueState) Enqueue(op ScheduledItem) { - key := op.Key() - i, enqueued := q.hit[key] - if enqueued { - q.items[i] = op - heap.Fix(q, i) - } else { - heap.Push(q, op) - } -} - -func (q *queueState) Dequeue() ScheduledItem { - item := heap.Pop(q).(ScheduledItem) - itemEvaluationLatency.Observe(time.Now().Sub(item.Scheduled()).Seconds()) - return item -} - -func (q *queueState) Front() ScheduledItem { - return q.items[0] -} - -func (q *queueState) Len() int { - return len(q.items) -} - -// SchedulingQueue is like a priority queue, but the first item is the oldest -// scheduled item. Items are only able to be dequeued after the time they are -// scheduled to be run. -type SchedulingQueue struct { - clock clockwork.Clock - add, next chan ScheduledItem -} - -// NewSchedulingQueue makes a new scheduling queue. -func NewSchedulingQueue(clock clockwork.Clock) *SchedulingQueue { - sq := &SchedulingQueue{ - clock: clock, - add: make(chan ScheduledItem), - next: make(chan ScheduledItem), - } - go sq.run() - return sq -} - -// Close the scheduling queue. No more items can be added. Items can be -// dequeued until there are none left. -func (sq *SchedulingQueue) Close() { - close(sq.add) -} - -func (sq *SchedulingQueue) run() { - q := queueState{ - items: []ScheduledItem{}, - hit: map[string]int{}, - } - - for { - // Nothing on the queue? Wait for something to be added. - if q.Len() == 0 { - next, open := <-sq.add - - // If sq.add is closed (and there is nothing on the queue), - // we can close sq.next and stop this goroutine - if !open { - close(sq.next) - return - } - - q.Enqueue(next) - continue - } - - next := q.Front() - delay := next.Scheduled().Sub(sq.clock.Now()) - - // Item on the queue that is ready now? - if delay <= 0 { - select { - case sq.next <- next: - q.Dequeue() - case justAdded, open := <-sq.add: - if open { - q.Enqueue(justAdded) - } - } - continue - } - - // Item on the queue that needs waiting for? - // Wait on a timer _or_ for something to be added. - select { - case <-sq.clock.After(delay): - case justAdded, open := <-sq.add: - if open { - q.Enqueue(justAdded) - } - } - } -} - -// Enqueue schedules an item for later Dequeueing. -func (sq *SchedulingQueue) Enqueue(item ScheduledItem) { - sq.add <- item -} - -// Dequeue takes an item from the queue. -// If there are no items, or the first item isn't ready to be scheduled, it -// blocks. If there queue is closed, this will return nil. -func (sq *SchedulingQueue) Dequeue() ScheduledItem { - return <-sq.next -} diff --git a/pkg/ruler/scheduling_queue_test.go b/pkg/ruler/scheduling_queue_test.go deleted file mode 100644 index e25184c45b8..00000000000 --- a/pkg/ruler/scheduling_queue_test.go +++ /dev/null @@ -1,236 +0,0 @@ -package ruler - -import ( - "fmt" - "runtime" - "testing" - "time" - - "github.com/jonboulle/clockwork" - "github.com/stretchr/testify/assert" -) - -type sched time.Time - -func (s sched) Scheduled() time.Time { - return time.Time(s) -} - -func (s sched) Key() string { - return time.Time(s).Format("2006-01-02 15:04:05.000") -} - -// assertDequeues asserts that queue.Dequeue() is simpleItem. -func assertDequeues(t *testing.T, item sched, queue *SchedulingQueue) { - assert.Equal(t, item, queue.Dequeue().(sched), fmt.Sprintf("Expected to dequeue %v", item)) -} - -func TestSchedulingQueuePriorities(t *testing.T) { - clock := clockwork.NewFakeClock() - queue := NewSchedulingQueue(clock) - oneHourAgo := sched(clock.Now().Add(-time.Hour)) - twoHoursAgo := sched(clock.Now().Add(-2 * time.Hour)) - queue.Enqueue(oneHourAgo) - queue.Enqueue(twoHoursAgo) - - assertDequeues(t, twoHoursAgo, queue) - assertDequeues(t, oneHourAgo, queue) - - queue.Close() - assert.Nil(t, queue.Dequeue(), "Expect nil dequeue") -} - -func TestSchedulingQueuePriorities2(t *testing.T) { - clock := clockwork.NewRealClock() - queue := NewSchedulingQueue(clock) - oneHourAgo := sched(clock.Now().Add(-time.Hour)) - twoHoursAgo := sched(clock.Now().Add(-2 * time.Hour)) - queue.Enqueue(twoHoursAgo) - queue.Enqueue(oneHourAgo) - - assertDequeues(t, twoHoursAgo, queue) - assertDequeues(t, oneHourAgo, queue) - - queue.Close() - assert.Nil(t, queue.Dequeue(), "Expect nil dequeue") -} - -func TestSchedulingQueueWaitOnEmpty(t *testing.T) { - clock := clockwork.NewFakeClock() - queue := NewSchedulingQueue(clock) - - done := make(chan struct{}) - go func() { - assert.Nil(t, queue.Dequeue(), "Expect nil dequeue") - close(done) - }() - - queue.Close() - runtime.Gosched() - select { - case <-done: - case <-time.After(100 * time.Millisecond): - t.Fatal("Close didn't unblock Dequeue.") - } -} - -func TestSchedulingQueueWaitOnItem(t *testing.T) { - clock := clockwork.NewFakeClock() - queue := NewSchedulingQueue(clock) - - oneHourAgo := sched(clock.Now().Add(-time.Hour)) - done := make(chan struct{}) - go func() { - assertDequeues(t, oneHourAgo, queue) - close(done) - }() - - queue.Enqueue(oneHourAgo) - runtime.Gosched() - select { - case <-done: - case <-time.After(100 * time.Millisecond): - t.Fatal("Queueing item didn't unblock Dequeue.") - } -} - -// We can dequeue all the items after closing the queue. -func TestSchedulingQueueEmptiesAfterClosing(t *testing.T) { - clock := clockwork.NewFakeClock() - queue := NewSchedulingQueue(clock) - - twoHoursAgo := sched(clock.Now().Add(-2 * time.Hour)) - oneHourAgo := sched(clock.Now().Add(-1 * time.Hour)) - - queue.Enqueue(twoHoursAgo) - queue.Enqueue(oneHourAgo) - queue.Close() - - assertDequeues(t, twoHoursAgo, queue) - assertDequeues(t, oneHourAgo, queue) - assert.Nil(t, queue.Dequeue(), "Expect nil dequeue") -} - -func TestSchedulingQueueBlockingBug(t *testing.T) { - clock := clockwork.NewFakeClock() - queue := NewSchedulingQueue(clock) - twoHoursAgo := sched(clock.Now().Add(-2 * time.Hour)) - queue.Enqueue(twoHoursAgo) - assertDequeues(t, twoHoursAgo, queue) - - done := make(chan struct{}) - delay := 1 * time.Hour - soon := sched(clock.Now().Add(delay)) - go func() { - assertDequeues(t, soon, queue) - close(done) - }() - - queue.Enqueue(soon) - clock.BlockUntil(1) - clock.Advance(2 * delay) - select { - case <-done: - case <-time.After(100 * time.Millisecond): - t.Fatal("Advancing the clock didn't trigger dequeue.") - } -} - -func TestSchedulingQueueRescheduling(t *testing.T) { - clock := clockwork.NewFakeClock() - queue := NewSchedulingQueue(clock) - - oneHourAgo := sched(clock.Now().Add(-1 * time.Hour)) - oneHourFromNow := sched(clock.Now().Add(1 * time.Hour)) - queue.Enqueue(oneHourFromNow) - - done := make(chan struct{}) - go func() { - assertDequeues(t, oneHourAgo, queue) - close(done) - }() - - queue.Enqueue(oneHourAgo) - select { - case <-done: - case <-time.After(100 * time.Millisecond): - t.Fatal("Dequeue never happens.") - } -} - -func TestSchedulingQueueCloseWhileWaiting(t *testing.T) { - clock := clockwork.NewFakeClock() - queue := NewSchedulingQueue(clock) - - oneHourFromNow := sched(clock.Now().Add(1 * time.Hour)) - queue.Enqueue(oneHourFromNow) - - clock.BlockUntil(1) - clock.Advance(30 * time.Minute) - queue.Close() - - clock.Advance(1 * time.Hour) - assertDequeues(t, oneHourFromNow, queue) - assert.Nil(t, queue.Dequeue(), "Expect nil dequeue") -} - -type richItem struct { - scheduled time.Time - key string - payload string -} - -func (r richItem) Scheduled() time.Time { - return r.scheduled -} - -func (r richItem) Key() string { - return r.key -} - -// If we enqueue a second item with the same key as an existing item, the new -// item replaces the old, adjusting priority if necessary. -func TestSchedulingQueueDedupe(t *testing.T) { - clock := clockwork.NewFakeClock() - queue := NewSchedulingQueue(clock) - - threeHoursAgo := clock.Now().Add(-3 * time.Hour) - twoHoursAgo := clock.Now().Add(-2 * time.Hour) - oneHourAgo := clock.Now().Add(-1 * time.Hour) - - bar := richItem{twoHoursAgo, "bar", "middling priority"} - foo1 := richItem{oneHourAgo, "foo", "less important than bar"} - foo2 := richItem{threeHoursAgo, "foo", "more important than bar"} - queue.Enqueue(bar) - queue.Enqueue(foo1) - queue.Enqueue(foo2) - queue.Close() - - assert.Equal(t, foo2, queue.Dequeue().(richItem), fmt.Sprintf("Expected to dequeue %v", foo2)) - assert.Equal(t, bar, queue.Dequeue().(richItem), fmt.Sprintf("Expected to dequeue %v", bar)) - assert.Nil(t, queue.Dequeue(), "Expect nil dequeue") -} - -func TestSchedulingQueueDedupe2(t *testing.T) { - clock := clockwork.NewFakeClock() - queue := NewSchedulingQueue(clock) - - threeHoursAgo := clock.Now().Add(-3 * time.Hour) - twoHoursAgo := clock.Now().Add(-2 * time.Hour) - oneHourAgo := clock.Now().Add(-1 * time.Hour) - - bar := richItem{twoHoursAgo, "bar", "middling priority"} - foo1 := richItem{oneHourAgo, "foo", "less important than bar"} - foo2 := richItem{threeHoursAgo, "foo", "more important than bar"} - queue.Enqueue(bar) - queue.Enqueue(foo2) - - assert.Equal(t, foo2, queue.Dequeue().(richItem), fmt.Sprintf("Expected to dequeue %v", foo2)) - - queue.Enqueue(foo1) - queue.Close() - - assert.Equal(t, bar, queue.Dequeue().(richItem), fmt.Sprintf("Expected to dequeue %v", bar)) - assert.Equal(t, foo1, queue.Dequeue().(richItem), fmt.Sprintf("Expected to dequeue %v", foo2)) - assert.Nil(t, queue.Dequeue(), "Expect nil dequeue") -} diff --git a/pkg/ruler/storage.go b/pkg/ruler/storage.go new file mode 100644 index 00000000000..0193362df4c --- /dev/null +++ b/pkg/ruler/storage.go @@ -0,0 +1,37 @@ +package ruler + +import ( + "flag" + "fmt" + + "github.com/cortexproject/cortex/pkg/configs/client" + "github.com/cortexproject/cortex/pkg/ruler/rules" +) + +// RuleStoreConfig conigures a rule store +type RuleStoreConfig struct { + Type string `yaml:"type"` + ConfigDB client.Config + + mock *mockRuleStore +} + +// RegisterFlags registers flags. +func (cfg *RuleStoreConfig) RegisterFlags(f *flag.FlagSet) { + cfg.ConfigDB.RegisterFlagsWithPrefix("ruler.", f) + f.StringVar(&cfg.Type, "ruler.storage.type", "configdb", "Method to use for backend rule storage (configdb, gcs)") +} + +// NewRuleStorage returns a new rule storage backend poller and store +func NewRuleStorage(cfg RuleStoreConfig) (rules.RuleStore, error) { + if cfg.mock != nil { + return cfg.mock, nil + } + + switch cfg.Type { + case "configdb": + return client.New(cfg.ConfigDB) + default: + return nil, fmt.Errorf("Unrecognized rule storage mode %v, choose one of: configdb, gcs", cfg.Type) + } +} diff --git a/pkg/ruler/worker.go b/pkg/ruler/worker.go deleted file mode 100644 index a56eae0f707..00000000000 --- a/pkg/ruler/worker.go +++ /dev/null @@ -1,67 +0,0 @@ -package ruler - -import ( - "time" - - "github.com/cortexproject/cortex/pkg/util" - "github.com/go-kit/kit/log/level" - "github.com/prometheus/client_golang/prometheus" -) - -var ( - blockedWorkers = prometheus.NewGauge(prometheus.GaugeOpts{ - Namespace: "cortex", - Name: "blocked_workers", - Help: "How many workers are waiting on an item to be ready.", - }) - workerIdleTime = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "worker_idle_seconds_total", - Help: "How long workers have spent waiting for work.", - }) - evalLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: "cortex", - Name: "group_evaluation_latency_seconds", - Help: "How far behind the target time each rule group executed.", - Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 25}, - }) -) - -// Worker does a thing until it's told to stop. -type Worker interface { - Run() - Stop() -} - -type worker struct { - scheduler *scheduler - ruler *Ruler -} - -func newWorker(ruler *Ruler) worker { - return worker{ - scheduler: ruler.scheduler, - ruler: ruler, - } -} - -func (w *worker) Run() { - for { - waitStart := time.Now() - blockedWorkers.Inc() - level.Debug(util.Logger).Log("msg", "waiting for next work item") - item := w.scheduler.nextWorkItem() - blockedWorkers.Dec() - waitElapsed := time.Now().Sub(waitStart) - if item == nil { - level.Debug(util.Logger).Log("msg", "queue closed and empty; terminating worker") - return - } - evalLatency.Observe(time.Since(item.scheduled).Seconds()) - workerIdleTime.Add(waitElapsed.Seconds()) - level.Debug(util.Logger).Log("msg", "processing item", "item", item) - w.ruler.Evaluate(item.userID, item) - w.scheduler.workItemDone(*item) - level.Debug(util.Logger).Log("msg", "item handed back to queue", "item", item) - } -} diff --git a/vendor/github.com/spf13/afero/.travis.yml b/vendor/github.com/spf13/afero/.travis.yml new file mode 100644 index 00000000000..0637db726de --- /dev/null +++ b/vendor/github.com/spf13/afero/.travis.yml @@ -0,0 +1,21 @@ +sudo: false +language: go + +go: + - 1.9 + - "1.10" + - tip + +os: + - linux + - osx + +matrix: + allow_failures: + - go: tip + fast_finish: true + +script: + - go build + - go test -race -v ./... + diff --git a/vendor/github.com/spf13/afero/LICENSE.txt b/vendor/github.com/spf13/afero/LICENSE.txt new file mode 100644 index 00000000000..298f0e2665e --- /dev/null +++ b/vendor/github.com/spf13/afero/LICENSE.txt @@ -0,0 +1,174 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/vendor/github.com/spf13/afero/README.md b/vendor/github.com/spf13/afero/README.md new file mode 100644 index 00000000000..0c9b04b53fc --- /dev/null +++ b/vendor/github.com/spf13/afero/README.md @@ -0,0 +1,452 @@ +![afero logo-sm](https://cloud.githubusercontent.com/assets/173412/11490338/d50e16dc-97a5-11e5-8b12-019a300d0fcb.png) + +A FileSystem Abstraction System for Go + +[![Build Status](https://travis-ci.org/spf13/afero.svg)](https://travis-ci.org/spf13/afero) [![Build status](https://ci.appveyor.com/api/projects/status/github/spf13/afero?branch=master&svg=true)](https://ci.appveyor.com/project/spf13/afero) [![GoDoc](https://godoc.org/github.com/spf13/afero?status.svg)](https://godoc.org/github.com/spf13/afero) [![Join the chat at https://gitter.im/spf13/afero](https://badges.gitter.im/Dev%20Chat.svg)](https://gitter.im/spf13/afero?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +# Overview + +Afero is an filesystem framework providing a simple, uniform and universal API +interacting with any filesystem, as an abstraction layer providing interfaces, +types and methods. Afero has an exceptionally clean interface and simple design +without needless constructors or initialization methods. + +Afero is also a library providing a base set of interoperable backend +filesystems that make it easy to work with afero while retaining all the power +and benefit of the os and ioutil packages. + +Afero provides significant improvements over using the os package alone, most +notably the ability to create mock and testing filesystems without relying on the disk. + +It is suitable for use in a any situation where you would consider using the OS +package as it provides an additional abstraction that makes it easy to use a +memory backed file system during testing. It also adds support for the http +filesystem for full interoperability. + + +## Afero Features + +* A single consistent API for accessing a variety of filesystems +* Interoperation between a variety of file system types +* A set of interfaces to encourage and enforce interoperability between backends +* An atomic cross platform memory backed file system +* Support for compositional (union) file systems by combining multiple file systems acting as one +* Specialized backends which modify existing filesystems (Read Only, Regexp filtered) +* A set of utility functions ported from io, ioutil & hugo to be afero aware + + +# Using Afero + +Afero is easy to use and easier to adopt. + +A few different ways you could use Afero: + +* Use the interfaces alone to define you own file system. +* Wrap for the OS packages. +* Define different filesystems for different parts of your application. +* Use Afero for mock filesystems while testing + +## Step 1: Install Afero + +First use go get to install the latest version of the library. + + $ go get github.com/spf13/afero + +Next include Afero in your application. +```go +import "github.com/spf13/afero" +``` + +## Step 2: Declare a backend + +First define a package variable and set it to a pointer to a filesystem. +```go +var AppFs = afero.NewMemMapFs() + +or + +var AppFs = afero.NewOsFs() +``` +It is important to note that if you repeat the composite literal you +will be using a completely new and isolated filesystem. In the case of +OsFs it will still use the same underlying filesystem but will reduce +the ability to drop in other filesystems as desired. + +## Step 3: Use it like you would the OS package + +Throughout your application use any function and method like you normally +would. + +So if my application before had: +```go +os.Open('/tmp/foo') +``` +We would replace it with: +```go +AppFs.Open('/tmp/foo') +``` + +`AppFs` being the variable we defined above. + + +## List of all available functions + +File System Methods Available: +```go +Chmod(name string, mode os.FileMode) : error +Chtimes(name string, atime time.Time, mtime time.Time) : error +Create(name string) : File, error +Mkdir(name string, perm os.FileMode) : error +MkdirAll(path string, perm os.FileMode) : error +Name() : string +Open(name string) : File, error +OpenFile(name string, flag int, perm os.FileMode) : File, error +Remove(name string) : error +RemoveAll(path string) : error +Rename(oldname, newname string) : error +Stat(name string) : os.FileInfo, error +``` +File Interfaces and Methods Available: +```go +io.Closer +io.Reader +io.ReaderAt +io.Seeker +io.Writer +io.WriterAt + +Name() : string +Readdir(count int) : []os.FileInfo, error +Readdirnames(n int) : []string, error +Stat() : os.FileInfo, error +Sync() : error +Truncate(size int64) : error +WriteString(s string) : ret int, err error +``` +In some applications it may make sense to define a new package that +simply exports the file system variable for easy access from anywhere. + +## Using Afero's utility functions + +Afero provides a set of functions to make it easier to use the underlying file systems. +These functions have been primarily ported from io & ioutil with some developed for Hugo. + +The afero utilities support all afero compatible backends. + +The list of utilities includes: + +```go +DirExists(path string) (bool, error) +Exists(path string) (bool, error) +FileContainsBytes(filename string, subslice []byte) (bool, error) +GetTempDir(subPath string) string +IsDir(path string) (bool, error) +IsEmpty(path string) (bool, error) +ReadDir(dirname string) ([]os.FileInfo, error) +ReadFile(filename string) ([]byte, error) +SafeWriteReader(path string, r io.Reader) (err error) +TempDir(dir, prefix string) (name string, err error) +TempFile(dir, prefix string) (f File, err error) +Walk(root string, walkFn filepath.WalkFunc) error +WriteFile(filename string, data []byte, perm os.FileMode) error +WriteReader(path string, r io.Reader) (err error) +``` +For a complete list see [Afero's GoDoc](https://godoc.org/github.com/spf13/afero) + +They are available under two different approaches to use. You can either call +them directly where the first parameter of each function will be the file +system, or you can declare a new `Afero`, a custom type used to bind these +functions as methods to a given filesystem. + +### Calling utilities directly + +```go +fs := new(afero.MemMapFs) +f, err := afero.TempFile(fs,"", "ioutil-test") + +``` + +### Calling via Afero + +```go +fs := afero.NewMemMapFs() +afs := &afero.Afero{Fs: fs} +f, err := afs.TempFile("", "ioutil-test") +``` + +## Using Afero for Testing + +There is a large benefit to using a mock filesystem for testing. It has a +completely blank state every time it is initialized and can be easily +reproducible regardless of OS. You could create files to your heart’s content +and the file access would be fast while also saving you from all the annoying +issues with deleting temporary files, Windows file locking, etc. The MemMapFs +backend is perfect for testing. + +* Much faster than performing I/O operations on disk +* Avoid security issues and permissions +* Far more control. 'rm -rf /' with confidence +* Test setup is far more easier to do +* No test cleanup needed + +One way to accomplish this is to define a variable as mentioned above. +In your application this will be set to afero.NewOsFs() during testing you +can set it to afero.NewMemMapFs(). + +It wouldn't be uncommon to have each test initialize a blank slate memory +backend. To do this I would define my `appFS = afero.NewOsFs()` somewhere +appropriate in my application code. This approach ensures that Tests are order +independent, with no test relying on the state left by an earlier test. + +Then in my tests I would initialize a new MemMapFs for each test: +```go +func TestExist(t *testing.T) { + appFS := afero.NewMemMapFs() + // create test files and directories + appFS.MkdirAll("src/a", 0755) + afero.WriteFile(appFS, "src/a/b", []byte("file b"), 0644) + afero.WriteFile(appFS, "src/c", []byte("file c"), 0644) + name := "src/c" + _, err := appFS.Stat(name) + if os.IsNotExist(err) { + t.Errorf("file \"%s\" does not exist.\n", name) + } +} +``` + +# Available Backends + +## Operating System Native + +### OsFs + +The first is simply a wrapper around the native OS calls. This makes it +very easy to use as all of the calls are the same as the existing OS +calls. It also makes it trivial to have your code use the OS during +operation and a mock filesystem during testing or as needed. + +```go +appfs := afero.NewOsFs() +appfs.MkdirAll("src/a", 0755)) +``` + +## Memory Backed Storage + +### MemMapFs + +Afero also provides a fully atomic memory backed filesystem perfect for use in +mocking and to speed up unnecessary disk io when persistence isn’t +necessary. It is fully concurrent and will work within go routines +safely. + +```go +mm := afero.NewMemMapFs() +mm.MkdirAll("src/a", 0755)) +``` + +#### InMemoryFile + +As part of MemMapFs, Afero also provides an atomic, fully concurrent memory +backed file implementation. This can be used in other memory backed file +systems with ease. Plans are to add a radix tree memory stored file +system using InMemoryFile. + +## Network Interfaces + +### SftpFs + +Afero has experimental support for secure file transfer protocol (sftp). Which can +be used to perform file operations over a encrypted channel. + +## Filtering Backends + +### BasePathFs + +The BasePathFs restricts all operations to a given path within an Fs. +The given file name to the operations on this Fs will be prepended with +the base path before calling the source Fs. + +```go +bp := afero.NewBasePathFs(afero.NewOsFs(), "/base/path") +``` + +### ReadOnlyFs + +A thin wrapper around the source Fs providing a read only view. + +```go +fs := afero.NewReadOnlyFs(afero.NewOsFs()) +_, err := fs.Create("/file.txt") +// err = syscall.EPERM +``` + +# RegexpFs + +A filtered view on file names, any file NOT matching +the passed regexp will be treated as non-existing. +Files not matching the regexp provided will not be created. +Directories are not filtered. + +```go +fs := afero.NewRegexpFs(afero.NewMemMapFs(), regexp.MustCompile(`\.txt$`)) +_, err := fs.Create("/file.html") +// err = syscall.ENOENT +``` + +### HttpFs + +Afero provides an http compatible backend which can wrap any of the existing +backends. + +The Http package requires a slightly specific version of Open which +returns an http.File type. + +Afero provides an httpFs file system which satisfies this requirement. +Any Afero FileSystem can be used as an httpFs. + +```go +httpFs := afero.NewHttpFs() +fileserver := http.FileServer(httpFs.Dir())) +http.Handle("/", fileserver) +``` + +## Composite Backends + +Afero provides the ability have two filesystems (or more) act as a single +file system. + +### CacheOnReadFs + +The CacheOnReadFs will lazily make copies of any accessed files from the base +layer into the overlay. Subsequent reads will be pulled from the overlay +directly permitting the request is within the cache duration of when it was +created in the overlay. + +If the base filesystem is writeable, any changes to files will be +done first to the base, then to the overlay layer. Write calls to open file +handles like `Write()` or `Truncate()` to the overlay first. + +To writing files to the overlay only, you can use the overlay Fs directly (not +via the union Fs). + +Cache files in the layer for the given time.Duration, a cache duration of 0 +means "forever" meaning the file will not be re-requested from the base ever. + +A read-only base will make the overlay also read-only but still copy files +from the base to the overlay when they're not present (or outdated) in the +caching layer. + +```go +base := afero.NewOsFs() +layer := afero.NewMemMapFs() +ufs := afero.NewCacheOnReadFs(base, layer, 100 * time.Second) +``` + +### CopyOnWriteFs() + +The CopyOnWriteFs is a read only base file system with a potentially +writeable layer on top. + +Read operations will first look in the overlay and if not found there, will +serve the file from the base. + +Changes to the file system will only be made in the overlay. + +Any attempt to modify a file found only in the base will copy the file to the +overlay layer before modification (including opening a file with a writable +handle). + +Removing and Renaming files present only in the base layer is not currently +permitted. If a file is present in the base layer and the overlay, only the +overlay will be removed/renamed. + +```go + base := afero.NewOsFs() + roBase := afero.NewReadOnlyFs(base) + ufs := afero.NewCopyOnWriteFs(roBase, afero.NewMemMapFs()) + + fh, _ = ufs.Create("/home/test/file2.txt") + fh.WriteString("This is a test") + fh.Close() +``` + +In this example all write operations will only occur in memory (MemMapFs) +leaving the base filesystem (OsFs) untouched. + + +## Desired/possible backends + +The following is a short list of possible backends we hope someone will +implement: + +* SSH +* ZIP +* TAR +* S3 + +# About the project + +## What's in the name + +Afero comes from the latin roots Ad-Facere. + +**"Ad"** is a prefix meaning "to". + +**"Facere"** is a form of the root "faciō" making "make or do". + +The literal meaning of afero is "to make" or "to do" which seems very fitting +for a library that allows one to make files and directories and do things with them. + +The English word that shares the same roots as Afero is "affair". Affair shares +the same concept but as a noun it means "something that is made or done" or "an +object of a particular type". + +It's also nice that unlike some of my other libraries (hugo, cobra, viper) it +Googles very well. + +## Release Notes + +* **0.10.0** 2015.12.10 + * Full compatibility with Windows + * Introduction of afero utilities + * Test suite rewritten to work cross platform + * Normalize paths for MemMapFs + * Adding Sync to the file interface + * **Breaking Change** Walk and ReadDir have changed parameter order + * Moving types used by MemMapFs to a subpackage + * General bugfixes and improvements +* **0.9.0** 2015.11.05 + * New Walk function similar to filepath.Walk + * MemMapFs.OpenFile handles O_CREATE, O_APPEND, O_TRUNC + * MemMapFs.Remove now really deletes the file + * InMemoryFile.Readdir and Readdirnames work correctly + * InMemoryFile functions lock it for concurrent access + * Test suite improvements +* **0.8.0** 2014.10.28 + * First public version + * Interfaces feel ready for people to build using + * Interfaces satisfy all known uses + * MemMapFs passes the majority of the OS test suite + * OsFs passes the majority of the OS test suite + +## Contributing + +1. Fork it +2. Create your feature branch (`git checkout -b my-new-feature`) +3. Commit your changes (`git commit -am 'Add some feature'`) +4. Push to the branch (`git push origin my-new-feature`) +5. Create new Pull Request + +## Contributors + +Names in no particular order: + +* [spf13](https://github.com/spf13) +* [jaqx0r](https://github.com/jaqx0r) +* [mbertschler](https://github.com/mbertschler) +* [xor-gate](https://github.com/xor-gate) + +## License + +Afero is released under the Apache 2.0 license. See +[LICENSE.txt](https://github.com/spf13/afero/blob/master/LICENSE.txt) diff --git a/vendor/github.com/spf13/afero/afero.go b/vendor/github.com/spf13/afero/afero.go new file mode 100644 index 00000000000..f5b5e127cd6 --- /dev/null +++ b/vendor/github.com/spf13/afero/afero.go @@ -0,0 +1,108 @@ +// Copyright © 2014 Steve Francia . +// Copyright 2013 tsuru authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package afero provides types and methods for interacting with the filesystem, +// as an abstraction layer. + +// Afero also provides a few implementations that are mostly interoperable. One that +// uses the operating system filesystem, one that uses memory to store files +// (cross platform) and an interface that should be implemented if you want to +// provide your own filesystem. + +package afero + +import ( + "errors" + "io" + "os" + "time" +) + +type Afero struct { + Fs +} + +// File represents a file in the filesystem. +type File interface { + io.Closer + io.Reader + io.ReaderAt + io.Seeker + io.Writer + io.WriterAt + + Name() string + Readdir(count int) ([]os.FileInfo, error) + Readdirnames(n int) ([]string, error) + Stat() (os.FileInfo, error) + Sync() error + Truncate(size int64) error + WriteString(s string) (ret int, err error) +} + +// Fs is the filesystem interface. +// +// Any simulated or real filesystem should implement this interface. +type Fs interface { + // Create creates a file in the filesystem, returning the file and an + // error, if any happens. + Create(name string) (File, error) + + // Mkdir creates a directory in the filesystem, return an error if any + // happens. + Mkdir(name string, perm os.FileMode) error + + // MkdirAll creates a directory path and all parents that does not exist + // yet. + MkdirAll(path string, perm os.FileMode) error + + // Open opens a file, returning it or an error, if any happens. + Open(name string) (File, error) + + // OpenFile opens a file using the given flags and the given mode. + OpenFile(name string, flag int, perm os.FileMode) (File, error) + + // Remove removes a file identified by name, returning an error, if any + // happens. + Remove(name string) error + + // RemoveAll removes a directory path and any children it contains. It + // does not fail if the path does not exist (return nil). + RemoveAll(path string) error + + // Rename renames a file. + Rename(oldname, newname string) error + + // Stat returns a FileInfo describing the named file, or an error, if any + // happens. + Stat(name string) (os.FileInfo, error) + + // The name of this FileSystem + Name() string + + //Chmod changes the mode of the named file to mode. + Chmod(name string, mode os.FileMode) error + + //Chtimes changes the access and modification times of the named file + Chtimes(name string, atime time.Time, mtime time.Time) error +} + +var ( + ErrFileClosed = errors.New("File is closed") + ErrOutOfRange = errors.New("Out of range") + ErrTooLarge = errors.New("Too large") + ErrFileNotFound = os.ErrNotExist + ErrFileExists = os.ErrExist + ErrDestinationExists = os.ErrExist +) diff --git a/vendor/github.com/spf13/afero/appveyor.yml b/vendor/github.com/spf13/afero/appveyor.yml new file mode 100644 index 00000000000..a633ad500c1 --- /dev/null +++ b/vendor/github.com/spf13/afero/appveyor.yml @@ -0,0 +1,15 @@ +version: '{build}' +clone_folder: C:\gopath\src\github.com\spf13\afero +environment: + GOPATH: C:\gopath +build_script: +- cmd: >- + go version + + go env + + go get -v github.com/spf13/afero/... + + go build github.com/spf13/afero +test_script: +- cmd: go test -race -v github.com/spf13/afero/... diff --git a/vendor/github.com/spf13/afero/basepath.go b/vendor/github.com/spf13/afero/basepath.go new file mode 100644 index 00000000000..616ff8ff74c --- /dev/null +++ b/vendor/github.com/spf13/afero/basepath.go @@ -0,0 +1,180 @@ +package afero + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "time" +) + +var _ Lstater = (*BasePathFs)(nil) + +// The BasePathFs restricts all operations to a given path within an Fs. +// The given file name to the operations on this Fs will be prepended with +// the base path before calling the base Fs. +// Any file name (after filepath.Clean()) outside this base path will be +// treated as non existing file. +// +// Note that it does not clean the error messages on return, so you may +// reveal the real path on errors. +type BasePathFs struct { + source Fs + path string +} + +type BasePathFile struct { + File + path string +} + +func (f *BasePathFile) Name() string { + sourcename := f.File.Name() + return strings.TrimPrefix(sourcename, filepath.Clean(f.path)) +} + +func NewBasePathFs(source Fs, path string) Fs { + return &BasePathFs{source: source, path: path} +} + +// on a file outside the base path it returns the given file name and an error, +// else the given file with the base path prepended +func (b *BasePathFs) RealPath(name string) (path string, err error) { + if err := validateBasePathName(name); err != nil { + return name, err + } + + bpath := filepath.Clean(b.path) + path = filepath.Clean(filepath.Join(bpath, name)) + if !strings.HasPrefix(path, bpath) { + return name, os.ErrNotExist + } + + return path, nil +} + +func validateBasePathName(name string) error { + if runtime.GOOS != "windows" { + // Not much to do here; + // the virtual file paths all look absolute on *nix. + return nil + } + + // On Windows a common mistake would be to provide an absolute OS path + // We could strip out the base part, but that would not be very portable. + if filepath.IsAbs(name) { + return os.ErrNotExist + } + + return nil +} + +func (b *BasePathFs) Chtimes(name string, atime, mtime time.Time) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "chtimes", Path: name, Err: err} + } + return b.source.Chtimes(name, atime, mtime) +} + +func (b *BasePathFs) Chmod(name string, mode os.FileMode) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "chmod", Path: name, Err: err} + } + return b.source.Chmod(name, mode) +} + +func (b *BasePathFs) Name() string { + return "BasePathFs" +} + +func (b *BasePathFs) Stat(name string) (fi os.FileInfo, err error) { + if name, err = b.RealPath(name); err != nil { + return nil, &os.PathError{Op: "stat", Path: name, Err: err} + } + return b.source.Stat(name) +} + +func (b *BasePathFs) Rename(oldname, newname string) (err error) { + if oldname, err = b.RealPath(oldname); err != nil { + return &os.PathError{Op: "rename", Path: oldname, Err: err} + } + if newname, err = b.RealPath(newname); err != nil { + return &os.PathError{Op: "rename", Path: newname, Err: err} + } + return b.source.Rename(oldname, newname) +} + +func (b *BasePathFs) RemoveAll(name string) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "remove_all", Path: name, Err: err} + } + return b.source.RemoveAll(name) +} + +func (b *BasePathFs) Remove(name string) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "remove", Path: name, Err: err} + } + return b.source.Remove(name) +} + +func (b *BasePathFs) OpenFile(name string, flag int, mode os.FileMode) (f File, err error) { + if name, err = b.RealPath(name); err != nil { + return nil, &os.PathError{Op: "openfile", Path: name, Err: err} + } + sourcef, err := b.source.OpenFile(name, flag, mode) + if err != nil { + return nil, err + } + return &BasePathFile{sourcef, b.path}, nil +} + +func (b *BasePathFs) Open(name string) (f File, err error) { + if name, err = b.RealPath(name); err != nil { + return nil, &os.PathError{Op: "open", Path: name, Err: err} + } + sourcef, err := b.source.Open(name) + if err != nil { + return nil, err + } + return &BasePathFile{File: sourcef, path: b.path}, nil +} + +func (b *BasePathFs) Mkdir(name string, mode os.FileMode) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "mkdir", Path: name, Err: err} + } + return b.source.Mkdir(name, mode) +} + +func (b *BasePathFs) MkdirAll(name string, mode os.FileMode) (err error) { + if name, err = b.RealPath(name); err != nil { + return &os.PathError{Op: "mkdir", Path: name, Err: err} + } + return b.source.MkdirAll(name, mode) +} + +func (b *BasePathFs) Create(name string) (f File, err error) { + if name, err = b.RealPath(name); err != nil { + return nil, &os.PathError{Op: "create", Path: name, Err: err} + } + sourcef, err := b.source.Create(name) + if err != nil { + return nil, err + } + return &BasePathFile{File: sourcef, path: b.path}, nil +} + +func (b *BasePathFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + name, err := b.RealPath(name) + if err != nil { + return nil, false, &os.PathError{Op: "lstat", Path: name, Err: err} + } + if lstater, ok := b.source.(Lstater); ok { + return lstater.LstatIfPossible(name) + } + fi, err := b.source.Stat(name) + return fi, false, err +} + +// vim: ts=4 sw=4 noexpandtab nolist syn=go diff --git a/vendor/github.com/spf13/afero/cacheOnReadFs.go b/vendor/github.com/spf13/afero/cacheOnReadFs.go new file mode 100644 index 00000000000..29a26c67dd5 --- /dev/null +++ b/vendor/github.com/spf13/afero/cacheOnReadFs.go @@ -0,0 +1,290 @@ +package afero + +import ( + "os" + "syscall" + "time" +) + +// If the cache duration is 0, cache time will be unlimited, i.e. once +// a file is in the layer, the base will never be read again for this file. +// +// For cache times greater than 0, the modification time of a file is +// checked. Note that a lot of file system implementations only allow a +// resolution of a second for timestamps... or as the godoc for os.Chtimes() +// states: "The underlying filesystem may truncate or round the values to a +// less precise time unit." +// +// This caching union will forward all write calls also to the base file +// system first. To prevent writing to the base Fs, wrap it in a read-only +// filter - Note: this will also make the overlay read-only, for writing files +// in the overlay, use the overlay Fs directly, not via the union Fs. +type CacheOnReadFs struct { + base Fs + layer Fs + cacheTime time.Duration +} + +func NewCacheOnReadFs(base Fs, layer Fs, cacheTime time.Duration) Fs { + return &CacheOnReadFs{base: base, layer: layer, cacheTime: cacheTime} +} + +type cacheState int + +const ( + // not present in the overlay, unknown if it exists in the base: + cacheMiss cacheState = iota + // present in the overlay and in base, base file is newer: + cacheStale + // present in the overlay - with cache time == 0 it may exist in the base, + // with cacheTime > 0 it exists in the base and is same age or newer in the + // overlay + cacheHit + // happens if someone writes directly to the overlay without + // going through this union + cacheLocal +) + +func (u *CacheOnReadFs) cacheStatus(name string) (state cacheState, fi os.FileInfo, err error) { + var lfi, bfi os.FileInfo + lfi, err = u.layer.Stat(name) + if err == nil { + if u.cacheTime == 0 { + return cacheHit, lfi, nil + } + if lfi.ModTime().Add(u.cacheTime).Before(time.Now()) { + bfi, err = u.base.Stat(name) + if err != nil { + return cacheLocal, lfi, nil + } + if bfi.ModTime().After(lfi.ModTime()) { + return cacheStale, bfi, nil + } + } + return cacheHit, lfi, nil + } + + if err == syscall.ENOENT || os.IsNotExist(err) { + return cacheMiss, nil, nil + } + + return cacheMiss, nil, err +} + +func (u *CacheOnReadFs) copyToLayer(name string) error { + return copyToLayer(u.base, u.layer, name) +} + +func (u *CacheOnReadFs) Chtimes(name string, atime, mtime time.Time) error { + st, _, err := u.cacheStatus(name) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit: + err = u.base.Chtimes(name, atime, mtime) + case cacheStale, cacheMiss: + if err := u.copyToLayer(name); err != nil { + return err + } + err = u.base.Chtimes(name, atime, mtime) + } + if err != nil { + return err + } + return u.layer.Chtimes(name, atime, mtime) +} + +func (u *CacheOnReadFs) Chmod(name string, mode os.FileMode) error { + st, _, err := u.cacheStatus(name) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit: + err = u.base.Chmod(name, mode) + case cacheStale, cacheMiss: + if err := u.copyToLayer(name); err != nil { + return err + } + err = u.base.Chmod(name, mode) + } + if err != nil { + return err + } + return u.layer.Chmod(name, mode) +} + +func (u *CacheOnReadFs) Stat(name string) (os.FileInfo, error) { + st, fi, err := u.cacheStatus(name) + if err != nil { + return nil, err + } + switch st { + case cacheMiss: + return u.base.Stat(name) + default: // cacheStale has base, cacheHit and cacheLocal the layer os.FileInfo + return fi, nil + } +} + +func (u *CacheOnReadFs) Rename(oldname, newname string) error { + st, _, err := u.cacheStatus(oldname) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit: + err = u.base.Rename(oldname, newname) + case cacheStale, cacheMiss: + if err := u.copyToLayer(oldname); err != nil { + return err + } + err = u.base.Rename(oldname, newname) + } + if err != nil { + return err + } + return u.layer.Rename(oldname, newname) +} + +func (u *CacheOnReadFs) Remove(name string) error { + st, _, err := u.cacheStatus(name) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit, cacheStale, cacheMiss: + err = u.base.Remove(name) + } + if err != nil { + return err + } + return u.layer.Remove(name) +} + +func (u *CacheOnReadFs) RemoveAll(name string) error { + st, _, err := u.cacheStatus(name) + if err != nil { + return err + } + switch st { + case cacheLocal: + case cacheHit, cacheStale, cacheMiss: + err = u.base.RemoveAll(name) + } + if err != nil { + return err + } + return u.layer.RemoveAll(name) +} + +func (u *CacheOnReadFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + st, _, err := u.cacheStatus(name) + if err != nil { + return nil, err + } + switch st { + case cacheLocal, cacheHit: + default: + if err := u.copyToLayer(name); err != nil { + return nil, err + } + } + if flag&(os.O_WRONLY|syscall.O_RDWR|os.O_APPEND|os.O_CREATE|os.O_TRUNC) != 0 { + bfi, err := u.base.OpenFile(name, flag, perm) + if err != nil { + return nil, err + } + lfi, err := u.layer.OpenFile(name, flag, perm) + if err != nil { + bfi.Close() // oops, what if O_TRUNC was set and file opening in the layer failed...? + return nil, err + } + return &UnionFile{Base: bfi, Layer: lfi}, nil + } + return u.layer.OpenFile(name, flag, perm) +} + +func (u *CacheOnReadFs) Open(name string) (File, error) { + st, fi, err := u.cacheStatus(name) + if err != nil { + return nil, err + } + + switch st { + case cacheLocal: + return u.layer.Open(name) + + case cacheMiss: + bfi, err := u.base.Stat(name) + if err != nil { + return nil, err + } + if bfi.IsDir() { + return u.base.Open(name) + } + if err := u.copyToLayer(name); err != nil { + return nil, err + } + return u.layer.Open(name) + + case cacheStale: + if !fi.IsDir() { + if err := u.copyToLayer(name); err != nil { + return nil, err + } + return u.layer.Open(name) + } + case cacheHit: + if !fi.IsDir() { + return u.layer.Open(name) + } + } + // the dirs from cacheHit, cacheStale fall down here: + bfile, _ := u.base.Open(name) + lfile, err := u.layer.Open(name) + if err != nil && bfile == nil { + return nil, err + } + return &UnionFile{Base: bfile, Layer: lfile}, nil +} + +func (u *CacheOnReadFs) Mkdir(name string, perm os.FileMode) error { + err := u.base.Mkdir(name, perm) + if err != nil { + return err + } + return u.layer.MkdirAll(name, perm) // yes, MkdirAll... we cannot assume it exists in the cache +} + +func (u *CacheOnReadFs) Name() string { + return "CacheOnReadFs" +} + +func (u *CacheOnReadFs) MkdirAll(name string, perm os.FileMode) error { + err := u.base.MkdirAll(name, perm) + if err != nil { + return err + } + return u.layer.MkdirAll(name, perm) +} + +func (u *CacheOnReadFs) Create(name string) (File, error) { + bfh, err := u.base.Create(name) + if err != nil { + return nil, err + } + lfh, err := u.layer.Create(name) + if err != nil { + // oops, see comment about OS_TRUNC above, should we remove? then we have to + // remember if the file did not exist before + bfh.Close() + return nil, err + } + return &UnionFile{Base: bfh, Layer: lfh}, nil +} diff --git a/vendor/github.com/spf13/afero/const_bsds.go b/vendor/github.com/spf13/afero/const_bsds.go new file mode 100644 index 00000000000..5728243d962 --- /dev/null +++ b/vendor/github.com/spf13/afero/const_bsds.go @@ -0,0 +1,22 @@ +// Copyright © 2016 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build darwin openbsd freebsd netbsd dragonfly + +package afero + +import ( + "syscall" +) + +const BADFD = syscall.EBADF diff --git a/vendor/github.com/spf13/afero/const_win_unix.go b/vendor/github.com/spf13/afero/const_win_unix.go new file mode 100644 index 00000000000..968fc2783e5 --- /dev/null +++ b/vendor/github.com/spf13/afero/const_win_unix.go @@ -0,0 +1,25 @@ +// Copyright © 2016 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +build !darwin +// +build !openbsd +// +build !freebsd +// +build !dragonfly +// +build !netbsd + +package afero + +import ( + "syscall" +) + +const BADFD = syscall.EBADFD diff --git a/vendor/github.com/spf13/afero/copyOnWriteFs.go b/vendor/github.com/spf13/afero/copyOnWriteFs.go new file mode 100644 index 00000000000..e8108a851e1 --- /dev/null +++ b/vendor/github.com/spf13/afero/copyOnWriteFs.go @@ -0,0 +1,293 @@ +package afero + +import ( + "fmt" + "os" + "path/filepath" + "syscall" + "time" +) + +var _ Lstater = (*CopyOnWriteFs)(nil) + +// The CopyOnWriteFs is a union filesystem: a read only base file system with +// a possibly writeable layer on top. Changes to the file system will only +// be made in the overlay: Changing an existing file in the base layer which +// is not present in the overlay will copy the file to the overlay ("changing" +// includes also calls to e.g. Chtimes() and Chmod()). +// +// Reading directories is currently only supported via Open(), not OpenFile(). +type CopyOnWriteFs struct { + base Fs + layer Fs +} + +func NewCopyOnWriteFs(base Fs, layer Fs) Fs { + return &CopyOnWriteFs{base: base, layer: layer} +} + +// Returns true if the file is not in the overlay +func (u *CopyOnWriteFs) isBaseFile(name string) (bool, error) { + if _, err := u.layer.Stat(name); err == nil { + return false, nil + } + _, err := u.base.Stat(name) + if err != nil { + if oerr, ok := err.(*os.PathError); ok { + if oerr.Err == os.ErrNotExist || oerr.Err == syscall.ENOENT || oerr.Err == syscall.ENOTDIR { + return false, nil + } + } + if err == syscall.ENOENT { + return false, nil + } + } + return true, err +} + +func (u *CopyOnWriteFs) copyToLayer(name string) error { + return copyToLayer(u.base, u.layer, name) +} + +func (u *CopyOnWriteFs) Chtimes(name string, atime, mtime time.Time) error { + b, err := u.isBaseFile(name) + if err != nil { + return err + } + if b { + if err := u.copyToLayer(name); err != nil { + return err + } + } + return u.layer.Chtimes(name, atime, mtime) +} + +func (u *CopyOnWriteFs) Chmod(name string, mode os.FileMode) error { + b, err := u.isBaseFile(name) + if err != nil { + return err + } + if b { + if err := u.copyToLayer(name); err != nil { + return err + } + } + return u.layer.Chmod(name, mode) +} + +func (u *CopyOnWriteFs) Stat(name string) (os.FileInfo, error) { + fi, err := u.layer.Stat(name) + if err != nil { + isNotExist := u.isNotExist(err) + if isNotExist { + return u.base.Stat(name) + } + return nil, err + } + return fi, nil +} + +func (u *CopyOnWriteFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + llayer, ok1 := u.layer.(Lstater) + lbase, ok2 := u.base.(Lstater) + + if ok1 { + fi, b, err := llayer.LstatIfPossible(name) + if err == nil { + return fi, b, nil + } + + if !u.isNotExist(err) { + return nil, b, err + } + } + + if ok2 { + fi, b, err := lbase.LstatIfPossible(name) + if err == nil { + return fi, b, nil + } + if !u.isNotExist(err) { + return nil, b, err + } + } + + fi, err := u.Stat(name) + + return fi, false, err +} + +func (u *CopyOnWriteFs) isNotExist(err error) bool { + if e, ok := err.(*os.PathError); ok { + err = e.Err + } + if err == os.ErrNotExist || err == syscall.ENOENT || err == syscall.ENOTDIR { + return true + } + return false +} + +// Renaming files present only in the base layer is not permitted +func (u *CopyOnWriteFs) Rename(oldname, newname string) error { + b, err := u.isBaseFile(oldname) + if err != nil { + return err + } + if b { + return syscall.EPERM + } + return u.layer.Rename(oldname, newname) +} + +// Removing files present only in the base layer is not permitted. If +// a file is present in the base layer and the overlay, only the overlay +// will be removed. +func (u *CopyOnWriteFs) Remove(name string) error { + err := u.layer.Remove(name) + switch err { + case syscall.ENOENT: + _, err = u.base.Stat(name) + if err == nil { + return syscall.EPERM + } + return syscall.ENOENT + default: + return err + } +} + +func (u *CopyOnWriteFs) RemoveAll(name string) error { + err := u.layer.RemoveAll(name) + switch err { + case syscall.ENOENT: + _, err = u.base.Stat(name) + if err == nil { + return syscall.EPERM + } + return syscall.ENOENT + default: + return err + } +} + +func (u *CopyOnWriteFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + b, err := u.isBaseFile(name) + if err != nil { + return nil, err + } + + if flag&(os.O_WRONLY|os.O_RDWR|os.O_APPEND|os.O_CREATE|os.O_TRUNC) != 0 { + if b { + if err = u.copyToLayer(name); err != nil { + return nil, err + } + return u.layer.OpenFile(name, flag, perm) + } + + dir := filepath.Dir(name) + isaDir, err := IsDir(u.base, dir) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + if isaDir { + if err = u.layer.MkdirAll(dir, 0777); err != nil { + return nil, err + } + return u.layer.OpenFile(name, flag, perm) + } + + isaDir, err = IsDir(u.layer, dir) + if err != nil { + return nil, err + } + if isaDir { + return u.layer.OpenFile(name, flag, perm) + } + + return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOTDIR} // ...or os.ErrNotExist? + } + if b { + return u.base.OpenFile(name, flag, perm) + } + return u.layer.OpenFile(name, flag, perm) +} + +// This function handles the 9 different possibilities caused +// by the union which are the intersection of the following... +// layer: doesn't exist, exists as a file, and exists as a directory +// base: doesn't exist, exists as a file, and exists as a directory +func (u *CopyOnWriteFs) Open(name string) (File, error) { + // Since the overlay overrides the base we check that first + b, err := u.isBaseFile(name) + if err != nil { + return nil, err + } + + // If overlay doesn't exist, return the base (base state irrelevant) + if b { + return u.base.Open(name) + } + + // If overlay is a file, return it (base state irrelevant) + dir, err := IsDir(u.layer, name) + if err != nil { + return nil, err + } + if !dir { + return u.layer.Open(name) + } + + // Overlay is a directory, base state now matters. + // Base state has 3 states to check but 2 outcomes: + // A. It's a file or non-readable in the base (return just the overlay) + // B. It's an accessible directory in the base (return a UnionFile) + + // If base is file or nonreadable, return overlay + dir, err = IsDir(u.base, name) + if !dir || err != nil { + return u.layer.Open(name) + } + + // Both base & layer are directories + // Return union file (if opens are without error) + bfile, bErr := u.base.Open(name) + lfile, lErr := u.layer.Open(name) + + // If either have errors at this point something is very wrong. Return nil and the errors + if bErr != nil || lErr != nil { + return nil, fmt.Errorf("BaseErr: %v\nOverlayErr: %v", bErr, lErr) + } + + return &UnionFile{Base: bfile, Layer: lfile}, nil +} + +func (u *CopyOnWriteFs) Mkdir(name string, perm os.FileMode) error { + dir, err := IsDir(u.base, name) + if err != nil { + return u.layer.MkdirAll(name, perm) + } + if dir { + return ErrFileExists + } + return u.layer.MkdirAll(name, perm) +} + +func (u *CopyOnWriteFs) Name() string { + return "CopyOnWriteFs" +} + +func (u *CopyOnWriteFs) MkdirAll(name string, perm os.FileMode) error { + dir, err := IsDir(u.base, name) + if err != nil { + return u.layer.MkdirAll(name, perm) + } + if dir { + // This is in line with how os.MkdirAll behaves. + return nil + } + return u.layer.MkdirAll(name, perm) +} + +func (u *CopyOnWriteFs) Create(name string) (File, error) { + return u.OpenFile(name, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0666) +} diff --git a/vendor/github.com/spf13/afero/go.mod b/vendor/github.com/spf13/afero/go.mod new file mode 100644 index 00000000000..08685509957 --- /dev/null +++ b/vendor/github.com/spf13/afero/go.mod @@ -0,0 +1,3 @@ +module github.com/spf13/afero + +require golang.org/x/text v0.3.0 diff --git a/vendor/github.com/spf13/afero/go.sum b/vendor/github.com/spf13/afero/go.sum new file mode 100644 index 00000000000..6bad37b2a77 --- /dev/null +++ b/vendor/github.com/spf13/afero/go.sum @@ -0,0 +1,2 @@ +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/vendor/github.com/spf13/afero/httpFs.go b/vendor/github.com/spf13/afero/httpFs.go new file mode 100644 index 00000000000..c42193688ce --- /dev/null +++ b/vendor/github.com/spf13/afero/httpFs.go @@ -0,0 +1,110 @@ +// Copyright © 2014 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "errors" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "time" +) + +type httpDir struct { + basePath string + fs HttpFs +} + +func (d httpDir) Open(name string) (http.File, error) { + if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 || + strings.Contains(name, "\x00") { + return nil, errors.New("http: invalid character in file path") + } + dir := string(d.basePath) + if dir == "" { + dir = "." + } + + f, err := d.fs.Open(filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name)))) + if err != nil { + return nil, err + } + return f, nil +} + +type HttpFs struct { + source Fs +} + +func NewHttpFs(source Fs) *HttpFs { + return &HttpFs{source: source} +} + +func (h HttpFs) Dir(s string) *httpDir { + return &httpDir{basePath: s, fs: h} +} + +func (h HttpFs) Name() string { return "h HttpFs" } + +func (h HttpFs) Create(name string) (File, error) { + return h.source.Create(name) +} + +func (h HttpFs) Chmod(name string, mode os.FileMode) error { + return h.source.Chmod(name, mode) +} + +func (h HttpFs) Chtimes(name string, atime time.Time, mtime time.Time) error { + return h.source.Chtimes(name, atime, mtime) +} + +func (h HttpFs) Mkdir(name string, perm os.FileMode) error { + return h.source.Mkdir(name, perm) +} + +func (h HttpFs) MkdirAll(path string, perm os.FileMode) error { + return h.source.MkdirAll(path, perm) +} + +func (h HttpFs) Open(name string) (http.File, error) { + f, err := h.source.Open(name) + if err == nil { + if httpfile, ok := f.(http.File); ok { + return httpfile, nil + } + } + return nil, err +} + +func (h HttpFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + return h.source.OpenFile(name, flag, perm) +} + +func (h HttpFs) Remove(name string) error { + return h.source.Remove(name) +} + +func (h HttpFs) RemoveAll(path string) error { + return h.source.RemoveAll(path) +} + +func (h HttpFs) Rename(oldname, newname string) error { + return h.source.Rename(oldname, newname) +} + +func (h HttpFs) Stat(name string) (os.FileInfo, error) { + return h.source.Stat(name) +} diff --git a/vendor/github.com/spf13/afero/ioutil.go b/vendor/github.com/spf13/afero/ioutil.go new file mode 100644 index 00000000000..5c3a3d8fffc --- /dev/null +++ b/vendor/github.com/spf13/afero/ioutil.go @@ -0,0 +1,230 @@ +// Copyright ©2015 The Go Authors +// Copyright ©2015 Steve Francia +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "bytes" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "sync" + "time" +) + +// byName implements sort.Interface. +type byName []os.FileInfo + +func (f byName) Len() int { return len(f) } +func (f byName) Less(i, j int) bool { return f[i].Name() < f[j].Name() } +func (f byName) Swap(i, j int) { f[i], f[j] = f[j], f[i] } + +// ReadDir reads the directory named by dirname and returns +// a list of sorted directory entries. +func (a Afero) ReadDir(dirname string) ([]os.FileInfo, error) { + return ReadDir(a.Fs, dirname) +} + +func ReadDir(fs Fs, dirname string) ([]os.FileInfo, error) { + f, err := fs.Open(dirname) + if err != nil { + return nil, err + } + list, err := f.Readdir(-1) + f.Close() + if err != nil { + return nil, err + } + sort.Sort(byName(list)) + return list, nil +} + +// ReadFile reads the file named by filename and returns the contents. +// A successful call returns err == nil, not err == EOF. Because ReadFile +// reads the whole file, it does not treat an EOF from Read as an error +// to be reported. +func (a Afero) ReadFile(filename string) ([]byte, error) { + return ReadFile(a.Fs, filename) +} + +func ReadFile(fs Fs, filename string) ([]byte, error) { + f, err := fs.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + // It's a good but not certain bet that FileInfo will tell us exactly how much to + // read, so let's try it but be prepared for the answer to be wrong. + var n int64 + + if fi, err := f.Stat(); err == nil { + // Don't preallocate a huge buffer, just in case. + if size := fi.Size(); size < 1e9 { + n = size + } + } + // As initial capacity for readAll, use n + a little extra in case Size is zero, + // and to avoid another allocation after Read has filled the buffer. The readAll + // call will read into its allocated internal buffer cheaply. If the size was + // wrong, we'll either waste some space off the end or reallocate as needed, but + // in the overwhelmingly common case we'll get it just right. + return readAll(f, n+bytes.MinRead) +} + +// readAll reads from r until an error or EOF and returns the data it read +// from the internal buffer allocated with a specified capacity. +func readAll(r io.Reader, capacity int64) (b []byte, err error) { + buf := bytes.NewBuffer(make([]byte, 0, capacity)) + // If the buffer overflows, we will get bytes.ErrTooLarge. + // Return that as an error. Any other panic remains. + defer func() { + e := recover() + if e == nil { + return + } + if panicErr, ok := e.(error); ok && panicErr == bytes.ErrTooLarge { + err = panicErr + } else { + panic(e) + } + }() + _, err = buf.ReadFrom(r) + return buf.Bytes(), err +} + +// ReadAll reads from r until an error or EOF and returns the data it read. +// A successful call returns err == nil, not err == EOF. Because ReadAll is +// defined to read from src until EOF, it does not treat an EOF from Read +// as an error to be reported. +func ReadAll(r io.Reader) ([]byte, error) { + return readAll(r, bytes.MinRead) +} + +// WriteFile writes data to a file named by filename. +// If the file does not exist, WriteFile creates it with permissions perm; +// otherwise WriteFile truncates it before writing. +func (a Afero) WriteFile(filename string, data []byte, perm os.FileMode) error { + return WriteFile(a.Fs, filename, data, perm) +} + +func WriteFile(fs Fs, filename string, data []byte, perm os.FileMode) error { + f, err := fs.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + n, err := f.Write(data) + if err == nil && n < len(data) { + err = io.ErrShortWrite + } + if err1 := f.Close(); err == nil { + err = err1 + } + return err +} + +// Random number state. +// We generate random temporary file names so that there's a good +// chance the file doesn't exist yet - keeps the number of tries in +// TempFile to a minimum. +var rand uint32 +var randmu sync.Mutex + +func reseed() uint32 { + return uint32(time.Now().UnixNano() + int64(os.Getpid())) +} + +func nextSuffix() string { + randmu.Lock() + r := rand + if r == 0 { + r = reseed() + } + r = r*1664525 + 1013904223 // constants from Numerical Recipes + rand = r + randmu.Unlock() + return strconv.Itoa(int(1e9 + r%1e9))[1:] +} + +// TempFile creates a new temporary file in the directory dir +// with a name beginning with prefix, opens the file for reading +// and writing, and returns the resulting *File. +// If dir is the empty string, TempFile uses the default directory +// for temporary files (see os.TempDir). +// Multiple programs calling TempFile simultaneously +// will not choose the same file. The caller can use f.Name() +// to find the pathname of the file. It is the caller's responsibility +// to remove the file when no longer needed. +func (a Afero) TempFile(dir, prefix string) (f File, err error) { + return TempFile(a.Fs, dir, prefix) +} + +func TempFile(fs Fs, dir, prefix string) (f File, err error) { + if dir == "" { + dir = os.TempDir() + } + + nconflict := 0 + for i := 0; i < 10000; i++ { + name := filepath.Join(dir, prefix+nextSuffix()) + f, err = fs.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + if os.IsExist(err) { + if nconflict++; nconflict > 10 { + randmu.Lock() + rand = reseed() + randmu.Unlock() + } + continue + } + break + } + return +} + +// TempDir creates a new temporary directory in the directory dir +// with a name beginning with prefix and returns the path of the +// new directory. If dir is the empty string, TempDir uses the +// default directory for temporary files (see os.TempDir). +// Multiple programs calling TempDir simultaneously +// will not choose the same directory. It is the caller's responsibility +// to remove the directory when no longer needed. +func (a Afero) TempDir(dir, prefix string) (name string, err error) { + return TempDir(a.Fs, dir, prefix) +} +func TempDir(fs Fs, dir, prefix string) (name string, err error) { + if dir == "" { + dir = os.TempDir() + } + + nconflict := 0 + for i := 0; i < 10000; i++ { + try := filepath.Join(dir, prefix+nextSuffix()) + err = fs.Mkdir(try, 0700) + if os.IsExist(err) { + if nconflict++; nconflict > 10 { + randmu.Lock() + rand = reseed() + randmu.Unlock() + } + continue + } + if err == nil { + name = try + } + break + } + return +} diff --git a/vendor/github.com/spf13/afero/lstater.go b/vendor/github.com/spf13/afero/lstater.go new file mode 100644 index 00000000000..89c1bfc0a7d --- /dev/null +++ b/vendor/github.com/spf13/afero/lstater.go @@ -0,0 +1,27 @@ +// Copyright © 2018 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "os" +) + +// Lstater is an optional interface in Afero. It is only implemented by the +// filesystems saying so. +// It will call Lstat if the filesystem iself is, or it delegates to, the os filesystem. +// Else it will call Stat. +// In addtion to the FileInfo, it will return a boolean telling whether Lstat was called or not. +type Lstater interface { + LstatIfPossible(name string) (os.FileInfo, bool, error) +} diff --git a/vendor/github.com/spf13/afero/match.go b/vendor/github.com/spf13/afero/match.go new file mode 100644 index 00000000000..c18a87fb713 --- /dev/null +++ b/vendor/github.com/spf13/afero/match.go @@ -0,0 +1,110 @@ +// Copyright © 2014 Steve Francia . +// Copyright 2009 The Go Authors. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "path/filepath" + "sort" + "strings" +) + +// Glob returns the names of all files matching pattern or nil +// if there is no matching file. The syntax of patterns is the same +// as in Match. The pattern may describe hierarchical names such as +// /usr/*/bin/ed (assuming the Separator is '/'). +// +// Glob ignores file system errors such as I/O errors reading directories. +// The only possible returned error is ErrBadPattern, when pattern +// is malformed. +// +// This was adapted from (http://golang.org/pkg/path/filepath) and uses several +// built-ins from that package. +func Glob(fs Fs, pattern string) (matches []string, err error) { + if !hasMeta(pattern) { + // Lstat not supported by a ll filesystems. + if _, err = lstatIfPossible(fs, pattern); err != nil { + return nil, nil + } + return []string{pattern}, nil + } + + dir, file := filepath.Split(pattern) + switch dir { + case "": + dir = "." + case string(filepath.Separator): + // nothing + default: + dir = dir[0 : len(dir)-1] // chop off trailing separator + } + + if !hasMeta(dir) { + return glob(fs, dir, file, nil) + } + + var m []string + m, err = Glob(fs, dir) + if err != nil { + return + } + for _, d := range m { + matches, err = glob(fs, d, file, matches) + if err != nil { + return + } + } + return +} + +// glob searches for files matching pattern in the directory dir +// and appends them to matches. If the directory cannot be +// opened, it returns the existing matches. New matches are +// added in lexicographical order. +func glob(fs Fs, dir, pattern string, matches []string) (m []string, e error) { + m = matches + fi, err := fs.Stat(dir) + if err != nil { + return + } + if !fi.IsDir() { + return + } + d, err := fs.Open(dir) + if err != nil { + return + } + defer d.Close() + + names, _ := d.Readdirnames(-1) + sort.Strings(names) + + for _, n := range names { + matched, err := filepath.Match(pattern, n) + if err != nil { + return m, err + } + if matched { + m = append(m, filepath.Join(dir, n)) + } + } + return +} + +// hasMeta reports whether path contains any of the magic characters +// recognized by Match. +func hasMeta(path string) bool { + // TODO(niemeyer): Should other magic characters be added here? + return strings.IndexAny(path, "*?[") >= 0 +} diff --git a/vendor/github.com/spf13/afero/mem/dir.go b/vendor/github.com/spf13/afero/mem/dir.go new file mode 100644 index 00000000000..e104013f457 --- /dev/null +++ b/vendor/github.com/spf13/afero/mem/dir.go @@ -0,0 +1,37 @@ +// Copyright © 2014 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mem + +type Dir interface { + Len() int + Names() []string + Files() []*FileData + Add(*FileData) + Remove(*FileData) +} + +func RemoveFromMemDir(dir *FileData, f *FileData) { + dir.memDir.Remove(f) +} + +func AddToMemDir(dir *FileData, f *FileData) { + dir.memDir.Add(f) +} + +func InitializeDir(d *FileData) { + if d.memDir == nil { + d.dir = true + d.memDir = &DirMap{} + } +} diff --git a/vendor/github.com/spf13/afero/mem/dirmap.go b/vendor/github.com/spf13/afero/mem/dirmap.go new file mode 100644 index 00000000000..03a57ee5b52 --- /dev/null +++ b/vendor/github.com/spf13/afero/mem/dirmap.go @@ -0,0 +1,43 @@ +// Copyright © 2015 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mem + +import "sort" + +type DirMap map[string]*FileData + +func (m DirMap) Len() int { return len(m) } +func (m DirMap) Add(f *FileData) { m[f.name] = f } +func (m DirMap) Remove(f *FileData) { delete(m, f.name) } +func (m DirMap) Files() (files []*FileData) { + for _, f := range m { + files = append(files, f) + } + sort.Sort(filesSorter(files)) + return files +} + +// implement sort.Interface for []*FileData +type filesSorter []*FileData + +func (s filesSorter) Len() int { return len(s) } +func (s filesSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s filesSorter) Less(i, j int) bool { return s[i].name < s[j].name } + +func (m DirMap) Names() (names []string) { + for x := range m { + names = append(names, x) + } + return names +} diff --git a/vendor/github.com/spf13/afero/mem/file.go b/vendor/github.com/spf13/afero/mem/file.go new file mode 100644 index 00000000000..7af2fb56ff4 --- /dev/null +++ b/vendor/github.com/spf13/afero/mem/file.go @@ -0,0 +1,317 @@ +// Copyright © 2015 Steve Francia . +// Copyright 2013 tsuru authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mem + +import ( + "bytes" + "errors" + "io" + "os" + "path/filepath" + "sync" + "sync/atomic" +) + +import "time" + +const FilePathSeparator = string(filepath.Separator) + +type File struct { + // atomic requires 64-bit alignment for struct field access + at int64 + readDirCount int64 + closed bool + readOnly bool + fileData *FileData +} + +func NewFileHandle(data *FileData) *File { + return &File{fileData: data} +} + +func NewReadOnlyFileHandle(data *FileData) *File { + return &File{fileData: data, readOnly: true} +} + +func (f File) Data() *FileData { + return f.fileData +} + +type FileData struct { + sync.Mutex + name string + data []byte + memDir Dir + dir bool + mode os.FileMode + modtime time.Time +} + +func (d *FileData) Name() string { + d.Lock() + defer d.Unlock() + return d.name +} + +func CreateFile(name string) *FileData { + return &FileData{name: name, mode: os.ModeTemporary, modtime: time.Now()} +} + +func CreateDir(name string) *FileData { + return &FileData{name: name, memDir: &DirMap{}, dir: true} +} + +func ChangeFileName(f *FileData, newname string) { + f.Lock() + f.name = newname + f.Unlock() +} + +func SetMode(f *FileData, mode os.FileMode) { + f.Lock() + f.mode = mode + f.Unlock() +} + +func SetModTime(f *FileData, mtime time.Time) { + f.Lock() + setModTime(f, mtime) + f.Unlock() +} + +func setModTime(f *FileData, mtime time.Time) { + f.modtime = mtime +} + +func GetFileInfo(f *FileData) *FileInfo { + return &FileInfo{f} +} + +func (f *File) Open() error { + atomic.StoreInt64(&f.at, 0) + atomic.StoreInt64(&f.readDirCount, 0) + f.fileData.Lock() + f.closed = false + f.fileData.Unlock() + return nil +} + +func (f *File) Close() error { + f.fileData.Lock() + f.closed = true + if !f.readOnly { + setModTime(f.fileData, time.Now()) + } + f.fileData.Unlock() + return nil +} + +func (f *File) Name() string { + return f.fileData.Name() +} + +func (f *File) Stat() (os.FileInfo, error) { + return &FileInfo{f.fileData}, nil +} + +func (f *File) Sync() error { + return nil +} + +func (f *File) Readdir(count int) (res []os.FileInfo, err error) { + if !f.fileData.dir { + return nil, &os.PathError{Op: "readdir", Path: f.fileData.name, Err: errors.New("not a dir")} + } + var outLength int64 + + f.fileData.Lock() + files := f.fileData.memDir.Files()[f.readDirCount:] + if count > 0 { + if len(files) < count { + outLength = int64(len(files)) + } else { + outLength = int64(count) + } + if len(files) == 0 { + err = io.EOF + } + } else { + outLength = int64(len(files)) + } + f.readDirCount += outLength + f.fileData.Unlock() + + res = make([]os.FileInfo, outLength) + for i := range res { + res[i] = &FileInfo{files[i]} + } + + return res, err +} + +func (f *File) Readdirnames(n int) (names []string, err error) { + fi, err := f.Readdir(n) + names = make([]string, len(fi)) + for i, f := range fi { + _, names[i] = filepath.Split(f.Name()) + } + return names, err +} + +func (f *File) Read(b []byte) (n int, err error) { + f.fileData.Lock() + defer f.fileData.Unlock() + if f.closed == true { + return 0, ErrFileClosed + } + if len(b) > 0 && int(f.at) == len(f.fileData.data) { + return 0, io.EOF + } + if int(f.at) > len(f.fileData.data) { + return 0, io.ErrUnexpectedEOF + } + if len(f.fileData.data)-int(f.at) >= len(b) { + n = len(b) + } else { + n = len(f.fileData.data) - int(f.at) + } + copy(b, f.fileData.data[f.at:f.at+int64(n)]) + atomic.AddInt64(&f.at, int64(n)) + return +} + +func (f *File) ReadAt(b []byte, off int64) (n int, err error) { + atomic.StoreInt64(&f.at, off) + return f.Read(b) +} + +func (f *File) Truncate(size int64) error { + if f.closed == true { + return ErrFileClosed + } + if f.readOnly { + return &os.PathError{Op: "truncate", Path: f.fileData.name, Err: errors.New("file handle is read only")} + } + if size < 0 { + return ErrOutOfRange + } + if size > int64(len(f.fileData.data)) { + diff := size - int64(len(f.fileData.data)) + f.fileData.data = append(f.fileData.data, bytes.Repeat([]byte{00}, int(diff))...) + } else { + f.fileData.data = f.fileData.data[0:size] + } + setModTime(f.fileData, time.Now()) + return nil +} + +func (f *File) Seek(offset int64, whence int) (int64, error) { + if f.closed == true { + return 0, ErrFileClosed + } + switch whence { + case 0: + atomic.StoreInt64(&f.at, offset) + case 1: + atomic.AddInt64(&f.at, int64(offset)) + case 2: + atomic.StoreInt64(&f.at, int64(len(f.fileData.data))+offset) + } + return f.at, nil +} + +func (f *File) Write(b []byte) (n int, err error) { + if f.readOnly { + return 0, &os.PathError{Op: "write", Path: f.fileData.name, Err: errors.New("file handle is read only")} + } + n = len(b) + cur := atomic.LoadInt64(&f.at) + f.fileData.Lock() + defer f.fileData.Unlock() + diff := cur - int64(len(f.fileData.data)) + var tail []byte + if n+int(cur) < len(f.fileData.data) { + tail = f.fileData.data[n+int(cur):] + } + if diff > 0 { + f.fileData.data = append(bytes.Repeat([]byte{00}, int(diff)), b...) + f.fileData.data = append(f.fileData.data, tail...) + } else { + f.fileData.data = append(f.fileData.data[:cur], b...) + f.fileData.data = append(f.fileData.data, tail...) + } + setModTime(f.fileData, time.Now()) + + atomic.StoreInt64(&f.at, int64(len(f.fileData.data))) + return +} + +func (f *File) WriteAt(b []byte, off int64) (n int, err error) { + atomic.StoreInt64(&f.at, off) + return f.Write(b) +} + +func (f *File) WriteString(s string) (ret int, err error) { + return f.Write([]byte(s)) +} + +func (f *File) Info() *FileInfo { + return &FileInfo{f.fileData} +} + +type FileInfo struct { + *FileData +} + +// Implements os.FileInfo +func (s *FileInfo) Name() string { + s.Lock() + _, name := filepath.Split(s.name) + s.Unlock() + return name +} +func (s *FileInfo) Mode() os.FileMode { + s.Lock() + defer s.Unlock() + return s.mode +} +func (s *FileInfo) ModTime() time.Time { + s.Lock() + defer s.Unlock() + return s.modtime +} +func (s *FileInfo) IsDir() bool { + s.Lock() + defer s.Unlock() + return s.dir +} +func (s *FileInfo) Sys() interface{} { return nil } +func (s *FileInfo) Size() int64 { + if s.IsDir() { + return int64(42) + } + s.Lock() + defer s.Unlock() + return int64(len(s.data)) +} + +var ( + ErrFileClosed = errors.New("File is closed") + ErrOutOfRange = errors.New("Out of range") + ErrTooLarge = errors.New("Too large") + ErrFileNotFound = os.ErrNotExist + ErrFileExists = os.ErrExist + ErrDestinationExists = os.ErrExist +) diff --git a/vendor/github.com/spf13/afero/memmap.go b/vendor/github.com/spf13/afero/memmap.go new file mode 100644 index 00000000000..09498e70fba --- /dev/null +++ b/vendor/github.com/spf13/afero/memmap.go @@ -0,0 +1,365 @@ +// Copyright © 2014 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "fmt" + "log" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/spf13/afero/mem" +) + +type MemMapFs struct { + mu sync.RWMutex + data map[string]*mem.FileData + init sync.Once +} + +func NewMemMapFs() Fs { + return &MemMapFs{} +} + +func (m *MemMapFs) getData() map[string]*mem.FileData { + m.init.Do(func() { + m.data = make(map[string]*mem.FileData) + // Root should always exist, right? + // TODO: what about windows? + m.data[FilePathSeparator] = mem.CreateDir(FilePathSeparator) + }) + return m.data +} + +func (*MemMapFs) Name() string { return "MemMapFS" } + +func (m *MemMapFs) Create(name string) (File, error) { + name = normalizePath(name) + m.mu.Lock() + file := mem.CreateFile(name) + m.getData()[name] = file + m.registerWithParent(file) + m.mu.Unlock() + return mem.NewFileHandle(file), nil +} + +func (m *MemMapFs) unRegisterWithParent(fileName string) error { + f, err := m.lockfreeOpen(fileName) + if err != nil { + return err + } + parent := m.findParent(f) + if parent == nil { + log.Panic("parent of ", f.Name(), " is nil") + } + + parent.Lock() + mem.RemoveFromMemDir(parent, f) + parent.Unlock() + return nil +} + +func (m *MemMapFs) findParent(f *mem.FileData) *mem.FileData { + pdir, _ := filepath.Split(f.Name()) + pdir = filepath.Clean(pdir) + pfile, err := m.lockfreeOpen(pdir) + if err != nil { + return nil + } + return pfile +} + +func (m *MemMapFs) registerWithParent(f *mem.FileData) { + if f == nil { + return + } + parent := m.findParent(f) + if parent == nil { + pdir := filepath.Dir(filepath.Clean(f.Name())) + err := m.lockfreeMkdir(pdir, 0777) + if err != nil { + //log.Println("Mkdir error:", err) + return + } + parent, err = m.lockfreeOpen(pdir) + if err != nil { + //log.Println("Open after Mkdir error:", err) + return + } + } + + parent.Lock() + mem.InitializeDir(parent) + mem.AddToMemDir(parent, f) + parent.Unlock() +} + +func (m *MemMapFs) lockfreeMkdir(name string, perm os.FileMode) error { + name = normalizePath(name) + x, ok := m.getData()[name] + if ok { + // Only return ErrFileExists if it's a file, not a directory. + i := mem.FileInfo{FileData: x} + if !i.IsDir() { + return ErrFileExists + } + } else { + item := mem.CreateDir(name) + m.getData()[name] = item + m.registerWithParent(item) + } + return nil +} + +func (m *MemMapFs) Mkdir(name string, perm os.FileMode) error { + name = normalizePath(name) + + m.mu.RLock() + _, ok := m.getData()[name] + m.mu.RUnlock() + if ok { + return &os.PathError{Op: "mkdir", Path: name, Err: ErrFileExists} + } + + m.mu.Lock() + item := mem.CreateDir(name) + m.getData()[name] = item + m.registerWithParent(item) + m.mu.Unlock() + + m.Chmod(name, perm|os.ModeDir) + + return nil +} + +func (m *MemMapFs) MkdirAll(path string, perm os.FileMode) error { + err := m.Mkdir(path, perm) + if err != nil { + if err.(*os.PathError).Err == ErrFileExists { + return nil + } + return err + } + return nil +} + +// Handle some relative paths +func normalizePath(path string) string { + path = filepath.Clean(path) + + switch path { + case ".": + return FilePathSeparator + case "..": + return FilePathSeparator + default: + return path + } +} + +func (m *MemMapFs) Open(name string) (File, error) { + f, err := m.open(name) + if f != nil { + return mem.NewReadOnlyFileHandle(f), err + } + return nil, err +} + +func (m *MemMapFs) openWrite(name string) (File, error) { + f, err := m.open(name) + if f != nil { + return mem.NewFileHandle(f), err + } + return nil, err +} + +func (m *MemMapFs) open(name string) (*mem.FileData, error) { + name = normalizePath(name) + + m.mu.RLock() + f, ok := m.getData()[name] + m.mu.RUnlock() + if !ok { + return nil, &os.PathError{Op: "open", Path: name, Err: ErrFileNotFound} + } + return f, nil +} + +func (m *MemMapFs) lockfreeOpen(name string) (*mem.FileData, error) { + name = normalizePath(name) + f, ok := m.getData()[name] + if ok { + return f, nil + } else { + return nil, ErrFileNotFound + } +} + +func (m *MemMapFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + chmod := false + file, err := m.openWrite(name) + if os.IsNotExist(err) && (flag&os.O_CREATE > 0) { + file, err = m.Create(name) + chmod = true + } + if err != nil { + return nil, err + } + if flag == os.O_RDONLY { + file = mem.NewReadOnlyFileHandle(file.(*mem.File).Data()) + } + if flag&os.O_APPEND > 0 { + _, err = file.Seek(0, os.SEEK_END) + if err != nil { + file.Close() + return nil, err + } + } + if flag&os.O_TRUNC > 0 && flag&(os.O_RDWR|os.O_WRONLY) > 0 { + err = file.Truncate(0) + if err != nil { + file.Close() + return nil, err + } + } + if chmod { + m.Chmod(name, perm) + } + return file, nil +} + +func (m *MemMapFs) Remove(name string) error { + name = normalizePath(name) + + m.mu.Lock() + defer m.mu.Unlock() + + if _, ok := m.getData()[name]; ok { + err := m.unRegisterWithParent(name) + if err != nil { + return &os.PathError{Op: "remove", Path: name, Err: err} + } + delete(m.getData(), name) + } else { + return &os.PathError{Op: "remove", Path: name, Err: os.ErrNotExist} + } + return nil +} + +func (m *MemMapFs) RemoveAll(path string) error { + path = normalizePath(path) + m.mu.Lock() + m.unRegisterWithParent(path) + m.mu.Unlock() + + m.mu.RLock() + defer m.mu.RUnlock() + + for p, _ := range m.getData() { + if strings.HasPrefix(p, path) { + m.mu.RUnlock() + m.mu.Lock() + delete(m.getData(), p) + m.mu.Unlock() + m.mu.RLock() + } + } + return nil +} + +func (m *MemMapFs) Rename(oldname, newname string) error { + oldname = normalizePath(oldname) + newname = normalizePath(newname) + + if oldname == newname { + return nil + } + + m.mu.RLock() + defer m.mu.RUnlock() + if _, ok := m.getData()[oldname]; ok { + m.mu.RUnlock() + m.mu.Lock() + m.unRegisterWithParent(oldname) + fileData := m.getData()[oldname] + delete(m.getData(), oldname) + mem.ChangeFileName(fileData, newname) + m.getData()[newname] = fileData + m.registerWithParent(fileData) + m.mu.Unlock() + m.mu.RLock() + } else { + return &os.PathError{Op: "rename", Path: oldname, Err: ErrFileNotFound} + } + return nil +} + +func (m *MemMapFs) Stat(name string) (os.FileInfo, error) { + f, err := m.Open(name) + if err != nil { + return nil, err + } + fi := mem.GetFileInfo(f.(*mem.File).Data()) + return fi, nil +} + +func (m *MemMapFs) Chmod(name string, mode os.FileMode) error { + name = normalizePath(name) + + m.mu.RLock() + f, ok := m.getData()[name] + m.mu.RUnlock() + if !ok { + return &os.PathError{Op: "chmod", Path: name, Err: ErrFileNotFound} + } + + m.mu.Lock() + mem.SetMode(f, mode) + m.mu.Unlock() + + return nil +} + +func (m *MemMapFs) Chtimes(name string, atime time.Time, mtime time.Time) error { + name = normalizePath(name) + + m.mu.RLock() + f, ok := m.getData()[name] + m.mu.RUnlock() + if !ok { + return &os.PathError{Op: "chtimes", Path: name, Err: ErrFileNotFound} + } + + m.mu.Lock() + mem.SetModTime(f, mtime) + m.mu.Unlock() + + return nil +} + +func (m *MemMapFs) List() { + for _, x := range m.data { + y := mem.FileInfo{FileData: x} + fmt.Println(x.Name(), y.Size()) + } +} + +// func debugMemMapList(fs Fs) { +// if x, ok := fs.(*MemMapFs); ok { +// x.List() +// } +// } diff --git a/vendor/github.com/spf13/afero/os.go b/vendor/github.com/spf13/afero/os.go new file mode 100644 index 00000000000..13cc1b84c93 --- /dev/null +++ b/vendor/github.com/spf13/afero/os.go @@ -0,0 +1,101 @@ +// Copyright © 2014 Steve Francia . +// Copyright 2013 tsuru authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "os" + "time" +) + +var _ Lstater = (*OsFs)(nil) + +// OsFs is a Fs implementation that uses functions provided by the os package. +// +// For details in any method, check the documentation of the os package +// (http://golang.org/pkg/os/). +type OsFs struct{} + +func NewOsFs() Fs { + return &OsFs{} +} + +func (OsFs) Name() string { return "OsFs" } + +func (OsFs) Create(name string) (File, error) { + f, e := os.Create(name) + if f == nil { + // while this looks strange, we need to return a bare nil (of type nil) not + // a nil value of type *os.File or nil won't be nil + return nil, e + } + return f, e +} + +func (OsFs) Mkdir(name string, perm os.FileMode) error { + return os.Mkdir(name, perm) +} + +func (OsFs) MkdirAll(path string, perm os.FileMode) error { + return os.MkdirAll(path, perm) +} + +func (OsFs) Open(name string) (File, error) { + f, e := os.Open(name) + if f == nil { + // while this looks strange, we need to return a bare nil (of type nil) not + // a nil value of type *os.File or nil won't be nil + return nil, e + } + return f, e +} + +func (OsFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + f, e := os.OpenFile(name, flag, perm) + if f == nil { + // while this looks strange, we need to return a bare nil (of type nil) not + // a nil value of type *os.File or nil won't be nil + return nil, e + } + return f, e +} + +func (OsFs) Remove(name string) error { + return os.Remove(name) +} + +func (OsFs) RemoveAll(path string) error { + return os.RemoveAll(path) +} + +func (OsFs) Rename(oldname, newname string) error { + return os.Rename(oldname, newname) +} + +func (OsFs) Stat(name string) (os.FileInfo, error) { + return os.Stat(name) +} + +func (OsFs) Chmod(name string, mode os.FileMode) error { + return os.Chmod(name, mode) +} + +func (OsFs) Chtimes(name string, atime time.Time, mtime time.Time) error { + return os.Chtimes(name, atime, mtime) +} + +func (OsFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + fi, err := os.Lstat(name) + return fi, true, err +} diff --git a/vendor/github.com/spf13/afero/path.go b/vendor/github.com/spf13/afero/path.go new file mode 100644 index 00000000000..18f60a0f6b6 --- /dev/null +++ b/vendor/github.com/spf13/afero/path.go @@ -0,0 +1,106 @@ +// Copyright ©2015 The Go Authors +// Copyright ©2015 Steve Francia +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "os" + "path/filepath" + "sort" +) + +// readDirNames reads the directory named by dirname and returns +// a sorted list of directory entries. +// adapted from https://golang.org/src/path/filepath/path.go +func readDirNames(fs Fs, dirname string) ([]string, error) { + f, err := fs.Open(dirname) + if err != nil { + return nil, err + } + names, err := f.Readdirnames(-1) + f.Close() + if err != nil { + return nil, err + } + sort.Strings(names) + return names, nil +} + +// walk recursively descends path, calling walkFn +// adapted from https://golang.org/src/path/filepath/path.go +func walk(fs Fs, path string, info os.FileInfo, walkFn filepath.WalkFunc) error { + err := walkFn(path, info, nil) + if err != nil { + if info.IsDir() && err == filepath.SkipDir { + return nil + } + return err + } + + if !info.IsDir() { + return nil + } + + names, err := readDirNames(fs, path) + if err != nil { + return walkFn(path, info, err) + } + + for _, name := range names { + filename := filepath.Join(path, name) + fileInfo, err := lstatIfPossible(fs, filename) + if err != nil { + if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { + return err + } + } else { + err = walk(fs, filename, fileInfo, walkFn) + if err != nil { + if !fileInfo.IsDir() || err != filepath.SkipDir { + return err + } + } + } + } + return nil +} + +// if the filesystem supports it, use Lstat, else use fs.Stat +func lstatIfPossible(fs Fs, path string) (os.FileInfo, error) { + if lfs, ok := fs.(Lstater); ok { + fi, _, err := lfs.LstatIfPossible(path) + return fi, err + } + return fs.Stat(path) +} + +// Walk walks the file tree rooted at root, calling walkFn for each file or +// directory in the tree, including root. All errors that arise visiting files +// and directories are filtered by walkFn. The files are walked in lexical +// order, which makes the output deterministic but means that for very +// large directories Walk can be inefficient. +// Walk does not follow symbolic links. + +func (a Afero) Walk(root string, walkFn filepath.WalkFunc) error { + return Walk(a.Fs, root, walkFn) +} + +func Walk(fs Fs, root string, walkFn filepath.WalkFunc) error { + info, err := lstatIfPossible(fs, root) + if err != nil { + return walkFn(root, nil, err) + } + return walk(fs, root, info, walkFn) +} diff --git a/vendor/github.com/spf13/afero/readonlyfs.go b/vendor/github.com/spf13/afero/readonlyfs.go new file mode 100644 index 00000000000..c6376ec373a --- /dev/null +++ b/vendor/github.com/spf13/afero/readonlyfs.go @@ -0,0 +1,80 @@ +package afero + +import ( + "os" + "syscall" + "time" +) + +var _ Lstater = (*ReadOnlyFs)(nil) + +type ReadOnlyFs struct { + source Fs +} + +func NewReadOnlyFs(source Fs) Fs { + return &ReadOnlyFs{source: source} +} + +func (r *ReadOnlyFs) ReadDir(name string) ([]os.FileInfo, error) { + return ReadDir(r.source, name) +} + +func (r *ReadOnlyFs) Chtimes(n string, a, m time.Time) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) Chmod(n string, m os.FileMode) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) Name() string { + return "ReadOnlyFilter" +} + +func (r *ReadOnlyFs) Stat(name string) (os.FileInfo, error) { + return r.source.Stat(name) +} + +func (r *ReadOnlyFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + if lsf, ok := r.source.(Lstater); ok { + return lsf.LstatIfPossible(name) + } + fi, err := r.Stat(name) + return fi, false, err +} + +func (r *ReadOnlyFs) Rename(o, n string) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) RemoveAll(p string) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) Remove(n string) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + if flag&(os.O_WRONLY|syscall.O_RDWR|os.O_APPEND|os.O_CREATE|os.O_TRUNC) != 0 { + return nil, syscall.EPERM + } + return r.source.OpenFile(name, flag, perm) +} + +func (r *ReadOnlyFs) Open(n string) (File, error) { + return r.source.Open(n) +} + +func (r *ReadOnlyFs) Mkdir(n string, p os.FileMode) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) MkdirAll(n string, p os.FileMode) error { + return syscall.EPERM +} + +func (r *ReadOnlyFs) Create(n string) (File, error) { + return nil, syscall.EPERM +} diff --git a/vendor/github.com/spf13/afero/regexpfs.go b/vendor/github.com/spf13/afero/regexpfs.go new file mode 100644 index 00000000000..9d92dbc051f --- /dev/null +++ b/vendor/github.com/spf13/afero/regexpfs.go @@ -0,0 +1,214 @@ +package afero + +import ( + "os" + "regexp" + "syscall" + "time" +) + +// The RegexpFs filters files (not directories) by regular expression. Only +// files matching the given regexp will be allowed, all others get a ENOENT error ( +// "No such file or directory"). +// +type RegexpFs struct { + re *regexp.Regexp + source Fs +} + +func NewRegexpFs(source Fs, re *regexp.Regexp) Fs { + return &RegexpFs{source: source, re: re} +} + +type RegexpFile struct { + f File + re *regexp.Regexp +} + +func (r *RegexpFs) matchesName(name string) error { + if r.re == nil { + return nil + } + if r.re.MatchString(name) { + return nil + } + return syscall.ENOENT +} + +func (r *RegexpFs) dirOrMatches(name string) error { + dir, err := IsDir(r.source, name) + if err != nil { + return err + } + if dir { + return nil + } + return r.matchesName(name) +} + +func (r *RegexpFs) Chtimes(name string, a, m time.Time) error { + if err := r.dirOrMatches(name); err != nil { + return err + } + return r.source.Chtimes(name, a, m) +} + +func (r *RegexpFs) Chmod(name string, mode os.FileMode) error { + if err := r.dirOrMatches(name); err != nil { + return err + } + return r.source.Chmod(name, mode) +} + +func (r *RegexpFs) Name() string { + return "RegexpFs" +} + +func (r *RegexpFs) Stat(name string) (os.FileInfo, error) { + if err := r.dirOrMatches(name); err != nil { + return nil, err + } + return r.source.Stat(name) +} + +func (r *RegexpFs) Rename(oldname, newname string) error { + dir, err := IsDir(r.source, oldname) + if err != nil { + return err + } + if dir { + return nil + } + if err := r.matchesName(oldname); err != nil { + return err + } + if err := r.matchesName(newname); err != nil { + return err + } + return r.source.Rename(oldname, newname) +} + +func (r *RegexpFs) RemoveAll(p string) error { + dir, err := IsDir(r.source, p) + if err != nil { + return err + } + if !dir { + if err := r.matchesName(p); err != nil { + return err + } + } + return r.source.RemoveAll(p) +} + +func (r *RegexpFs) Remove(name string) error { + if err := r.dirOrMatches(name); err != nil { + return err + } + return r.source.Remove(name) +} + +func (r *RegexpFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + if err := r.dirOrMatches(name); err != nil { + return nil, err + } + return r.source.OpenFile(name, flag, perm) +} + +func (r *RegexpFs) Open(name string) (File, error) { + dir, err := IsDir(r.source, name) + if err != nil { + return nil, err + } + if !dir { + if err := r.matchesName(name); err != nil { + return nil, err + } + } + f, err := r.source.Open(name) + return &RegexpFile{f: f, re: r.re}, nil +} + +func (r *RegexpFs) Mkdir(n string, p os.FileMode) error { + return r.source.Mkdir(n, p) +} + +func (r *RegexpFs) MkdirAll(n string, p os.FileMode) error { + return r.source.MkdirAll(n, p) +} + +func (r *RegexpFs) Create(name string) (File, error) { + if err := r.matchesName(name); err != nil { + return nil, err + } + return r.source.Create(name) +} + +func (f *RegexpFile) Close() error { + return f.f.Close() +} + +func (f *RegexpFile) Read(s []byte) (int, error) { + return f.f.Read(s) +} + +func (f *RegexpFile) ReadAt(s []byte, o int64) (int, error) { + return f.f.ReadAt(s, o) +} + +func (f *RegexpFile) Seek(o int64, w int) (int64, error) { + return f.f.Seek(o, w) +} + +func (f *RegexpFile) Write(s []byte) (int, error) { + return f.f.Write(s) +} + +func (f *RegexpFile) WriteAt(s []byte, o int64) (int, error) { + return f.f.WriteAt(s, o) +} + +func (f *RegexpFile) Name() string { + return f.f.Name() +} + +func (f *RegexpFile) Readdir(c int) (fi []os.FileInfo, err error) { + var rfi []os.FileInfo + rfi, err = f.f.Readdir(c) + if err != nil { + return nil, err + } + for _, i := range rfi { + if i.IsDir() || f.re.MatchString(i.Name()) { + fi = append(fi, i) + } + } + return fi, nil +} + +func (f *RegexpFile) Readdirnames(c int) (n []string, err error) { + fi, err := f.Readdir(c) + if err != nil { + return nil, err + } + for _, s := range fi { + n = append(n, s.Name()) + } + return n, nil +} + +func (f *RegexpFile) Stat() (os.FileInfo, error) { + return f.f.Stat() +} + +func (f *RegexpFile) Sync() error { + return f.f.Sync() +} + +func (f *RegexpFile) Truncate(s int64) error { + return f.f.Truncate(s) +} + +func (f *RegexpFile) WriteString(s string) (int, error) { + return f.f.WriteString(s) +} diff --git a/vendor/github.com/spf13/afero/unionFile.go b/vendor/github.com/spf13/afero/unionFile.go new file mode 100644 index 00000000000..eda96312df6 --- /dev/null +++ b/vendor/github.com/spf13/afero/unionFile.go @@ -0,0 +1,320 @@ +package afero + +import ( + "io" + "os" + "path/filepath" + "syscall" +) + +// The UnionFile implements the afero.File interface and will be returned +// when reading a directory present at least in the overlay or opening a file +// for writing. +// +// The calls to +// Readdir() and Readdirnames() merge the file os.FileInfo / names from the +// base and the overlay - for files present in both layers, only those +// from the overlay will be used. +// +// When opening files for writing (Create() / OpenFile() with the right flags) +// the operations will be done in both layers, starting with the overlay. A +// successful read in the overlay will move the cursor position in the base layer +// by the number of bytes read. +type UnionFile struct { + Base File + Layer File + Merger DirsMerger + off int + files []os.FileInfo +} + +func (f *UnionFile) Close() error { + // first close base, so we have a newer timestamp in the overlay. If we'd close + // the overlay first, we'd get a cacheStale the next time we access this file + // -> cache would be useless ;-) + if f.Base != nil { + f.Base.Close() + } + if f.Layer != nil { + return f.Layer.Close() + } + return BADFD +} + +func (f *UnionFile) Read(s []byte) (int, error) { + if f.Layer != nil { + n, err := f.Layer.Read(s) + if (err == nil || err == io.EOF) && f.Base != nil { + // advance the file position also in the base file, the next + // call may be a write at this position (or a seek with SEEK_CUR) + if _, seekErr := f.Base.Seek(int64(n), os.SEEK_CUR); seekErr != nil { + // only overwrite err in case the seek fails: we need to + // report an eventual io.EOF to the caller + err = seekErr + } + } + return n, err + } + if f.Base != nil { + return f.Base.Read(s) + } + return 0, BADFD +} + +func (f *UnionFile) ReadAt(s []byte, o int64) (int, error) { + if f.Layer != nil { + n, err := f.Layer.ReadAt(s, o) + if (err == nil || err == io.EOF) && f.Base != nil { + _, err = f.Base.Seek(o+int64(n), os.SEEK_SET) + } + return n, err + } + if f.Base != nil { + return f.Base.ReadAt(s, o) + } + return 0, BADFD +} + +func (f *UnionFile) Seek(o int64, w int) (pos int64, err error) { + if f.Layer != nil { + pos, err = f.Layer.Seek(o, w) + if (err == nil || err == io.EOF) && f.Base != nil { + _, err = f.Base.Seek(o, w) + } + return pos, err + } + if f.Base != nil { + return f.Base.Seek(o, w) + } + return 0, BADFD +} + +func (f *UnionFile) Write(s []byte) (n int, err error) { + if f.Layer != nil { + n, err = f.Layer.Write(s) + if err == nil && f.Base != nil { // hmm, do we have fixed size files where a write may hit the EOF mark? + _, err = f.Base.Write(s) + } + return n, err + } + if f.Base != nil { + return f.Base.Write(s) + } + return 0, BADFD +} + +func (f *UnionFile) WriteAt(s []byte, o int64) (n int, err error) { + if f.Layer != nil { + n, err = f.Layer.WriteAt(s, o) + if err == nil && f.Base != nil { + _, err = f.Base.WriteAt(s, o) + } + return n, err + } + if f.Base != nil { + return f.Base.WriteAt(s, o) + } + return 0, BADFD +} + +func (f *UnionFile) Name() string { + if f.Layer != nil { + return f.Layer.Name() + } + return f.Base.Name() +} + +// DirsMerger is how UnionFile weaves two directories together. +// It takes the FileInfo slices from the layer and the base and returns a +// single view. +type DirsMerger func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) + +var defaultUnionMergeDirsFn = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) { + var files = make(map[string]os.FileInfo) + + for _, fi := range lofi { + files[fi.Name()] = fi + } + + for _, fi := range bofi { + if _, exists := files[fi.Name()]; !exists { + files[fi.Name()] = fi + } + } + + rfi := make([]os.FileInfo, len(files)) + + i := 0 + for _, fi := range files { + rfi[i] = fi + i++ + } + + return rfi, nil + +} + +// Readdir will weave the two directories together and +// return a single view of the overlayed directories. +// At the end of the directory view, the error is io.EOF if c > 0. +func (f *UnionFile) Readdir(c int) (ofi []os.FileInfo, err error) { + var merge DirsMerger = f.Merger + if merge == nil { + merge = defaultUnionMergeDirsFn + } + + if f.off == 0 { + var lfi []os.FileInfo + if f.Layer != nil { + lfi, err = f.Layer.Readdir(-1) + if err != nil { + return nil, err + } + } + + var bfi []os.FileInfo + if f.Base != nil { + bfi, err = f.Base.Readdir(-1) + if err != nil { + return nil, err + } + + } + merged, err := merge(lfi, bfi) + if err != nil { + return nil, err + } + f.files = append(f.files, merged...) + } + + if c <= 0 && len(f.files) == 0 { + return f.files, nil + } + + if f.off >= len(f.files) { + return nil, io.EOF + } + + if c <= 0 { + return f.files[f.off:], nil + } + + if c > len(f.files) { + c = len(f.files) + } + + defer func() { f.off += c }() + return f.files[f.off:c], nil +} + +func (f *UnionFile) Readdirnames(c int) ([]string, error) { + rfi, err := f.Readdir(c) + if err != nil { + return nil, err + } + var names []string + for _, fi := range rfi { + names = append(names, fi.Name()) + } + return names, nil +} + +func (f *UnionFile) Stat() (os.FileInfo, error) { + if f.Layer != nil { + return f.Layer.Stat() + } + if f.Base != nil { + return f.Base.Stat() + } + return nil, BADFD +} + +func (f *UnionFile) Sync() (err error) { + if f.Layer != nil { + err = f.Layer.Sync() + if err == nil && f.Base != nil { + err = f.Base.Sync() + } + return err + } + if f.Base != nil { + return f.Base.Sync() + } + return BADFD +} + +func (f *UnionFile) Truncate(s int64) (err error) { + if f.Layer != nil { + err = f.Layer.Truncate(s) + if err == nil && f.Base != nil { + err = f.Base.Truncate(s) + } + return err + } + if f.Base != nil { + return f.Base.Truncate(s) + } + return BADFD +} + +func (f *UnionFile) WriteString(s string) (n int, err error) { + if f.Layer != nil { + n, err = f.Layer.WriteString(s) + if err == nil && f.Base != nil { + _, err = f.Base.WriteString(s) + } + return n, err + } + if f.Base != nil { + return f.Base.WriteString(s) + } + return 0, BADFD +} + +func copyToLayer(base Fs, layer Fs, name string) error { + bfh, err := base.Open(name) + if err != nil { + return err + } + defer bfh.Close() + + // First make sure the directory exists + exists, err := Exists(layer, filepath.Dir(name)) + if err != nil { + return err + } + if !exists { + err = layer.MkdirAll(filepath.Dir(name), 0777) // FIXME? + if err != nil { + return err + } + } + + // Create the file on the overlay + lfh, err := layer.Create(name) + if err != nil { + return err + } + n, err := io.Copy(lfh, bfh) + if err != nil { + // If anything fails, clean up the file + layer.Remove(name) + lfh.Close() + return err + } + + bfi, err := bfh.Stat() + if err != nil || bfi.Size() != n { + layer.Remove(name) + lfh.Close() + return syscall.EIO + } + + err = lfh.Close() + if err != nil { + layer.Remove(name) + lfh.Close() + return err + } + return layer.Chtimes(name, bfi.ModTime(), bfi.ModTime()) +} diff --git a/vendor/github.com/spf13/afero/util.go b/vendor/github.com/spf13/afero/util.go new file mode 100644 index 00000000000..4f253f481ed --- /dev/null +++ b/vendor/github.com/spf13/afero/util.go @@ -0,0 +1,330 @@ +// Copyright ©2015 Steve Francia +// Portions Copyright ©2015 The Hugo Authors +// Portions Copyright 2016-present Bjørn Erik Pedersen +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "unicode" + + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" +) + +// Filepath separator defined by os.Separator. +const FilePathSeparator = string(filepath.Separator) + +// Takes a reader and a path and writes the content +func (a Afero) WriteReader(path string, r io.Reader) (err error) { + return WriteReader(a.Fs, path, r) +} + +func WriteReader(fs Fs, path string, r io.Reader) (err error) { + dir, _ := filepath.Split(path) + ospath := filepath.FromSlash(dir) + + if ospath != "" { + err = fs.MkdirAll(ospath, 0777) // rwx, rw, r + if err != nil { + if err != os.ErrExist { + return err + } + } + } + + file, err := fs.Create(path) + if err != nil { + return + } + defer file.Close() + + _, err = io.Copy(file, r) + return +} + +// Same as WriteReader but checks to see if file/directory already exists. +func (a Afero) SafeWriteReader(path string, r io.Reader) (err error) { + return SafeWriteReader(a.Fs, path, r) +} + +func SafeWriteReader(fs Fs, path string, r io.Reader) (err error) { + dir, _ := filepath.Split(path) + ospath := filepath.FromSlash(dir) + + if ospath != "" { + err = fs.MkdirAll(ospath, 0777) // rwx, rw, r + if err != nil { + return + } + } + + exists, err := Exists(fs, path) + if err != nil { + return + } + if exists { + return fmt.Errorf("%v already exists", path) + } + + file, err := fs.Create(path) + if err != nil { + return + } + defer file.Close() + + _, err = io.Copy(file, r) + return +} + +func (a Afero) GetTempDir(subPath string) string { + return GetTempDir(a.Fs, subPath) +} + +// GetTempDir returns the default temp directory with trailing slash +// if subPath is not empty then it will be created recursively with mode 777 rwx rwx rwx +func GetTempDir(fs Fs, subPath string) string { + addSlash := func(p string) string { + if FilePathSeparator != p[len(p)-1:] { + p = p + FilePathSeparator + } + return p + } + dir := addSlash(os.TempDir()) + + if subPath != "" { + // preserve windows backslash :-( + if FilePathSeparator == "\\" { + subPath = strings.Replace(subPath, "\\", "____", -1) + } + dir = dir + UnicodeSanitize((subPath)) + if FilePathSeparator == "\\" { + dir = strings.Replace(dir, "____", "\\", -1) + } + + if exists, _ := Exists(fs, dir); exists { + return addSlash(dir) + } + + err := fs.MkdirAll(dir, 0777) + if err != nil { + panic(err) + } + dir = addSlash(dir) + } + return dir +} + +// Rewrite string to remove non-standard path characters +func UnicodeSanitize(s string) string { + source := []rune(s) + target := make([]rune, 0, len(source)) + + for _, r := range source { + if unicode.IsLetter(r) || + unicode.IsDigit(r) || + unicode.IsMark(r) || + r == '.' || + r == '/' || + r == '\\' || + r == '_' || + r == '-' || + r == '%' || + r == ' ' || + r == '#' { + target = append(target, r) + } + } + + return string(target) +} + +// Transform characters with accents into plain forms. +func NeuterAccents(s string) string { + t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC) + result, _, _ := transform.String(t, string(s)) + + return result +} + +func isMn(r rune) bool { + return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks +} + +func (a Afero) FileContainsBytes(filename string, subslice []byte) (bool, error) { + return FileContainsBytes(a.Fs, filename, subslice) +} + +// Check if a file contains a specified byte slice. +func FileContainsBytes(fs Fs, filename string, subslice []byte) (bool, error) { + f, err := fs.Open(filename) + if err != nil { + return false, err + } + defer f.Close() + + return readerContainsAny(f, subslice), nil +} + +func (a Afero) FileContainsAnyBytes(filename string, subslices [][]byte) (bool, error) { + return FileContainsAnyBytes(a.Fs, filename, subslices) +} + +// Check if a file contains any of the specified byte slices. +func FileContainsAnyBytes(fs Fs, filename string, subslices [][]byte) (bool, error) { + f, err := fs.Open(filename) + if err != nil { + return false, err + } + defer f.Close() + + return readerContainsAny(f, subslices...), nil +} + +// readerContains reports whether any of the subslices is within r. +func readerContainsAny(r io.Reader, subslices ...[]byte) bool { + + if r == nil || len(subslices) == 0 { + return false + } + + largestSlice := 0 + + for _, sl := range subslices { + if len(sl) > largestSlice { + largestSlice = len(sl) + } + } + + if largestSlice == 0 { + return false + } + + bufflen := largestSlice * 4 + halflen := bufflen / 2 + buff := make([]byte, bufflen) + var err error + var n, i int + + for { + i++ + if i == 1 { + n, err = io.ReadAtLeast(r, buff[:halflen], halflen) + } else { + if i != 2 { + // shift left to catch overlapping matches + copy(buff[:], buff[halflen:]) + } + n, err = io.ReadAtLeast(r, buff[halflen:], halflen) + } + + if n > 0 { + for _, sl := range subslices { + if bytes.Contains(buff, sl) { + return true + } + } + } + + if err != nil { + break + } + } + return false +} + +func (a Afero) DirExists(path string) (bool, error) { + return DirExists(a.Fs, path) +} + +// DirExists checks if a path exists and is a directory. +func DirExists(fs Fs, path string) (bool, error) { + fi, err := fs.Stat(path) + if err == nil && fi.IsDir() { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +func (a Afero) IsDir(path string) (bool, error) { + return IsDir(a.Fs, path) +} + +// IsDir checks if a given path is a directory. +func IsDir(fs Fs, path string) (bool, error) { + fi, err := fs.Stat(path) + if err != nil { + return false, err + } + return fi.IsDir(), nil +} + +func (a Afero) IsEmpty(path string) (bool, error) { + return IsEmpty(a.Fs, path) +} + +// IsEmpty checks if a given file or directory is empty. +func IsEmpty(fs Fs, path string) (bool, error) { + if b, _ := Exists(fs, path); !b { + return false, fmt.Errorf("%q path does not exist", path) + } + fi, err := fs.Stat(path) + if err != nil { + return false, err + } + if fi.IsDir() { + f, err := fs.Open(path) + if err != nil { + return false, err + } + defer f.Close() + list, err := f.Readdir(-1) + return len(list) == 0, nil + } + return fi.Size() == 0, nil +} + +func (a Afero) Exists(path string) (bool, error) { + return Exists(a.Fs, path) +} + +// Check if a file or directory exists. +func Exists(fs Fs, path string) (bool, error) { + _, err := fs.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +func FullBaseFsPath(basePathFs *BasePathFs, relativePath string) string { + combinedPath := filepath.Join(basePathFs.path, relativePath) + if parent, ok := basePathFs.source.(*BasePathFs); ok { + return FullBaseFsPath(parent, combinedPath) + } + + return combinedPath +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 498087ac02d..dba8d632a15 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -392,6 +392,9 @@ github.com/sercand/kuberesolver github.com/sirupsen/logrus # github.com/soheilhy/cmux v0.1.4 github.com/soheilhy/cmux +# github.com/spf13/afero v1.2.2 +github.com/spf13/afero +github.com/spf13/afero/mem # github.com/spf13/pflag v1.0.3 github.com/spf13/pflag # github.com/stretchr/testify v1.3.0 @@ -581,17 +584,17 @@ golang.org/x/sync/semaphore golang.org/x/sys/windows golang.org/x/sys/unix # golang.org/x/text v0.3.2 +golang.org/x/text/transform +golang.org/x/text/unicode/norm golang.org/x/text/secure/bidirule golang.org/x/text/unicode/bidi -golang.org/x/text/unicode/norm -golang.org/x/text/transform # golang.org/x/time v0.0.0-20181108054448-85acf8d2951c golang.org/x/time/rate # google.golang.org/api v0.4.0 google.golang.org/api/option google.golang.org/api/transport/http -google.golang.org/api/cloudresourcemanager/v1 google.golang.org/api/iterator +google.golang.org/api/cloudresourcemanager/v1 google.golang.org/api/transport/grpc google.golang.org/api/googleapi google.golang.org/api/storage/v1 From 726e7561b5658a4bd0bdba1ef8283331ececa964 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 6 Sep 2019 12:07:39 -0400 Subject: [PATCH 2/5] fix proto rules after rebase Signed-off-by: Jacob Lisi --- pkg/ruler/rules/compat.go | 4 ++-- pkg/ruler/rules/rules.proto | 41 ++++++++++++++++++++++--------------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/pkg/ruler/rules/compat.go b/pkg/ruler/rules/compat.go index 5e535c1cffb..2d9422ee20d 100644 --- a/pkg/ruler/rules/compat.go +++ b/pkg/ruler/rules/compat.go @@ -34,8 +34,8 @@ func formattedRuleToProto(rls []rulefmt.Rule) []*RuleDesc { Alert: rls[i].Alert, For: &f, - Labels: client.FromLabelsToLabelAdapaters(labels.FromMap(rls[i].Labels)), - Annotations: client.FromLabelsToLabelAdapaters(labels.FromMap(rls[i].Annotations)), + Labels: client.FromLabelsToLabelAdapters(labels.FromMap(rls[i].Labels)), + Annotations: client.FromLabelsToLabelAdapters(labels.FromMap(rls[i].Annotations)), } } diff --git a/pkg/ruler/rules/rules.proto b/pkg/ruler/rules/rules.proto index 1043668d45b..9a52ce4636b 100644 --- a/pkg/ruler/rules/rules.proto +++ b/pkg/ruler/rules/rules.proto @@ -12,22 +12,29 @@ option (gogoproto.marshaler_all) = true; option (gogoproto.unmarshaler_all) = true; message RuleGroupDesc { - string name = 1; - string namespace = 2; - google.protobuf.Duration interval = 3 [(gogoproto.stdduration) = true]; - - repeated RuleDesc rules = 4; - - bool deleted = 5; + string name = 1; + string namespace = 2; + google.protobuf.Duration interval = 3 [ (gogoproto.stdduration) = true ]; + repeated RuleDesc rules = 4; + bool deleted = 5; + string user = 6; +} - string user = 6; - } - - message RuleDesc { - string expr = 1; - string record = 2; - string alert = 3; +message RuleDesc { + string expr = 1; + string record = 2; + string alert = 3; google.protobuf.Duration for = 4 [(gogoproto.stdduration) = true]; - repeated cortex.LabelPair labels = 5 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter"];; - repeated cortex.LabelPair annotations = 6 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter"];; - } \ No newline at end of file + repeated cortex.LabelPair labels = 5 [ + (gogoproto.nullable) = false, + (gogoproto.customtype) = + "github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter" + ]; + ; + repeated cortex.LabelPair annotations = 6 [ + (gogoproto.nullable) = false, + (gogoproto.customtype) = + "github.com/cortexproject/cortex/pkg/ingester/client.LabelAdapter" + ]; + ; +} \ No newline at end of file From 8e2fad71305121bf76011c0b44e43c0451f6f341 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 6 Sep 2019 12:11:43 -0400 Subject: [PATCH 3/5] go mod vendor Signed-off-by: Jacob Lisi --- vendor/modules.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/modules.txt b/vendor/modules.txt index dba8d632a15..7124bf21efb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -593,8 +593,8 @@ golang.org/x/time/rate # google.golang.org/api v0.4.0 google.golang.org/api/option google.golang.org/api/transport/http -google.golang.org/api/iterator google.golang.org/api/cloudresourcemanager/v1 +google.golang.org/api/iterator google.golang.org/api/transport/grpc google.golang.org/api/googleapi google.golang.org/api/storage/v1 From c130cac31db9ff59918760d0b5e7bdd41da69196 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 6 Sep 2019 12:32:26 -0400 Subject: [PATCH 4/5] update rules protos Signed-off-by: Jacob Lisi --- Makefile | 2 +- pkg/ruler/rules/rules.pb.go | 314 ++++-------------------------------- 2 files changed, 32 insertions(+), 284 deletions(-) diff --git a/Makefile b/Makefile index 66d6d7dae20..e3064152703 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ pkg/querier/frontend/frontend.pb.go: pkg/querier/frontend/frontend.proto pkg/querier/queryrange/queryrange.pb.go: pkg/querier/queryrange/queryrange.proto pkg/chunk/storage/caching_index_client.pb.go: pkg/chunk/storage/caching_index_client.proto pkg/distributor/ha_tracker.pb.go: pkg/distributor/ha_tracker.proto -/pkg/ruler/rules/rules.pb.go: /pkg/ruler/rules/rules.proto +pkg/ruler/rules/rules.pb.go: pkg/ruler/rules/rules.proto all: $(UPTODATE_FILES) test: protos mod-check: protos diff --git a/pkg/ruler/rules/rules.pb.go b/pkg/ruler/rules/rules.pb.go index d4c15283775..a8da0c4e2c3 100644 --- a/pkg/ruler/rules/rules.pb.go +++ b/pkg/ruler/rules/rules.pb.go @@ -30,57 +30,6 @@ var _ = time.Kitchen // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package -type RuleUpdateDesc struct { - User string `protobuf:"bytes,1,opt,name=user,proto3" json:"user,omitempty"` - UpdatedAt int64 `protobuf:"varint,2,opt,name=updatedAt,proto3" json:"updatedAt,omitempty"` -} - -func (m *RuleUpdateDesc) Reset() { *m = RuleUpdateDesc{} } -func (*RuleUpdateDesc) ProtoMessage() {} -func (*RuleUpdateDesc) Descriptor() ([]byte, []int) { - return fileDescriptor_8e722d3e922f0937, []int{0} -} -func (m *RuleUpdateDesc) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *RuleUpdateDesc) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_RuleUpdateDesc.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *RuleUpdateDesc) XXX_Merge(src proto.Message) { - xxx_messageInfo_RuleUpdateDesc.Merge(m, src) -} -func (m *RuleUpdateDesc) XXX_Size() int { - return m.Size() -} -func (m *RuleUpdateDesc) XXX_DiscardUnknown() { - xxx_messageInfo_RuleUpdateDesc.DiscardUnknown(m) -} - -var xxx_messageInfo_RuleUpdateDesc proto.InternalMessageInfo - -func (m *RuleUpdateDesc) GetUser() string { - if m != nil { - return m.User - } - return "" -} - -func (m *RuleUpdateDesc) GetUpdatedAt() int64 { - if m != nil { - return m.UpdatedAt - } - return 0 -} - type RuleGroupDesc struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Namespace string `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` @@ -93,7 +42,7 @@ type RuleGroupDesc struct { func (m *RuleGroupDesc) Reset() { *m = RuleGroupDesc{} } func (*RuleGroupDesc) ProtoMessage() {} func (*RuleGroupDesc) Descriptor() ([]byte, []int) { - return fileDescriptor_8e722d3e922f0937, []int{1} + return fileDescriptor_8e722d3e922f0937, []int{0} } func (m *RuleGroupDesc) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -176,7 +125,7 @@ type RuleDesc struct { func (m *RuleDesc) Reset() { *m = RuleDesc{} } func (*RuleDesc) ProtoMessage() {} func (*RuleDesc) Descriptor() ([]byte, []int) { - return fileDescriptor_8e722d3e922f0937, []int{2} + return fileDescriptor_8e722d3e922f0937, []int{1} } func (m *RuleDesc) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -234,7 +183,6 @@ func (m *RuleDesc) GetFor() *time.Duration { } func init() { - proto.RegisterType((*RuleUpdateDesc)(nil), "rules.RuleUpdateDesc") proto.RegisterType((*RuleGroupDesc)(nil), "rules.RuleGroupDesc") proto.RegisterType((*RuleDesc)(nil), "rules.RuleDesc") } @@ -242,66 +190,37 @@ func init() { func init() { proto.RegisterFile("rules.proto", fileDescriptor_8e722d3e922f0937) } var fileDescriptor_8e722d3e922f0937 = []byte{ - // 478 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x92, 0xbf, 0x8e, 0xd4, 0x30, - 0x10, 0xc6, 0xe3, 0xdb, 0x3f, 0xec, 0x7a, 0x05, 0x08, 0x0b, 0x21, 0x73, 0x42, 0xde, 0x68, 0x25, - 0xa4, 0x34, 0x24, 0xe2, 0x28, 0xaf, 0xe1, 0x56, 0x27, 0x41, 0x41, 0x81, 0x2c, 0xd1, 0xd0, 0x79, - 0x93, 0xb9, 0x10, 0xf0, 0xc5, 0x91, 0xe3, 0x20, 0x1a, 0x24, 0x1e, 0x81, 0x92, 0x47, 0xe0, 0x51, - 0xae, 0x5c, 0x89, 0xe6, 0x44, 0x71, 0xb0, 0xd9, 0x86, 0xf2, 0x24, 0x5e, 0x00, 0xd9, 0x4e, 0x6e, - 0xaf, 0x44, 0x48, 0x54, 0x99, 0xcf, 0x33, 0xfe, 0x66, 0x7e, 0xe3, 0xe0, 0x99, 0x6e, 0x24, 0xd4, - 0x71, 0xa5, 0x95, 0x51, 0x64, 0xe4, 0xc4, 0xfe, 0xa3, 0xbc, 0x30, 0x6f, 0x9a, 0x55, 0x9c, 0xaa, - 0xd3, 0x24, 0x57, 0xb9, 0x4a, 0x5c, 0x76, 0xd5, 0x9c, 0x38, 0xe5, 0x84, 0x8b, 0xfc, 0xad, 0x7d, - 0x96, 0x2b, 0x95, 0x4b, 0xd8, 0x55, 0x65, 0x8d, 0x16, 0xa6, 0x50, 0x65, 0x97, 0x7f, 0x7a, 0xcd, - 0x2e, 0x55, 0xda, 0xc0, 0x87, 0x4a, 0xab, 0xb7, 0x90, 0x9a, 0x4e, 0x25, 0xd5, 0xbb, 0x3c, 0x29, - 0xca, 0x1c, 0x6a, 0x03, 0x3a, 0x49, 0x65, 0x01, 0x65, 0x9f, 0xf2, 0x0e, 0x8b, 0x25, 0xbe, 0xc5, - 0x1b, 0x09, 0xaf, 0xaa, 0x4c, 0x18, 0x38, 0x86, 0x3a, 0x25, 0x04, 0x0f, 0x9b, 0x1a, 0x34, 0x45, - 0x21, 0x8a, 0xa6, 0xdc, 0xc5, 0xe4, 0x01, 0x9e, 0x36, 0xae, 0x22, 0x3b, 0x32, 0x74, 0x2f, 0x44, - 0xd1, 0x80, 0xef, 0x0e, 0x16, 0xdf, 0x10, 0xbe, 0x69, 0x4d, 0x9e, 0x69, 0xd5, 0x54, 0xbd, 0x47, - 0x29, 0x4e, 0xa1, 0xf7, 0xb0, 0xb1, 0xf5, 0xb0, 0xdf, 0xba, 0x12, 0x29, 0x38, 0x8f, 0x29, 0xdf, - 0x1d, 0x90, 0x43, 0x3c, 0x29, 0x4a, 0x03, 0xfa, 0xbd, 0x90, 0x74, 0x10, 0xa2, 0x68, 0x76, 0x70, - 0x3f, 0xf6, 0xf0, 0x71, 0x0f, 0x1f, 0x1f, 0x77, 0xf0, 0xcb, 0xe1, 0x97, 0x1f, 0x73, 0xc4, 0xaf, - 0x2e, 0x90, 0x87, 0xd8, 0xaf, 0x97, 0x0e, 0xc3, 0x41, 0x34, 0x3b, 0xb8, 0x1d, 0xfb, 0xcd, 0xdb, - 0x99, 0xec, 0x38, 0xdc, 0x67, 0x09, 0xc5, 0x37, 0x32, 0x90, 0x60, 0x20, 0xa3, 0xa3, 0x10, 0x45, - 0x13, 0xde, 0xcb, 0x2b, 0xe6, 0xf1, 0x8e, 0x79, 0xf1, 0x7b, 0x0f, 0x4f, 0x7a, 0x07, 0x5b, 0x60, - 0x77, 0xdb, 0x03, 0xd9, 0x98, 0xdc, 0xc3, 0x63, 0x0d, 0xa9, 0xd2, 0x59, 0x47, 0xd3, 0x29, 0x72, - 0x17, 0x8f, 0x84, 0x04, 0x6d, 0x1c, 0xc7, 0x94, 0x7b, 0x41, 0x1e, 0xe3, 0xc1, 0x89, 0xd2, 0x74, - 0xf8, 0x77, 0x6c, 0xb6, 0x96, 0xd4, 0x78, 0x2c, 0xc5, 0x0a, 0x64, 0x4d, 0x47, 0x8e, 0xeb, 0x4e, - 0xdc, 0x3d, 0xdd, 0x0b, 0x7b, 0xfa, 0x52, 0x14, 0x7a, 0xf9, 0xfc, 0xec, 0x62, 0x1e, 0x7c, 0xbf, - 0x98, 0xff, 0xcb, 0x8f, 0xe0, 0x6d, 0x8e, 0x32, 0x51, 0x19, 0xd0, 0xbc, 0x6b, 0x45, 0x3e, 0xe2, - 0x99, 0x28, 0x4b, 0x65, 0xdc, 0x34, 0x35, 0x1d, 0xff, 0xff, 0xce, 0xd7, 0xfb, 0x2d, 0x0f, 0xd7, - 0x1b, 0x16, 0x9c, 0x6f, 0x58, 0x70, 0xb9, 0x61, 0xe8, 0x53, 0xcb, 0xd0, 0xd7, 0x96, 0xa1, 0xb3, - 0x96, 0xa1, 0x75, 0xcb, 0xd0, 0xcf, 0x96, 0xa1, 0x5f, 0x2d, 0x0b, 0x2e, 0x5b, 0x86, 0x3e, 0x6f, - 0x59, 0xb0, 0xde, 0xb2, 0xe0, 0x7c, 0xcb, 0x82, 0xd7, 0xfe, 0x81, 0x57, 0x63, 0xb7, 0xce, 0x27, - 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x70, 0xb7, 0x00, 0xbd, 0x7a, 0x03, 0x00, 0x00, + // 448 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x51, 0x41, 0x8b, 0xd4, 0x30, + 0x14, 0x6e, 0x76, 0x66, 0xea, 0x4c, 0x06, 0x11, 0x83, 0x48, 0x5c, 0x24, 0x53, 0x16, 0x84, 0x5e, + 0x6c, 0x71, 0x3d, 0xee, 0x45, 0x87, 0x05, 0x3d, 0x78, 0x90, 0x1e, 0xbd, 0xa5, 0xed, 0xdb, 0x5a, + 0xcd, 0x36, 0x25, 0x49, 0xc5, 0x8b, 0xe0, 0x4f, 0xf0, 0xe8, 0x4f, 0xf0, 0xa7, 0xec, 0x71, 0xc0, + 0xcb, 0xe2, 0x61, 0x75, 0x3a, 0x17, 0x8f, 0x0b, 0xfe, 0x01, 0x49, 0xd2, 0xba, 0x7b, 0x14, 0xc1, + 0x53, 0xde, 0x97, 0xef, 0xe5, 0xbd, 0xef, 0xfb, 0x82, 0x97, 0xaa, 0x13, 0xa0, 0x93, 0x56, 0x49, + 0x23, 0xc9, 0xcc, 0x81, 0xfd, 0x87, 0x55, 0x6d, 0x5e, 0x77, 0x79, 0x52, 0xc8, 0xd3, 0xb4, 0x92, + 0x95, 0x4c, 0x1d, 0x9b, 0x77, 0x27, 0x0e, 0x39, 0xe0, 0x2a, 0xff, 0x6a, 0x9f, 0x55, 0x52, 0x56, + 0x02, 0xae, 0xba, 0xca, 0x4e, 0x71, 0x53, 0xcb, 0x66, 0xe0, 0x9f, 0x5c, 0x1b, 0x57, 0x48, 0x65, + 0xe0, 0x7d, 0xab, 0xe4, 0x1b, 0x28, 0xcc, 0x80, 0xd2, 0xf6, 0x6d, 0x95, 0xd6, 0x4d, 0x05, 0xda, + 0x80, 0x4a, 0x0b, 0x51, 0x43, 0x33, 0x52, 0x7e, 0xc2, 0xc1, 0x57, 0x84, 0x6f, 0x66, 0x9d, 0x80, + 0x67, 0x4a, 0x76, 0xed, 0x31, 0xe8, 0x82, 0x10, 0x3c, 0x6d, 0xf8, 0x29, 0x50, 0x14, 0xa1, 0x78, + 0x91, 0xb9, 0x9a, 0xdc, 0xc7, 0x0b, 0x7b, 0xea, 0x96, 0x17, 0x40, 0xf7, 0x1c, 0x71, 0x75, 0x41, + 0x8e, 0xf0, 0xbc, 0x6e, 0x0c, 0xa8, 0x77, 0x5c, 0xd0, 0x49, 0x84, 0xe2, 0xe5, 0xe1, 0xbd, 0xc4, + 0x0b, 0x4f, 0x46, 0xe1, 0xc9, 0xf1, 0x20, 0x7c, 0x3d, 0xfd, 0xfc, 0x7d, 0x85, 0xb2, 0x3f, 0x0f, + 0xc8, 0x03, 0xec, 0xa3, 0xa1, 0xd3, 0x68, 0x12, 0x2f, 0x0f, 0x6f, 0x25, 0x3e, 0x35, 0xab, 0xc9, + 0xca, 0xc9, 0x3c, 0x4b, 0x28, 0xbe, 0x51, 0x82, 0x00, 0x03, 0x25, 0x9d, 0x45, 0x28, 0x9e, 0x67, + 0x23, 0xb4, 0x7a, 0x3b, 0x0d, 0x8a, 0x86, 0x5e, 0xaf, 0xad, 0x0f, 0x7e, 0xed, 0xe1, 0xf9, 0x38, + 0xc1, 0x36, 0xd8, 0x5c, 0x46, 0x43, 0xb6, 0x26, 0x77, 0x71, 0xa8, 0xa0, 0x90, 0xaa, 0x1c, 0xdc, + 0x0c, 0x88, 0xdc, 0xc1, 0x33, 0x2e, 0x40, 0x19, 0xe7, 0x63, 0x91, 0x79, 0x40, 0x1e, 0xe1, 0xc9, + 0x89, 0x54, 0x74, 0xfa, 0x77, 0xde, 0x6c, 0x2f, 0xd1, 0x38, 0x14, 0x3c, 0x07, 0xa1, 0xe9, 0xcc, + 0xf9, 0xba, 0x9d, 0x0c, 0xb1, 0xbf, 0xb0, 0xb7, 0x2f, 0x79, 0xad, 0xd6, 0xcf, 0xcf, 0x2e, 0x56, + 0xc1, 0xb7, 0x8b, 0xd5, 0xbf, 0x7c, 0xa2, 0x1f, 0xf3, 0xb4, 0xe4, 0xad, 0x01, 0x95, 0x0d, 0xab, + 0xc8, 0x07, 0xbc, 0xe4, 0x4d, 0x23, 0x8d, 0x53, 0xa3, 0x69, 0xf8, 0xff, 0x37, 0x5f, 0xdf, 0xb7, + 0x3e, 0xda, 0x6c, 0x59, 0x70, 0xbe, 0x65, 0xc1, 0xe5, 0x96, 0xa1, 0x8f, 0x3d, 0x43, 0x5f, 0x7a, + 0x86, 0xce, 0x7a, 0x86, 0x36, 0x3d, 0x43, 0x3f, 0x7a, 0x86, 0x7e, 0xf6, 0x2c, 0xb8, 0xec, 0x19, + 0xfa, 0xb4, 0x63, 0xc1, 0x66, 0xc7, 0x82, 0xf3, 0x1d, 0x0b, 0x5e, 0xf9, 0x0f, 0xce, 0x43, 0x17, + 0xe7, 0xe3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x67, 0x3a, 0xe5, 0x13, 0x36, 0x03, 0x00, 0x00, } -func (this *RuleUpdateDesc) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*RuleUpdateDesc) - if !ok { - that2, ok := that.(RuleUpdateDesc) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if this.User != that1.User { - return false - } - if this.UpdatedAt != that1.UpdatedAt { - return false - } - return true -} func (this *RuleGroupDesc) Equal(that interface{}) bool { if that == nil { return this == nil @@ -407,17 +326,6 @@ func (this *RuleDesc) Equal(that interface{}) bool { } return true } -func (this *RuleUpdateDesc) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 6) - s = append(s, "&rules.RuleUpdateDesc{") - s = append(s, "User: "+fmt.Sprintf("%#v", this.User)+",\n") - s = append(s, "UpdatedAt: "+fmt.Sprintf("%#v", this.UpdatedAt)+",\n") - s = append(s, "}") - return strings.Join(s, "") -} func (this *RuleGroupDesc) GoString() string { if this == nil { return "nil" @@ -458,35 +366,6 @@ func valueToGoStringRules(v interface{}, typ string) string { pv := reflect.Indirect(rv).Interface() return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) } -func (m *RuleUpdateDesc) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *RuleUpdateDesc) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.User) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintRules(dAtA, i, uint64(len(m.User))) - i += copy(dAtA[i:], m.User) - } - if m.UpdatedAt != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintRules(dAtA, i, uint64(m.UpdatedAt)) - } - return i, nil -} - func (m *RuleGroupDesc) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -634,22 +513,6 @@ func encodeVarintRules(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return offset + 1 } -func (m *RuleUpdateDesc) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.User) - if l > 0 { - n += 1 + l + sovRules(uint64(l)) - } - if m.UpdatedAt != 0 { - n += 1 + sovRules(uint64(m.UpdatedAt)) - } - return n -} - func (m *RuleGroupDesc) Size() (n int) { if m == nil { return 0 @@ -734,17 +597,6 @@ func sovRules(x uint64) (n int) { func sozRules(x uint64) (n int) { return sovRules(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (this *RuleUpdateDesc) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&RuleUpdateDesc{`, - `User:` + fmt.Sprintf("%v", this.User) + `,`, - `UpdatedAt:` + fmt.Sprintf("%v", this.UpdatedAt) + `,`, - `}`, - }, "") - return s -} func (this *RuleGroupDesc) String() string { if this == nil { return "nil" @@ -783,110 +635,6 @@ func valueToStringRules(v interface{}) string { pv := reflect.Indirect(rv).Interface() return fmt.Sprintf("*%v", pv) } -func (m *RuleUpdateDesc) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRules - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: RuleUpdateDesc: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: RuleUpdateDesc: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field User", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRules - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthRules - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthRules - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.User = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field UpdatedAt", wireType) - } - m.UpdatedAt = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRules - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.UpdatedAt |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipRules(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthRules - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthRules - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 From 99a942ecefc3fbd7d292e2f93ffd6f964caecce8 Mon Sep 17 00:00:00 2001 From: Jacob Lisi Date: Fri, 6 Sep 2019 12:58:25 -0400 Subject: [PATCH 5/5] add rules proto to circle ci Signed-off-by: Jacob Lisi --- .circleci/config.yml | 244 ++++++++++++++++++++++--------------------- 1 file changed, 123 insertions(+), 121 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4dc4a13d6b6..6b13f609786 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,70 +11,70 @@ workflows: test-build-deploy: # You must specify a tag filter for each job that deploy depends on. jobs: - - lint: - filters: - tags: - only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ - - test: - filters: - tags: - only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ - - integration: - requires: - - build - filters: - tags: - only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ - - build: - filters: - tags: - only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ - - deploy: - requires: - - build - - test - - lint - - integration - filters: - branches: - only: master - tags: - only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + - lint: + filters: + tags: + only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + - test: + filters: + tags: + only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + - integration: + requires: + - build + filters: + tags: + only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + - build: + filters: + tags: + only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ + - deploy: + requires: + - build + - test + - lint + - integration + filters: + branches: + only: master + tags: + only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ commands: install-docker: steps: - - run: - name: Install Docker client - command: | - set -x - VER="17.03.0-ce" - curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz - tar -xz -C /tmp -f /tmp/docker-$VER.tgz - mv /tmp/docker/* /usr/bin + - run: + name: Install Docker client + command: | + set -x + VER="17.03.0-ce" + curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz + tar -xz -C /tmp -f /tmp/docker-$VER.tgz + mv /tmp/docker/* /usr/bin jobs: lint: <<: *defaults steps: - - checkout - - run: - name: Lint - command: make BUILD_IN_CONTAINER=false lint - - run: - name: Check vendor directory is consistent. - command: make BUILD_IN_CONTAINER=false mod-check - - run: - name: Check protos are consistent. - command: make BUILD_IN_CONTAINER=false check-protos + - checkout + - run: + name: Lint + command: make BUILD_IN_CONTAINER=false lint + - run: + name: Check vendor directory is consistent. + command: make BUILD_IN_CONTAINER=false mod-check + - run: + name: Check protos are consistent. + command: make BUILD_IN_CONTAINER=false check-protos test: <<: *defaults steps: - - checkout - - run: - name: Test - command: make BUILD_IN_CONTAINER=false test + - checkout + - run: + name: Test + command: make BUILD_IN_CONTAINER=false test integration: machine: @@ -82,81 +82,83 @@ jobs: working_directory: ~/src/github.com/cortexproject/cortex steps: - - checkout - - restore_cache: - key: v1-cortex-{{ .Branch }}-{{ .Revision }} - - run: - name: Integration Test - command: | - touch build-image/.uptodate - MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations make BUILD_IMAGE=cortexproject/build-image:1370-pin-protoc-7ae297930 configs-integration-test - - run: - name: Load Images - command: ln -s /tmp/images ./images; make BUILD_IN_CONTAINER=false load-images - - run: - name: Ingester hand-over test - command: ./integration-tests/test-handover.sh - - run: - name: Ingester flush test - command: | - ./integration-tests/stop-components.sh - ./integration-tests/test-flush.sh + - checkout + - restore_cache: + key: v1-cortex-{{ .Branch }}-{{ .Revision }} + - run: + name: Integration Test + command: | + touch build-image/.uptodate + MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations make BUILD_IMAGE=cortexproject/build-image:1370-pin-protoc-7ae297930 configs-integration-test + - run: + name: Load Images + command: ln -s /tmp/images ./images; make BUILD_IN_CONTAINER=false load-images + - run: + name: Ingester hand-over test + command: ./integration-tests/test-handover.sh + - run: + name: Ingester flush test + command: | + ./integration-tests/stop-components.sh + ./integration-tests/test-flush.sh build: <<: *defaults steps: - - checkout - - setup_remote_docker - - install-docker - - - run: - name: Build - command: | - touch build-image/.uptodate - make BUILD_IN_CONTAINER=false - - - store_artifacts: - path: pkg/querier/frontend/frontend.pb.go - - store_artifacts: - path: pkg/chunk/storage/caching_index_client.pb.go - - store_artifacts: - path: pkg/ring/ring.pb.go - - store_artifacts: - path: pkg/ingester/client/cortex.pb.go - - - run: - name: Save Images - command: | - mkdir /tmp/images - ln -s /tmp/images ./images - make BUILD_IN_CONTAINER=false save-images - - - save_cache: - key: v1-cortex-{{ .Branch }}-{{ .Revision }} - paths: - - /tmp/images/ + - checkout + - setup_remote_docker + - install-docker + + - run: + name: Build + command: | + touch build-image/.uptodate + make BUILD_IN_CONTAINER=false + + - store_artifacts: + path: pkg/querier/frontend/frontend.pb.go + - store_artifacts: + path: pkg/chunk/storage/caching_index_client.pb.go + - store_artifacts: + path: pkg/ring/ring.pb.go + - store_artifacts: + path: pkg/ingester/client/cortex.pb.go + - store_artifacts: + path: pkg/ruler/rules/rules.pb.go + + - run: + name: Save Images + command: | + mkdir /tmp/images + ln -s /tmp/images ./images + make BUILD_IN_CONTAINER=false save-images + + - save_cache: + key: v1-cortex-{{ .Branch }}-{{ .Revision }} + paths: + - /tmp/images/ deploy: <<: *defaults steps: - - checkout - - setup_remote_docker - - install-docker - - - restore_cache: - key: v1-cortex-{{ .Branch }}-{{ .Revision }} - - - run: - name: Load Images - command: ln -s /tmp/images ./images; make BUILD_IN_CONTAINER=false load-images - - - run: - name: Deploy - command: | - if [ -n "$DOCKER_REGISTRY_PASSWORD" ]; then - docker login -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASSWORD" - fi - if [ -n "$QUAY_PASSWORD" ]; then - docker login -u "$QUAY_USER" -p "$QUAY_PASSWORD" quay.io; - fi - IMAGE_TAG=$CIRCLE_TAG ./push-images $NOQUAY + - checkout + - setup_remote_docker + - install-docker + + - restore_cache: + key: v1-cortex-{{ .Branch }}-{{ .Revision }} + + - run: + name: Load Images + command: ln -s /tmp/images ./images; make BUILD_IN_CONTAINER=false load-images + + - run: + name: Deploy + command: | + if [ -n "$DOCKER_REGISTRY_PASSWORD" ]; then + docker login -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASSWORD" + fi + if [ -n "$QUAY_PASSWORD" ]; then + docker login -u "$QUAY_USER" -p "$QUAY_PASSWORD" quay.io; + fi + IMAGE_TAG=$CIRCLE_TAG ./push-images $NOQUAY