Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions docs/operations/blocks-storage.md
Original file line number Diff line number Diff line change
Expand Up @@ -201,11 +201,16 @@ compactor:
# CLI flag: -compactor.block-ranges
[block_ranges: <list of duration> | default = [2h,12h,24h]]

# Number of Go routines to use when syncing block metadata from the long-term
# storage.
# Number of Go routines to use when syncing block index and chunks files
# from the long term storage.
# CLI flag: -compactor.block-sync-concurrency
[block_sync_concurrency: <int> | default = 20]

# Number of Go routines to use when syncing block meta files from the long
# term storage.
# CLI flag: -compactor.meta-sync-concurrency
[meta_sync_concurrency: <int> | default = 20]

# Minimum age of fresh (non-compacted) blocks before they are being processed,
# in order to skip blocks that are still uploading from ingesters. Malformed
# blocks older than the maximum of consistency-delay and 30m will be removed.
Expand Down
8 changes: 4 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ require (
github.com/NYTimes/gziphandler v1.1.1
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d
github.com/armon/go-metrics v0.3.0
github.com/aws/aws-sdk-go v1.25.35
github.com/aws/aws-sdk-go v1.25.48
github.com/blang/semver v3.5.0+incompatible
github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b
github.com/cenkalti/backoff v1.0.0 // indirect
github.com/cespare/xxhash v1.1.0
github.com/coreos/go-semver v0.3.0 // indirect
Expand Down Expand Up @@ -52,12 +52,12 @@ require (
github.com/prometheus/client_golang v1.2.1
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4
github.com/prometheus/common v0.7.0
github.com/prometheus/prometheus v1.8.2-0.20191126064551-80ba03c67da1
github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef
github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1
github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e
github.com/spf13/afero v1.2.2
github.com/stretchr/testify v1.4.0
github.com/thanos-io/thanos v0.8.1-0.20200102143048-a37ac093a67a
github.com/thanos-io/thanos v0.8.1-0.20200109203923-552ffa4c1a0d
github.com/tinylib/msgp v0.0.0-20161221055906-38a6f61a768d // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
github.com/uber-go/atomic v1.4.0
Expand Down
17 changes: 8 additions & 9 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -111,9 +111,8 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l
github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM=
github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.22.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.23.12/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.25.35 h1:fe2tJnqty/v/50pyngKdNk/NP8PFphYDA1Z7N3EiiiE=
github.com/aws/aws-sdk-go v1.25.35/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.25.48 h1:J82DYDGZHOKHdhx6hD24Tm30c2C3GchYGfN0mf9iKUk=
github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
Expand All @@ -127,8 +126,8 @@ github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17
github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668 h1:U/lr3Dgy4WK+hNk4tyD+nuGjpVLPEHuJSFXMw11/HPA=
github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0=
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
github.com/cenkalti/backoff v0.0.0-20181003080854-62661b46c409/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/cenkalti/backoff v1.0.0 h1:2XeuDgvPv/6QDyzIuxb6n36ADVocyqTLlOSpYBGYtvM=
github.com/cenkalti/backoff v1.0.0/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
Expand Down Expand Up @@ -676,8 +675,8 @@ github.com/prometheus/procfs v0.0.6 h1:0qbH+Yqu/cj1ViVLvEWCP6qMQ4efWUj6bQqOEA0V0
github.com/prometheus/procfs v0.0.6/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s=
github.com/prometheus/prometheus v0.0.0-20190818123050-43acd0e2e93f/go.mod h1:rMTlmxGCvukf2KMu3fClMDKLLoJ5hl61MhcJ7xKakf0=
github.com/prometheus/prometheus v1.8.2-0.20191126064551-80ba03c67da1 h1:5ee1ewJCJYB7Bp314qaPcRNFaAPsdHN6BFzBC1wMVbQ=
github.com/prometheus/prometheus v1.8.2-0.20191126064551-80ba03c67da1/go.mod h1:PVTKYlgELGIDbIKIyWRzD4WKjnavPynGOFLSuDpvOwU=
github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef h1:pYYKXo/zGx25kyViw+Gdbxd0ItIg+vkVKpwgWUEyIc4=
github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef/go.mod h1:7U90zPoLkWjEIQcy/rweQla82OCTUzxVHE51G3OhJbI=
github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 h1:+kGqA4dNN5hn7WwvKdzHl0rdN5AEkbNZd0VjRltAiZg=
github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
Expand Down Expand Up @@ -745,8 +744,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/thanos-io/thanos v0.8.1-0.20200102143048-a37ac093a67a h1:mpeMU82+ssaKeqLV65euG/hVH1yyJmvNNciX/BbKYwM=
github.com/thanos-io/thanos v0.8.1-0.20200102143048-a37ac093a67a/go.mod h1:vFgLQqNITtnP9Qnlc9UpEMi40QdFUIgUV8nsBetzuvE=
github.com/thanos-io/thanos v0.8.1-0.20200109203923-552ffa4c1a0d h1:kmHcmvQm1NUfBjJCf/j4ykZutUjHqUEdyfgMN8YmW38=
github.com/thanos-io/thanos v0.8.1-0.20200109203923-552ffa4c1a0d/go.mod h1:usT/TxtJQ7DzinTt+G9kinDQmRS5sxwu0unVKZ9vdcw=
github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
Expand Down
27 changes: 22 additions & 5 deletions pkg/compactor/compactor.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ import (
"github.com/go-kit/kit/log/level"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/prometheus/pkg/relabel"
"github.com/prometheus/prometheus/tsdb"
"github.com/thanos-io/thanos/pkg/block"
"github.com/thanos-io/thanos/pkg/compact"
"github.com/thanos-io/thanos/pkg/compact/downsample"
"github.com/thanos-io/thanos/pkg/objstore"
Expand All @@ -26,6 +26,7 @@ import (
type Config struct {
BlockRanges cortex_tsdb.DurationList `yaml:"block_ranges"`
BlockSyncConcurrency int `yaml:"block_sync_concurrency"`
MetaSyncConcurrency int `yaml:"meta_sync_concurrency"`
ConsistencyDelay time.Duration `yaml:"consistency_delay"`
DataDir string `yaml:"data_dir"`
CompactionInterval time.Duration `yaml:"compaction_interval"`
Expand All @@ -45,8 +46,9 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
cfg.retryMaxBackoff = time.Minute

f.Var(&cfg.BlockRanges, "compactor.block-ranges", "Comma separated list of compaction ranges expressed in the time duration format")
f.DurationVar(&cfg.ConsistencyDelay, "compactor.consistency-delay", 30*time.Minute, fmt.Sprintf("Minimum age of fresh (non-compacted) blocks before they are being processed. Malformed blocks older than the maximum of consistency-delay and %s will be removed.", compact.MinimumAgeForRemoval))
f.IntVar(&cfg.BlockSyncConcurrency, "compactor.block-sync-concurrency", 20, "Number of goroutines to use when syncing block metadata from object storage")
f.DurationVar(&cfg.ConsistencyDelay, "compactor.consistency-delay", 30*time.Minute, fmt.Sprintf("Minimum age of fresh (non-compacted) blocks before they are being processed. Malformed blocks older than the maximum of consistency-delay and %s will be removed.", compact.PartialUploadThresholdAge))
f.IntVar(&cfg.BlockSyncConcurrency, "compactor.block-sync-concurrency", 20, "Number of Go routines to use when syncing block index and chunks files from the long term storage.")
f.IntVar(&cfg.MetaSyncConcurrency, "compactor.meta-sync-concurrency", 20, "Number of Go routines to use when syncing block meta files from the long term storage.")
f.StringVar(&cfg.DataDir, "compactor.data-dir", "./data", "Data directory in which to cache blocks and process compactions")
f.DurationVar(&cfg.CompactionInterval, "compactor.compaction-interval", time.Hour, "The frequency at which the compaction runs")
f.IntVar(&cfg.CompactionRetries, "compactor.compaction-retries", 3, "How many times to retry a failed compaction during a single compaction interval")
Expand Down Expand Up @@ -231,15 +233,30 @@ func (c *Compactor) compactUser(ctx context.Context, userID string) error {
reg := prometheus.NewRegistry()
defer c.syncerMetrics.gatherThanosSyncerMetrics(reg)

fetcher, err := block.NewMetaFetcher(
c.logger,
c.compactorCfg.MetaSyncConcurrency,
bucket,
// The fetcher stores cached metas in the "meta-syncer/" sub directory,
// but we prefix it with "meta-" in order to guarantee no clashing with
// the directory used by the Thanos Syncer, whatever is the user ID.
path.Join(c.compactorCfg.DataDir, "meta-"+userID),
reg,
// No filters
)
if err != nil {
return err
}

syncer, err := compact.NewSyncer(
c.logger,
reg,
bucket,
c.compactorCfg.ConsistencyDelay,
fetcher,
c.compactorCfg.BlockSyncConcurrency,
false, // Do not accept malformed indexes
true, // Enable vertical compaction
[]*relabel.Config{})
)
if err != nil {
return errors.Wrap(err, "failed to create syncer")
}
Expand Down
63 changes: 38 additions & 25 deletions pkg/compactor/compactor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,19 +118,19 @@ func TestCompactor_ShouldDoNothingOnNoUserBlocks(t *testing.T) {
# TYPE cortex_compactor_garbage_collection_total counter
cortex_compactor_garbage_collection_total 0

# HELP cortex_compactor_sync_meta_duration_seconds TSDB Syncer: Time it took to sync meta files.
# TYPE cortex_compactor_sync_meta_duration_seconds histogram
cortex_compactor_sync_meta_duration_seconds_bucket{le="+Inf"} 0
cortex_compactor_sync_meta_duration_seconds_sum 0
cortex_compactor_sync_meta_duration_seconds_count 0
# HELP cortex_compactor_meta_sync_duration_seconds TSDB Syncer: Duration of the blocks metadata synchronization in seconds.
# TYPE cortex_compactor_meta_sync_duration_seconds histogram
cortex_compactor_meta_sync_duration_seconds_bucket{le="+Inf"} 0
cortex_compactor_meta_sync_duration_seconds_sum 0
cortex_compactor_meta_sync_duration_seconds_count 0

# HELP cortex_compactor_sync_meta_failures_total TSDB Syncer: Total number of failed sync meta operations.
# TYPE cortex_compactor_sync_meta_failures_total counter
cortex_compactor_sync_meta_failures_total 0
# HELP cortex_compactor_meta_sync_failures_total TSDB Syncer: Total blocks metadata synchronization failures.
# TYPE cortex_compactor_meta_sync_failures_total counter
cortex_compactor_meta_sync_failures_total 0

# HELP cortex_compactor_sync_meta_total TSDB Syncer: Total number of sync meta operations.
# TYPE cortex_compactor_sync_meta_total counter
cortex_compactor_sync_meta_total 0
# HELP cortex_compactor_meta_syncs_total TSDB Syncer: Total blocks metadata synchronization attempts.
# TYPE cortex_compactor_meta_syncs_total counter
cortex_compactor_meta_syncs_total 0

# HELP cortex_compactor_group_compaction_runs_completed_total TSDB Syncer: Total number of group completed compaction runs. This also includes compactor group runs that resulted with no compaction.
# TYPE cortex_compactor_group_compaction_runs_completed_total counter
Expand Down Expand Up @@ -213,19 +213,19 @@ func TestCompactor_ShouldRetryOnFailureWhileDiscoveringUsersFromBucket(t *testin
# TYPE cortex_compactor_garbage_collection_total counter
cortex_compactor_garbage_collection_total 0

# HELP cortex_compactor_sync_meta_duration_seconds TSDB Syncer: Time it took to sync meta files.
# TYPE cortex_compactor_sync_meta_duration_seconds histogram
cortex_compactor_sync_meta_duration_seconds_bucket{le="+Inf"} 0
cortex_compactor_sync_meta_duration_seconds_sum 0
cortex_compactor_sync_meta_duration_seconds_count 0
# HELP cortex_compactor_meta_sync_duration_seconds TSDB Syncer: Duration of the blocks metadata synchronization in seconds.
# TYPE cortex_compactor_meta_sync_duration_seconds histogram
cortex_compactor_meta_sync_duration_seconds_bucket{le="+Inf"} 0
cortex_compactor_meta_sync_duration_seconds_sum 0
cortex_compactor_meta_sync_duration_seconds_count 0

# HELP cortex_compactor_sync_meta_failures_total TSDB Syncer: Total number of failed sync meta operations.
# TYPE cortex_compactor_sync_meta_failures_total counter
cortex_compactor_sync_meta_failures_total 0
# HELP cortex_compactor_meta_sync_failures_total TSDB Syncer: Total blocks metadata synchronization failures.
# TYPE cortex_compactor_meta_sync_failures_total counter
cortex_compactor_meta_sync_failures_total 0

# HELP cortex_compactor_sync_meta_total TSDB Syncer: Total number of sync meta operations.
# TYPE cortex_compactor_sync_meta_total counter
cortex_compactor_sync_meta_total 0
# HELP cortex_compactor_meta_syncs_total TSDB Syncer: Total blocks metadata synchronization attempts.
# TYPE cortex_compactor_meta_syncs_total counter
cortex_compactor_meta_syncs_total 0

# HELP cortex_compactor_group_compaction_runs_completed_total TSDB Syncer: Total number of group completed compaction runs. This also includes compactor group runs that resulted with no compaction.
# TYPE cortex_compactor_group_compaction_runs_completed_total counter
Expand Down Expand Up @@ -283,17 +283,17 @@ func TestCompactor_ShouldIterateOverUsersAndRunCompaction(t *testing.T) {
`level=info msg="discovered users from bucket" users=2`,
`level=info msg="starting compaction of user blocks" user=user-1`,
`level=info msg="start sync of metas"`,
`level=debug msg="download meta" block=01DTVP434PA9VFXSW2JKB3392D`,
`level=info msg="start of GC"`,
`level=info msg="start of compaction"`,
`level=info msg="compaction iterations done"`,
`level=info msg="successfully compacted user blocks" user=user-1`,
`level=info msg="starting compaction of user blocks" user=user-2`,
`level=info msg="start sync of metas"`,
`level=debug msg="download meta" block=01DTW0ZCPDDNV4BV83Q2SV4QAZ`,
`level=info msg="start of GC"`,
`level=info msg="start of compaction"`,
`level=info msg="compaction iterations done"`,
`level=info msg="successfully compacted user blocks" user=user-2`,
}, strings.Split(strings.TrimSpace(logs.String()), "\n"))
}, removeMetaFetcherLogs(strings.Split(strings.TrimSpace(logs.String()), "\n")))

// Instead of testing for shipper metrics, we only check our metrics here.
// Real shipper metrics are too variable to embed into a test.
Expand All @@ -313,6 +313,18 @@ func TestCompactor_ShouldIterateOverUsersAndRunCompaction(t *testing.T) {
`), testedMetrics...))
}

func removeMetaFetcherLogs(input []string) []string {
out := make([]string, 0, len(input))

for i := 0; i < len(input); i++ {
if !strings.Contains(input[i], "block.MetaFetcher") {
out = append(out, input[i])
}
}

return out
}

func prepare(t *testing.T, bucketClient *cortex_tsdb.BucketClientMock) (*Compactor, *tsdbCompactorMock, *bytes.Buffer, prometheus.Gatherer) {
compactorCfg := Config{}
storageCfg := cortex_tsdb.Config{}
Expand Down Expand Up @@ -353,6 +365,7 @@ func (m *tsdbCompactorMock) Compact(dest string, dirs []string, open []*tsdb.Blo

func mockBlockMetaJSON(id string) string {
meta := tsdb.BlockMeta{
Version: 1,
ULID: ulid.MustParse(id),
MinTime: 1574776800000,
MaxTime: 1574784000000,
Expand Down
37 changes: 19 additions & 18 deletions pkg/compactor/syncer_metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ import (
// Copied from Thanos, pkg/compact/compact.go.
// Here we aggregate metrics from all finished syncers.
type syncerMetrics struct {
syncMetas prometheus.Counter
syncMetaFailures prometheus.Counter
syncMetaDuration *util.HistogramDataCollector // was prometheus.Histogram before
metaSync prometheus.Counter
metaSyncFailures prometheus.Counter
metaSyncDuration *util.HistogramDataCollector // was prometheus.Histogram before
garbageCollectedBlocks prometheus.Counter
garbageCollections prometheus.Counter
garbageCollectionFailures prometheus.Counter
Expand All @@ -28,17 +28,17 @@ type syncerMetrics struct {
func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics {
var m syncerMetrics

m.syncMetas = prometheus.NewCounter(prometheus.CounterOpts{
Name: "cortex_compactor_sync_meta_total",
Help: "TSDB Syncer: Total number of sync meta operations.",
m.metaSync = prometheus.NewCounter(prometheus.CounterOpts{
Name: "cortex_compactor_meta_syncs_total",
Help: "TSDB Syncer: Total blocks metadata synchronization attempts.",
})
m.syncMetaFailures = prometheus.NewCounter(prometheus.CounterOpts{
Name: "cortex_compactor_sync_meta_failures_total",
Help: "TSDB Syncer: Total number of failed sync meta operations.",
m.metaSyncFailures = prometheus.NewCounter(prometheus.CounterOpts{
Name: "cortex_compactor_meta_sync_failures_total",
Help: "TSDB Syncer: Total blocks metadata synchronization failures.",
})
m.syncMetaDuration = util.NewHistogramDataCollector(prometheus.NewDesc(
"cortex_compactor_sync_meta_duration_seconds",
"TSDB Syncer: Time it took to sync meta files.",
m.metaSyncDuration = util.NewHistogramDataCollector(prometheus.NewDesc(
"cortex_compactor_meta_sync_duration_seconds",
"TSDB Syncer: Duration of the blocks metadata synchronization in seconds.",
nil, nil))

m.garbageCollectedBlocks = prometheus.NewCounter(prometheus.CounterOpts{
Expand Down Expand Up @@ -81,9 +81,9 @@ func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics {

if reg != nil {
reg.MustRegister(
m.syncMetas,
m.syncMetaFailures,
m.syncMetaDuration,
m.metaSync,
m.metaSyncFailures,
m.metaSyncDuration,
m.garbageCollectedBlocks,
m.garbageCollections,
m.garbageCollectionFailures,
Expand Down Expand Up @@ -115,9 +115,10 @@ func (m *syncerMetrics) gatherThanosSyncerMetrics(reg *prometheus.Registry) {
return
}

m.syncMetas.Add(mfm.SumCounters("thanos_compact_sync_meta_total"))
m.syncMetaFailures.Add(mfm.SumCounters("thanos_compact_sync_meta_failures_total"))
m.syncMetaDuration.Add(mfm.SumHistograms("thanos_compact_sync_meta_duration_seconds"))
m.metaSync.Add(mfm.SumCounters("blocks_meta_syncs_total"))
m.metaSyncFailures.Add(mfm.SumCounters("blocks_meta_sync_failures_total"))
m.metaSyncDuration.Add(mfm.SumHistograms("blocks_meta_sync_duration_seconds"))

m.garbageCollectedBlocks.Add(mfm.SumCounters("thanos_compact_garbage_collected_blocks_total"))
m.garbageCollections.Add(mfm.SumCounters("thanos_compact_garbage_collection_total"))
m.garbageCollectionFailures.Add(mfm.SumCounters("thanos_compact_garbage_collection_failures_total"))
Expand Down
Loading