From 23d51d14b6ecd24ec7c772f1ea09bb7ba9e3ba91 Mon Sep 17 00:00:00 2001 From: glorv Date: Fri, 10 Jul 2020 10:44:14 +0800 Subject: [PATCH 1/9] add compression level --- go.mod | 2 +- go.sum | 4 ++-- pkg/task/backup.go | 19 ++++++++++++++----- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 2400cd0d8..f7a9566f9 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 github.com/pingcap/failpoint v0.0.0-20200603062251-b230c36c413c - github.com/pingcap/kvproto v0.0.0-20200706115936-1e0910aabe6c + github.com/pingcap/kvproto v0.0.0-20200730045059-3177b43b983c github.com/pingcap/log v0.0.0-20200511115504-543df19646ad github.com/pingcap/parser v0.0.0-20200623082809-b74301ac298b github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200714122454-1a64f969cb3c diff --git a/go.sum b/go.sum index 3ba4003a2..1bb9ad5a2 100644 --- a/go.sum +++ b/go.sum @@ -503,8 +503,8 @@ github.com/pingcap/kvproto v0.0.0-20200518112156-d4aeb467de29 h1:NpW1OuYrIl+IQrS github.com/pingcap/kvproto v0.0.0-20200518112156-d4aeb467de29/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200608081027-d02a6f65e956/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200701055533-4ef28cac01f8/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20200706115936-1e0910aabe6c h1:VnLpCAxMAeDxc7HXTetwDQB+/MtDQjHAOBsd4QnGVwA= -github.com/pingcap/kvproto v0.0.0-20200706115936-1e0910aabe6c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20200730045059-3177b43b983c h1:k0frWfFFVEobqVJDPNj0YHm8cVAZ8wxWCrRvW8MNTgQ= +github.com/pingcap/kvproto v0.0.0-20200730045059-3177b43b983c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM= diff --git a/pkg/task/backup.go b/pkg/task/backup.go index b33a0f5d3..d484c0da5 100644 --- a/pkg/task/backup.go +++ b/pkg/task/backup.go @@ -32,6 +32,7 @@ const ( flagBackupTS = "backupts" flagLastBackupTS = "lastbackupts" flagCompressionType = "compression" + flagCompressionLevel = "compression-level" flagRemoveSchedulers = "remove-schedulers" flagGCTTL = "gcttl" @@ -49,6 +50,7 @@ type BackupConfig struct { LastBackupTS uint64 `json:"last-backup-ts" toml:"last-backup-ts"` GCTTL int64 `json:"gc-ttl" toml:"gc-ttl"` CompressionType kvproto.CompressionType `json:"compression-type" toml:"compression-type"` + CompressionLevel int32 `json:"compression-level" toml:"compression-level"` RemoveSchedulers bool `json:"remove-schedulers" toml:"remove-schedulers"` } @@ -66,6 +68,7 @@ func DefineBackupFlags(flags *pflag.FlagSet) { flags.Int64(flagGCTTL, backup.DefaultBRGCSafePointTTL, "the TTL (in seconds) that PD holds for BR's GC safepoint") flags.String(flagCompressionType, "zstd", "backup sst file compression algorithm, value can be one of 'lz4|zstd|snappy'") + flags.Int32(flagCompressionLevel, 0, "compression level used for sst file compression") flags.Bool(flagRemoveSchedulers, false, "disable the balance, shuffle and region-merge schedulers in PD to speed up backup") @@ -110,6 +113,11 @@ func (cfg *BackupConfig) ParseFromFlags(flags *pflag.FlagSet) error { return errors.Trace(err) } cfg.CompressionType = compressionType + level, err := flags.GetInt32(flagCompressionLevel) + if err != nil { + return errors.Trace(err) + } + cfg.CompressionLevel = level if err = cfg.Config.ParseFromFlags(flags); err != nil { return errors.Trace(err) @@ -206,11 +214,12 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig } req := kvproto.BackupRequest{ - StartVersion: cfg.LastBackupTS, - EndVersion: backupTS, - RateLimit: cfg.RateLimit, - Concurrency: defaultBackupConcurrency, - CompressionType: cfg.CompressionType, + StartVersion: cfg.LastBackupTS, + EndVersion: backupTS, + RateLimit: cfg.RateLimit, + Concurrency: defaultBackupConcurrency, + CompressionType: cfg.CompressionType, + CompressionLevel: cfg.CompressionLevel, } ranges, backupSchemas, err := backup.BuildBackupRangeAndSchema( From ac19bb855939db847b78c3f307d6d35554bf7c09 Mon Sep 17 00:00:00 2001 From: glorv Date: Thu, 6 Aug 2020 21:17:52 +0800 Subject: [PATCH 2/9] update deps --- go.mod | 11 +++++------ go.sum | 31 +++++++++++++++++++++++++++++++ pkg/restore/split_client.go | 6 +++--- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index f7a9566f9..69831a1e1 100644 --- a/go.mod +++ b/go.mod @@ -12,17 +12,16 @@ require ( github.com/gogo/protobuf v1.3.1 github.com/google/btree v1.0.0 github.com/google/uuid v1.1.1 - github.com/juju/testing v0.0.0-20200608005635-e4eedbc6f7aa // indirect github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 - github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 + github.com/pingcap/errors v0.11.5-0.20200729012136-4e113ddee29e github.com/pingcap/failpoint v0.0.0-20200603062251-b230c36c413c github.com/pingcap/kvproto v0.0.0-20200730045059-3177b43b983c github.com/pingcap/log v0.0.0-20200511115504-543df19646ad - github.com/pingcap/parser v0.0.0-20200623082809-b74301ac298b - github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200714122454-1a64f969cb3c - github.com/pingcap/tidb v1.1.0-beta.0.20200721005019-f5c6e59f0daf + github.com/pingcap/parser v0.0.0-20200731033026-84f62115187c + github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200730093003-dc8c75cf7ca0 + github.com/pingcap/tidb v1.1.0-beta.0.20200806130549-32963d338791 github.com/pingcap/tidb-tools v4.0.1+incompatible - github.com/pingcap/tipb v0.0.0-20200615034523-dcfcea0b5965 + github.com/pingcap/tipb v0.0.0-20200618092958-4fad48b4c8c3 github.com/prometheus/client_golang v1.5.1 github.com/prometheus/common v0.9.1 github.com/sirupsen/logrus v1.6.0 diff --git a/go.sum b/go.sum index 1bb9ad5a2..8b9d53613 100644 --- a/go.sum +++ b/go.sum @@ -426,6 +426,8 @@ github.com/ngaut/unistore v0.0.0-20200604061006-d8e9dc0ad154 h1:yuiHbYQTMgvroGV9 github.com/ngaut/unistore v0.0.0-20200604061006-d8e9dc0ad154/go.mod h1:YGQzxn9cVy0q2puXVt1X8l5OohRHv2djR/ziz1k14XQ= github.com/ngaut/unistore v0.0.0-20200630072006-0c4035925f69 h1:tn65bmj+EBa8gXYKQHPRlGcDdo+tDEjUK88gO4QK11s= github.com/ngaut/unistore v0.0.0-20200630072006-0c4035925f69/go.mod h1:Hxlp5VAoPOHwcXLUw/E+P3XjJX1EP38NWjXPpc4nuOE= +github.com/ngaut/unistore v0.0.0-20200803051709-607d96233b1d h1:5wZOROIRh9/E4kyX6pXT7OtYyQtS9iUNcQtyt0Agil4= +github.com/ngaut/unistore v0.0.0-20200803051709-607d96233b1d/go.mod h1:2QAH8tXCjeHuCSLEWKLYAzHPz2dB59VnhpPA2IDVeW4= github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -452,6 +454,8 @@ github.com/pelletier/go-toml v1.3.0 h1:e5+lF2E4Y2WCIxBefVowBuB0iHrUH4HZ8q+6mGF7f github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= +github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= +github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= @@ -459,12 +463,18 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200407064406-b2b8ad403d01/go.mod h1:77fCh8d3oKzC5ceOJWeZXAS/mLzVgdZ7rKniwmOyFuo= github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200514075710-eecc9a4525b5/go.mod h1:8q+yDx0STBPri8xS4A2duS1dAf+xO0cMtjwe0t6MWJk= github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200710045508-523e95bc5ec9/go.mod h1:9yaAM77sPfa5/f6sdxr3jSkKfIz463KRHyiFHiGjdes= +github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200715070228-47f5de8a6992/go.mod h1:9yaAM77sPfa5/f6sdxr3jSkKfIz463KRHyiFHiGjdes= github.com/pingcap/badger v1.5.1-0.20200604041313-19c397305fcc h1:aL83XYtYRGJHGruiw5Kk/vJiJl2xFiWfqCgnvHin7ek= github.com/pingcap/badger v1.5.1-0.20200604041313-19c397305fcc/go.mod h1:LyrqUOHZrUDf9oGi1yoz1+qw9ckSIhQb5eMa1acOLNQ= +github.com/pingcap/badger v1.5.1-0.20200714132513-80ba2000f159 h1:cmZSuRbdfOJd3kJjRIClrLbt3nD0xi4oqYR1c/ZrPKg= +github.com/pingcap/badger v1.5.1-0.20200714132513-80ba2000f159/go.mod h1:LyrqUOHZrUDf9oGi1yoz1+qw9ckSIhQb5eMa1acOLNQ= github.com/pingcap/br v0.0.0-20200426093517-dd11ae28b885/go.mod h1:4w3meMnk7HDNpNgjuRAxavruTeKJvUiXxoEWTjzXPnA= github.com/pingcap/br v0.0.0-20200521085655-53201addd4ad/go.mod h1:SlSUHWY7QUoooiYxOKuJ8kUh2KjI29ogBh89YXz2dLA= github.com/pingcap/br v0.0.0-20200610051721-b057d65ff579/go.mod h1:Gq6o66nDReG0fMxqSdl8JHLZhMnxKhVxtHxqubUWAyg= github.com/pingcap/br v0.0.0-20200617120402-56e151ad8b67/go.mod h1:/3QzpDG7YTPrDsrg8i1lwdYUrplJ0jVD+9pxhh19+k4= +github.com/pingcap/br v0.0.0-20200727092753-a475692725db/go.mod h1:4iTqZAMbEPmjBggYixqIg2FwIHBQtyImTM/QYlpTBGk= +github.com/pingcap/br v0.0.0-20200803052654-e6f63fc1807a/go.mod h1:8j7vGUfHCETYbeBfASLTDywC3NFSx90z9nuk0PV9rpo= +github.com/pingcap/br v0.0.0-20200805121136-181c081ba6ac/go.mod h1:9P24mNzNmXjggYBm4pnb08slSbua8FA6QIyg68GpuhQ= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4 h1:iRtOAQ6FXkY/BGvst3CDfTva4nTqh6CL8WXvanLdbu0= @@ -479,6 +489,8 @@ github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 h1:58naV4XMEqm0hl9LcYo6cZoGBGiLtefMQMF/vo3XLgQ= github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.5-0.20200729012136-4e113ddee29e h1:/EGWHNOyEgizEBuAujWsb9vXrPZtt1b7ooDPyjEkjDw= +github.com/pingcap/errors v0.11.5-0.20200729012136-4e113ddee29e/go.mod h1:g4vx//d6VakjJ0mk7iLBlKA8LFavV/sAVINT/1PFxeQ= github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d h1:F8vp38kTAckN+v8Jlc98uMBvKIzr1a+UhnLyVYn8Q5Q= github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= github.com/pingcap/failpoint v0.0.0-20200210140405-f8f9fb234798/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= @@ -503,6 +515,8 @@ github.com/pingcap/kvproto v0.0.0-20200518112156-d4aeb467de29 h1:NpW1OuYrIl+IQrS github.com/pingcap/kvproto v0.0.0-20200518112156-d4aeb467de29/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200608081027-d02a6f65e956/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200701055533-4ef28cac01f8/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20200706115936-1e0910aabe6c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20200715040832-c3e2e0b163ee/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200730045059-3177b43b983c h1:k0frWfFFVEobqVJDPNj0YHm8cVAZ8wxWCrRvW8MNTgQ= github.com/pingcap/kvproto v0.0.0-20200730045059-3177b43b983c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA= @@ -520,11 +534,17 @@ github.com/pingcap/parser v0.0.0-20200609110328-c65941b9fbb3 h1:nPE21utCCSq0c0Qj github.com/pingcap/parser v0.0.0-20200609110328-c65941b9fbb3/go.mod h1:vQdbJqobJAgFyiRNNtXahpMoGWwPEuWciVEK5A20NS0= github.com/pingcap/parser v0.0.0-20200623082809-b74301ac298b h1:mrIaC+MfZi4jfsaD8ErzpLNAGRgDICPMrsmoHj2zi6Q= github.com/pingcap/parser v0.0.0-20200623082809-b74301ac298b/go.mod h1:vQdbJqobJAgFyiRNNtXahpMoGWwPEuWciVEK5A20NS0= +github.com/pingcap/parser v0.0.0-20200730092557-34a468e9b774/go.mod h1:vQdbJqobJAgFyiRNNtXahpMoGWwPEuWciVEK5A20NS0= +github.com/pingcap/parser v0.0.0-20200731033026-84f62115187c h1:UG7JwE9UbEpiTOJeUXHQBJFfMrU+yAtdpzEbLRiAnkI= +github.com/pingcap/parser v0.0.0-20200731033026-84f62115187c/go.mod h1:vQdbJqobJAgFyiRNNtXahpMoGWwPEuWciVEK5A20NS0= github.com/pingcap/pd/v4 v4.0.0-rc.1.0.20200422143320-428acd53eba2/go.mod h1:s+utZtXDznOiL24VK0qGmtoHjjXNsscJx3m1n8cC56s= github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200520083007-2c251bd8f181 h1:FM+PzdoR3fmWAJx3ug+p5aOgs5aZYwFkoDL7Potdsz0= github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200520083007-2c251bd8f181/go.mod h1:q4HTx/bA8aKBa4S7L+SQKHvjRPXCRV0tA0yRw0qkZSA= github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200714122454-1a64f969cb3c h1:aOiyGetA256/LUkdmhny0Q/PWTBQiF/TPNhJuJMGRSY= github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200714122454-1a64f969cb3c/go.mod h1:v/dY4mVkt3dh/Liphhk0E4ScOkZpIk0m0GvWJ9FapDs= +github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200715061836-4971825321cf/go.mod h1:Ikvk5tw09nV6ZR4vHgrmAIAbZJttZ2cKQvOjN5lekBk= +github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200730093003-dc8c75cf7ca0 h1:cSHKKU5Tt4oI+Bky4KCnLr8prM/4HS0S0LWtVOm9bh4= +github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200730093003-dc8c75cf7ca0/go.mod h1:szYFB2rf8yrSGJuI8hm9RLWvsK+xt1exLTj511WPCnE= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/pingcap/sysutil v0.0.0-20200408114249-ed3bd6f7fdb1 h1:PI8YpTl45F8ilNkrPtT4IdbcZB1SCEa+gK/U5GJYl3E= github.com/pingcap/sysutil v0.0.0-20200408114249-ed3bd6f7fdb1/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= @@ -540,6 +560,11 @@ github.com/pingcap/tidb v1.1.0-beta.0.20200610060912-f12cdc42010f h1:FS/jkQj5YJF github.com/pingcap/tidb v1.1.0-beta.0.20200610060912-f12cdc42010f/go.mod h1:jyXOvS9k0PTxYHju2OgySOe9FtydA52TiQ5bXAaKyQE= github.com/pingcap/tidb v1.1.0-beta.0.20200721005019-f5c6e59f0daf h1:reSW+RjEvLFVilpDYJaKa7vr7FvHVYwwf8BJj3yXbJ8= github.com/pingcap/tidb v1.1.0-beta.0.20200721005019-f5c6e59f0daf/go.mod h1:dYCOFXJsoqBumpxAyBqCG3WZriIY7JgeBZHgvfARDa8= +github.com/pingcap/tidb v1.1.0-beta.0.20200803035726-41c23700d8d1/go.mod h1:YFuuPMuceYoXIr4sCrtv1FUyJLvtYp9KisDsTk5dxlE= +github.com/pingcap/tidb v1.1.0-beta.0.20200803051932-e291f8fbd1e0/go.mod h1:YFuuPMuceYoXIr4sCrtv1FUyJLvtYp9KisDsTk5dxlE= +github.com/pingcap/tidb v1.1.0-beta.0.20200805053026-cd3e5ed82671/go.mod h1:+r9tlyUKG2zYzs2ajvEHiQlTx6WM0K2L1yabCHZwgGw= +github.com/pingcap/tidb v1.1.0-beta.0.20200806130549-32963d338791 h1:pMyU6dB+5G4ktdLI43bvRpzW3mi4dVei72cQHqgyCSQ= +github.com/pingcap/tidb v1.1.0-beta.0.20200806130549-32963d338791/go.mod h1:NHcZH46dkYwDd2IWUJaLOB0m54j7v2P5WdS4FvPR81w= github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200421113014-507d2bb3a15e+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200514040632-f76b3e428e19+incompatible h1:/JKsYjsa5Ug8v5CN4zIbJGIqsvgBUkGwaP/rEScVvWM= @@ -554,6 +579,8 @@ github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee/go.mod h1:RtkHW8WbcNx github.com/pingcap/tipb v0.0.0-20200604070248-508f03b0b342/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pingcap/tipb v0.0.0-20200615034523-dcfcea0b5965 h1:a0kZ+iaj/sbzJa5mt5310t1XJSpY+wmmIauAkrr7gU4= github.com/pingcap/tipb v0.0.0-20200615034523-dcfcea0b5965/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tipb v0.0.0-20200618092958-4fad48b4c8c3 h1:ESL3eIt1kUt8IMvR1011ejZlAyDcOzw89ARvVHvpD5k= +github.com/pingcap/tipb v0.0.0-20200618092958-4fad48b4c8c3/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -886,6 +913,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= @@ -1035,6 +1064,8 @@ honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.5 h1:nI5egYTGJakVyOryqLs1cQO5dO0ksin5XXs2pspk75k= +honnef.co/go/tools v0.0.1-2020.1.5/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= diff --git a/pkg/restore/split_client.go b/pkg/restore/split_client.go index 4dca78209..69e38c0d1 100755 --- a/pkg/restore/split_client.go +++ b/pkg/restore/split_client.go @@ -355,15 +355,15 @@ func (c *pdClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetO } func (c *pdClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*RegionInfo, error) { - regions, leaders, err := c.client.ScanRegions(ctx, key, endKey, limit) + regions, err := c.client.ScanRegions(ctx, key, endKey, limit) if err != nil { return nil, err } regionInfos := make([]*RegionInfo, 0, len(regions)) for i := range regions { regionInfos = append(regionInfos, &RegionInfo{ - Region: regions[i], - Leader: leaders[i], + Region: regions[i].Meta, + Leader: regions[i].Leader, }) } return regionInfos, nil From 621619195a80dfc4118a7868867f8dfd9325ce0a Mon Sep 17 00:00:00 2001 From: glorv Date: Fri, 7 Aug 2020 10:18:34 +0800 Subject: [PATCH 3/9] add raw backup --- pkg/backup/client.go | 25 ++++++++++++--------- pkg/task/backup.go | 51 +++++++++++++++++++++++++++--------------- pkg/task/backup_raw.go | 36 ++++++++++++++++------------- 3 files changed, 67 insertions(+), 45 deletions(-) diff --git a/pkg/backup/client.go b/pkg/backup/client.go index 718451e22..2dc3a2f7d 100644 --- a/pkg/backup/client.go +++ b/pkg/backup/client.go @@ -507,7 +507,7 @@ func (bc *Client) BackupRange( // Find and backup remaining ranges. // TODO: test fine grained backup. err = bc.fineGrainedBackup( - ctx, startKey, endKey, req.StartVersion, req.EndVersion, req.CompressionType, + ctx, startKey, endKey, req.StartVersion, req.EndVersion, req.CompressionType, req.CompressionLevel, req.RateLimit, req.Concurrency, results, updateCh) if err != nil { return nil, err @@ -569,6 +569,7 @@ func (bc *Client) fineGrainedBackup( lastBackupTS uint64, backupTS uint64, compressType kvproto.CompressionType, + compressLevel int32, rateLimit uint64, concurrency uint32, rangeTree rtree.RangeTree, @@ -599,7 +600,7 @@ func (bc *Client) fineGrainedBackup( defer wg.Done() for rg := range retry { backoffMs, err := - bc.handleFineGrained(ctx, boFork, rg, lastBackupTS, backupTS, compressType, rateLimit, concurrency, respCh) + bc.handleFineGrained(ctx, boFork, rg, lastBackupTS, backupTS, compressType, compressLevel, rateLimit, concurrency, respCh) if err != nil { errCh <- err return @@ -734,6 +735,7 @@ func (bc *Client) handleFineGrained( lastBackupTS uint64, backupTS uint64, compressType kvproto.CompressionType, + compressionLevel int32, rateLimit uint64, concurrency uint32, respCh chan<- *kvproto.BackupResponse, @@ -746,15 +748,16 @@ func (bc *Client) handleFineGrained( max := 0 req := kvproto.BackupRequest{ - ClusterId: bc.clusterID, - StartKey: rg.StartKey, // TODO: the range may cross region. - EndKey: rg.EndKey, - StartVersion: lastBackupTS, - EndVersion: backupTS, - StorageBackend: bc.backend, - RateLimit: rateLimit, - Concurrency: concurrency, - CompressionType: compressType, + ClusterId: bc.clusterID, + StartKey: rg.StartKey, // TODO: the range may cross region. + EndKey: rg.EndKey, + StartVersion: lastBackupTS, + EndVersion: backupTS, + StorageBackend: bc.backend, + RateLimit: rateLimit, + Concurrency: concurrency, + CompressionType: compressType, + CompressionLevel: compressionLevel, } lockResolver := bc.mgr.GetLockResolver() client, err := bc.mgr.GetBackupClient(ctx, storeID) diff --git a/pkg/task/backup.go b/pkg/task/backup.go index d484c0da5..6e182bfae 100644 --- a/pkg/task/backup.go +++ b/pkg/task/backup.go @@ -41,17 +41,21 @@ const ( maxBackupConcurrency = 256 ) +type CompressionConfig struct { + CompressionType kvproto.CompressionType `json:"compression-type" toml:"compression-type"` + CompressionLevel int32 `json:"compression-level" toml:"compression-level"` +} + // BackupConfig is the configuration specific for backup tasks. type BackupConfig struct { Config - TimeAgo time.Duration `json:"time-ago" toml:"time-ago"` - BackupTS uint64 `json:"backup-ts" toml:"backup-ts"` - LastBackupTS uint64 `json:"last-backup-ts" toml:"last-backup-ts"` - GCTTL int64 `json:"gc-ttl" toml:"gc-ttl"` - CompressionType kvproto.CompressionType `json:"compression-type" toml:"compression-type"` - CompressionLevel int32 `json:"compression-level" toml:"compression-level"` - RemoveSchedulers bool `json:"remove-schedulers" toml:"remove-schedulers"` + TimeAgo time.Duration `json:"time-ago" toml:"time-ago"` + BackupTS uint64 `json:"backup-ts" toml:"backup-ts"` + LastBackupTS uint64 `json:"last-backup-ts" toml:"last-backup-ts"` + GCTTL int64 `json:"gc-ttl" toml:"gc-ttl"` + RemoveSchedulers bool `json:"remove-schedulers" toml:"remove-schedulers"` + CompressionConfig } // DefineBackupFlags defines common flags for the backup command. @@ -104,20 +108,11 @@ func (cfg *BackupConfig) ParseFromFlags(flags *pflag.FlagSet) error { } cfg.GCTTL = gcTTL - compressionStr, err := flags.GetString(flagCompressionType) + compressionCfg, err := parseCompressionFlags(flags) if err != nil { return errors.Trace(err) } - compressionType, err := parseCompressionType(compressionStr) - if err != nil { - return errors.Trace(err) - } - cfg.CompressionType = compressionType - level, err := flags.GetInt32(flagCompressionLevel) - if err != nil { - return errors.Trace(err) - } - cfg.CompressionLevel = level + cfg.CompressionConfig = *compressionCfg if err = cfg.Config.ParseFromFlags(flags); err != nil { return errors.Trace(err) @@ -127,6 +122,26 @@ func (cfg *BackupConfig) ParseFromFlags(flags *pflag.FlagSet) error { return err } +// ParseFromFlags parses the backup-related flags from the flag set. +func parseCompressionFlags(flags *pflag.FlagSet) (*CompressionConfig, error) { + compressionStr, err := flags.GetString(flagCompressionType) + if err != nil { + return nil, errors.Trace(err) + } + compressionType, err := parseCompressionType(compressionStr) + if err != nil { + return nil, errors.Trace(err) + } + level, err := flags.GetInt32(flagCompressionLevel) + if err != nil { + return nil, errors.Trace(err) + } + return &CompressionConfig{ + CompressionLevel: level, + CompressionType: compressionType, + }, nil +} + // adjustBackupConfig is use for BR(binary) and BR in TiDB. // When new config was add and not included in parser. // we should set proper value in this function. diff --git a/pkg/task/backup_raw.go b/pkg/task/backup_raw.go index 833477f90..20f8f9ed2 100644 --- a/pkg/task/backup_raw.go +++ b/pkg/task/backup_raw.go @@ -33,11 +33,11 @@ const ( type RawKvConfig struct { Config - StartKey []byte `json:"start-key" toml:"start-key"` - EndKey []byte `json:"end-key" toml:"end-key"` - CF string `json:"cf" toml:"cf"` - CompressionType kvproto.CompressionType `json:"compression-type" toml:"compression-type"` - RemoveSchedulers bool `json:"remove-schedulers" toml:"remove-schedulers"` + StartKey []byte `json:"start-key" toml:"start-key"` + EndKey []byte `json:"end-key" toml:"end-key"` + CF string `json:"cf" toml:"cf"` + CompressionConfig + RemoveSchedulers bool `json:"remove-schedulers" toml:"remove-schedulers"` } // DefineRawBackupFlags defines common flags for the backup command. @@ -97,19 +97,22 @@ func (cfg *RawKvConfig) ParseBackupConfigFromFlags(flags *pflag.FlagSet) error { return err } - compressionStr, err := flags.GetString(flagCompressionType) + compressionCfg, err := parseCompressionFlags(flags) if err != nil { return errors.Trace(err) } - compressionType, err := parseCompressionType(compressionStr) + cfg.CompressionConfig = *compressionCfg + + cfg.RemoveSchedulers, err = flags.GetBool(flagRemoveSchedulers) if err != nil { return errors.Trace(err) } - cfg.CompressionType = compressionType - cfg.RemoveSchedulers, err = flags.GetBool(flagRemoveSchedulers) + level, err := flags.GetInt32(flagCompressionLevel) if err != nil { return errors.Trace(err) } + cfg.CompressionLevel = level + return nil } @@ -165,13 +168,14 @@ func RunBackupRaw(c context.Context, g glue.Glue, cmdName string, cfg *RawKvConf ctx, cmdName, int64(approximateRegions), !cfg.LogProgress) req := kvproto.BackupRequest{ - StartVersion: 0, - EndVersion: 0, - RateLimit: cfg.RateLimit, - Concurrency: cfg.Concurrency, - IsRawKv: true, - Cf: cfg.CF, - CompressionType: cfg.CompressionType, + StartVersion: 0, + EndVersion: 0, + RateLimit: cfg.RateLimit, + Concurrency: cfg.Concurrency, + IsRawKv: true, + Cf: cfg.CF, + CompressionType: cfg.CompressionType, + CompressionLevel: cfg.CompressionLevel, } files, err := client.BackupRange(ctx, backupRange.StartKey, backupRange.EndKey, req, updateCh) if err != nil { From 27851c7ebe69cd5029ab3d35776c500995cb7b57 Mon Sep 17 00:00:00 2001 From: glorv Date: Fri, 7 Aug 2020 10:21:15 +0800 Subject: [PATCH 4/9] add comments --- pkg/task/backup.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/task/backup.go b/pkg/task/backup.go index 6e182bfae..75874e898 100644 --- a/pkg/task/backup.go +++ b/pkg/task/backup.go @@ -41,6 +41,7 @@ const ( maxBackupConcurrency = 256 ) +// CompressionConfig is the configuration for sst file compression type CompressionConfig struct { CompressionType kvproto.CompressionType `json:"compression-type" toml:"compression-type"` CompressionLevel int32 `json:"compression-level" toml:"compression-level"` From f355aa7c6bc210217c93d3b158b2d454fccfc62d Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 10 Aug 2020 15:12:14 +0800 Subject: [PATCH 5/9] update --- go.mod | 4 ++-- go.sum | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 69831a1e1..418ceca68 100644 --- a/go.mod +++ b/go.mod @@ -15,11 +15,11 @@ require ( github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 github.com/pingcap/errors v0.11.5-0.20200729012136-4e113ddee29e github.com/pingcap/failpoint v0.0.0-20200603062251-b230c36c413c - github.com/pingcap/kvproto v0.0.0-20200730045059-3177b43b983c + github.com/pingcap/kvproto v0.0.0-20200803054707-ebd5de15093f github.com/pingcap/log v0.0.0-20200511115504-543df19646ad github.com/pingcap/parser v0.0.0-20200731033026-84f62115187c github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200730093003-dc8c75cf7ca0 - github.com/pingcap/tidb v1.1.0-beta.0.20200806130549-32963d338791 + github.com/pingcap/tidb v1.1.0-beta.0.20200810064414-d81150394f9d github.com/pingcap/tidb-tools v4.0.1+incompatible github.com/pingcap/tipb v0.0.0-20200618092958-4fad48b4c8c3 github.com/prometheus/client_golang v1.5.1 diff --git a/go.sum b/go.sum index 413874dc7..65a06664a 100644 --- a/go.sum +++ b/go.sum @@ -420,6 +420,8 @@ github.com/ngaut/unistore v0.0.0-20200604061006-d8e9dc0ad154/go.mod h1:YGQzxn9cV github.com/ngaut/unistore v0.0.0-20200630072006-0c4035925f69/go.mod h1:Hxlp5VAoPOHwcXLUw/E+P3XjJX1EP38NWjXPpc4nuOE= github.com/ngaut/unistore v0.0.0-20200803051709-607d96233b1d h1:5wZOROIRh9/E4kyX6pXT7OtYyQtS9iUNcQtyt0Agil4= github.com/ngaut/unistore v0.0.0-20200803051709-607d96233b1d/go.mod h1:2QAH8tXCjeHuCSLEWKLYAzHPz2dB59VnhpPA2IDVeW4= +github.com/ngaut/unistore v0.0.0-20200806113332-5b9f73333a19 h1:N8pW0PTJEGTyHzZuN7sofxVsFmuvR+vFD0BNJ243k2o= +github.com/ngaut/unistore v0.0.0-20200806113332-5b9f73333a19/go.mod h1:RtZJKyiaHRiII+b9/g/4339rSikSvfrUJmIbrUkYVi4= github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -507,8 +509,8 @@ github.com/pingcap/kvproto v0.0.0-20200608081027-d02a6f65e956/go.mod h1:IOdRDPLy github.com/pingcap/kvproto v0.0.0-20200701055533-4ef28cac01f8/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200706115936-1e0910aabe6c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200715040832-c3e2e0b163ee/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20200730045059-3177b43b983c h1:k0frWfFFVEobqVJDPNj0YHm8cVAZ8wxWCrRvW8MNTgQ= -github.com/pingcap/kvproto v0.0.0-20200730045059-3177b43b983c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20200803054707-ebd5de15093f h1:y247vlXfpe8MYGt6pkVKqIoYmx2KoeF8jmBLbAlse6Y= +github.com/pingcap/kvproto v0.0.0-20200803054707-ebd5de15093f/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM= @@ -549,8 +551,9 @@ github.com/pingcap/tidb v1.1.0-beta.0.20200721005019-f5c6e59f0daf/go.mod h1:dYCO github.com/pingcap/tidb v1.1.0-beta.0.20200803035726-41c23700d8d1/go.mod h1:YFuuPMuceYoXIr4sCrtv1FUyJLvtYp9KisDsTk5dxlE= github.com/pingcap/tidb v1.1.0-beta.0.20200803051932-e291f8fbd1e0/go.mod h1:YFuuPMuceYoXIr4sCrtv1FUyJLvtYp9KisDsTk5dxlE= github.com/pingcap/tidb v1.1.0-beta.0.20200805053026-cd3e5ed82671/go.mod h1:+r9tlyUKG2zYzs2ajvEHiQlTx6WM0K2L1yabCHZwgGw= -github.com/pingcap/tidb v1.1.0-beta.0.20200806130549-32963d338791 h1:pMyU6dB+5G4ktdLI43bvRpzW3mi4dVei72cQHqgyCSQ= -github.com/pingcap/tidb v1.1.0-beta.0.20200806130549-32963d338791/go.mod h1:NHcZH46dkYwDd2IWUJaLOB0m54j7v2P5WdS4FvPR81w= +github.com/pingcap/tidb v1.1.0-beta.0.20200806060043-574540aa06ba/go.mod h1:NHcZH46dkYwDd2IWUJaLOB0m54j7v2P5WdS4FvPR81w= +github.com/pingcap/tidb v1.1.0-beta.0.20200810064414-d81150394f9d h1:01jQWk7xGtZCQWmu1d0uaVO6gq+nDbSC2YjMWdlBnIA= +github.com/pingcap/tidb v1.1.0-beta.0.20200810064414-d81150394f9d/go.mod h1:vLYo4E7Q6kzKYTskhP2MHBsodmZIRRUU63qdiFjlULA= github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200421113014-507d2bb3a15e+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200514040632-f76b3e428e19+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= From b08af1da0e82c96181539ca1c4734cf080192d41 Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 10 Aug 2020 15:18:07 +0800 Subject: [PATCH 6/9] fix format --- pkg/backup/client.go | 3 ++- pkg/task/backup.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/backup/client.go b/pkg/backup/client.go index 2dc3a2f7d..6bfa57827 100644 --- a/pkg/backup/client.go +++ b/pkg/backup/client.go @@ -600,7 +600,8 @@ func (bc *Client) fineGrainedBackup( defer wg.Done() for rg := range retry { backoffMs, err := - bc.handleFineGrained(ctx, boFork, rg, lastBackupTS, backupTS, compressType, compressLevel, rateLimit, concurrency, respCh) + bc.handleFineGrained(ctx, boFork, rg, lastBackupTS, backupTS, + compressType, compressLevel, rateLimit, concurrency, respCh) if err != nil { errCh <- err return diff --git a/pkg/task/backup.go b/pkg/task/backup.go index 75874e898..7083c229a 100644 --- a/pkg/task/backup.go +++ b/pkg/task/backup.go @@ -41,7 +41,7 @@ const ( maxBackupConcurrency = 256 ) -// CompressionConfig is the configuration for sst file compression +// CompressionConfig is the configuration for sst file compression. type CompressionConfig struct { CompressionType kvproto.CompressionType `json:"compression-type" toml:"compression-type"` CompressionLevel int32 `json:"compression-level" toml:"compression-level"` From 9c5059d1b5b733df7c1b374d09bc7ccb008d5d0b Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 10 Aug 2020 16:27:38 +0800 Subject: [PATCH 7/9] add compession test --- tests/br_full/run.sh | 60 ++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/tests/br_full/run.sh b/tests/br_full/run.sh index 6e5fe44a4..eee2999ce 100755 --- a/tests/br_full/run.sh +++ b/tests/br_full/run.sh @@ -28,35 +28,47 @@ for i in $(seq $DB_COUNT); do done # backup full -echo "backup start..." -run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB" --ratelimit 5 --concurrency 4 +echo "backup with lz4 start..." +run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB-lz4" --ratelimit 5 --concurrency 4 --compression lz4 +size_lz4=$(du -d 0 $TEST_DIR/$DB-lz4 | awk '{print $1}') -for i in $(seq $DB_COUNT); do - run_sql "DROP DATABASE $DB${i};" -done +echo "backup with zstd start..." +run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB-zstd" --ratelimit 5 --concurrency 4 --compression zstd --compression-level 6 +size_zstd=$(du -d 0 $TEST_DIR/$DB-zstd | awk '{print $1}') -# restore full -echo "restore start..." -run_br restore full -s "local://$TEST_DIR/$DB" --pd $PD_ADDR --ratelimit 1024 +if [ "$size_lz4" -le "$size_zstd" ]; then + echo "full backup lz4 size $size_lz4 is small than backup with zstd $size_zstd" + exit -1 +fi -for i in $(seq $DB_COUNT); do - row_count_new[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}') -done +for ct in (lz4 zstd); do + for i in $(seq $DB_COUNT); do + run_sql "DROP DATABASE $DB${i};" + done -fail=false -for i in $(seq $DB_COUNT); do - if [ "${row_count_ori[i]}" != "${row_count_new[i]}" ];then - fail=true - echo "TEST: [$TEST_NAME] fail on database $DB${i}" - fi - echo "database $DB${i} [original] row count: ${row_count_ori[i]}, [after br] row count: ${row_count_new[i]}" -done + # restore full + echo "restore with $ct backup start..." + run_br restore full -s "local://$TEST_DIR/$DB-$ct" --pd $PD_ADDR --ratelimit 1024 + + for i in $(seq $DB_COUNT); do + row_count_new[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}') + done + + fail=false + for i in $(seq $DB_COUNT); do + if [ "${row_count_ori[i]}" != "${row_count_new[i]}" ];then + fail=true + echo "TEST: [$TEST_NAME] fail on database $DB${i}" + fi + echo "database $DB${i} [original] row count: ${row_count_ori[i]}, [after br] row count: ${row_count_new[i]}" + done -if $fail; then - echo "TEST: [$TEST_NAME] failed!" - exit 1 -else - echo "TEST: [$TEST_NAME] successed!" + if $fail; then + echo "TEST: [$TEST_NAME] failed!" + exit 1 + else + echo "TEST: [$TEST_NAME] successed!" + fi fi for i in $(seq $DB_COUNT); do From 0377254ce56525470865f74b150a9b4b637b1890 Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 10 Aug 2020 17:31:19 +0800 Subject: [PATCH 8/9] fix test --- tests/br_full/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/br_full/run.sh b/tests/br_full/run.sh index eee2999ce..e92551550 100755 --- a/tests/br_full/run.sh +++ b/tests/br_full/run.sh @@ -41,7 +41,7 @@ if [ "$size_lz4" -le "$size_zstd" ]; then exit -1 fi -for ct in (lz4 zstd); do +for ct in lz4 zstd; do for i in $(seq $DB_COUNT); do run_sql "DROP DATABASE $DB${i};" done From 603015585f5da853b6b69a2d1ca5ba09f7880871 Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 10 Aug 2020 17:55:33 +0800 Subject: [PATCH 9/9] fix test --- tests/br_full/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/br_full/run.sh b/tests/br_full/run.sh index e92551550..6100c980d 100755 --- a/tests/br_full/run.sh +++ b/tests/br_full/run.sh @@ -69,7 +69,7 @@ for ct in lz4 zstd; do else echo "TEST: [$TEST_NAME] successed!" fi -fi +done for i in $(seq $DB_COUNT); do run_sql "DROP DATABASE $DB${i};"