diff --git a/cmd/backup.go b/cmd/backup.go index 7f3b562f5..488a1d4fc 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -84,6 +84,7 @@ func newFullBackupCommand() *cobra.Command { return runBackupCommand(command, "Full backup") }, } + task.DefineFilterFlags(command) return command } diff --git a/cmd/restore.go b/cmd/restore.go index 343473b3a..c700b4bf2 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -96,6 +96,7 @@ func newFullRestoreCommand() *cobra.Command { return runRestoreCommand(cmd, "Full restore") }, } + task.DefineFilterFlags(command) return command } diff --git a/go.mod b/go.mod index 64725fdd7..02a0f130d 100644 --- a/go.mod +++ b/go.mod @@ -12,20 +12,19 @@ require ( github.com/gogo/protobuf v1.3.1 github.com/google/btree v1.0.0 github.com/google/uuid v1.1.1 - github.com/klauspost/cpuid v1.2.0 // indirect github.com/montanaflynn/stats v0.5.0 // indirect github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 github.com/pingcap/failpoint v0.0.0-20200506114213-c17f16071c53 - github.com/pingcap/kvproto v0.0.0-20200506072111-3a6b8b9cb29e + github.com/pingcap/kvproto v0.0.0-20200518112156-d4aeb467de29 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd github.com/pingcap/parser v3.1.0-beta.2.0.20200425032215-994651e9b6df+incompatible github.com/pingcap/pd/v3 v3.1.1-0.20200426091027-e639f0b1e62b github.com/pingcap/tidb v1.1.0-beta.0.20200426082429-52b31342cb0e - github.com/pingcap/tidb-tools v4.0.0-beta.2.0.20200514140446-49a686035c8a+incompatible + github.com/pingcap/tidb-tools v4.0.0-beta.2.0.20200709092834-22f96ccb95f7+incompatible github.com/pingcap/tipb v0.0.0-20200426072603-ce17d2d03251 - github.com/prometheus/client_golang v1.0.0 - github.com/prometheus/common v0.4.1 + github.com/prometheus/client_golang v1.2.1 + github.com/prometheus/common v0.7.0 github.com/sirupsen/logrus v1.4.2 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 33f8d2422..e52cd96d3 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafo github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.30.24 h1:y3JPD51VuEmVqN3BEDVm4amGpDma2cKJcDPuAU1OR58= @@ -48,10 +50,14 @@ github.com/aws/aws-sdk-go v1.30.24/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZve github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d h1:rQlvB2AYWme2bIB18r/SipGiMEVJYE9U0z+MGoU/LtQ= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= +github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cheggaaa/pb/v3 v3.0.4 h1:QZEPYOj2ix6d5oEg63fbHmpolrnNiwjUsk+h74Yt4bM= github.com/cheggaaa/pb/v3 v3.0.4/go.mod h1:7rgWxLrAUcFMkvJuv09+DYi7mMUYi8nO9iOWcvGJPfw= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -123,7 +129,9 @@ github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxm github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= @@ -252,8 +260,6 @@ github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= @@ -352,8 +358,8 @@ github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rG github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20200213074014-83e827908584/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200317043902-2838e21ca222/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20200506072111-3a6b8b9cb29e h1:X1t/m9VcPBzx1BtSfziqILkMz5gRwBWdbgpeR129p8o= -github.com/pingcap/kvproto v0.0.0-20200506072111-3a6b8b9cb29e/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20200518112156-d4aeb467de29 h1:NpW1OuYrIl+IQrSsVbtyHpHpazmSCHy+ysrOixY0xY4= +github.com/pingcap/kvproto v0.0.0-20200518112156-d4aeb467de29/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM= @@ -366,9 +372,10 @@ github.com/pingcap/pd/v3 v3.1.1-0.20200426091027-e639f0b1e62b h1:oYWX0dRvRdN55HU github.com/pingcap/pd/v3 v3.1.1-0.20200426091027-e639f0b1e62b/go.mod h1:su5qKHuRSE0oSO8DBmck0WK17T18OPjCNi3XToM+uCM= github.com/pingcap/tidb v1.1.0-beta.0.20200426082429-52b31342cb0e h1:IGNsARumk3Sqev3gtodBGL2mqqYcJ5vERE1pTF8QtxI= github.com/pingcap/tidb v1.1.0-beta.0.20200426082429-52b31342cb0e/go.mod h1:DnbLUiiSBC3TTeLIfsmJeVdgkmY+h3y4mL01ZmoPJ8I= +github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200317092225-ed6b2a87af54+incompatible h1:tYADqdmWwgDOwf/qEN0trJAy6H3c3Tt/QZx1z4qVrRQ= github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200317092225-ed6b2a87af54+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tidb-tools v4.0.0-beta.2.0.20200514140446-49a686035c8a+incompatible h1:+TayhJMmgQ3E5vDP3z6iuCofGAv8SzO/sFXDJsi0QHA= -github.com/pingcap/tidb-tools v4.0.0-beta.2.0.20200514140446-49a686035c8a+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= +github.com/pingcap/tidb-tools v4.0.0-beta.2.0.20200709092834-22f96ccb95f7+incompatible h1:Iq1nSvEP37l3c0lR3YbVXq5+eGheATEECjnKXl7aTW0= +github.com/pingcap/tidb-tools v4.0.0-beta.2.0.20200709092834-22f96ccb95f7+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20200401093201-cc8b75c53383 h1:y1ayhtouCaO0u74JNMN8s20CGJT0yIuAb8UXOYnCALc= github.com/pingcap/tipb v0.0.0-20200401093201-cc8b75c53383/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pingcap/tipb v0.0.0-20200426072603-ce17d2d03251 h1:DnrF3jRNUvV37jb4zGT6SL3g06b7kQcD2i1oqgSKSOE= @@ -383,6 +390,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= +github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -390,9 +399,13 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCb github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7 h1:FUL3b97ZY2EPqg2NbXKuMHs5pXJB9hjj1fDHnF2vl28= github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -621,6 +634,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/backup/client.go b/pkg/backup/client.go index f384e4f42..a3254257c 100644 --- a/pkg/backup/client.go +++ b/pkg/backup/client.go @@ -18,7 +18,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/parser/model" pd "github.com/pingcap/pd/v3/client" - "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb-tools/pkg/table-filter" "github.com/pingcap/tidb/distsql" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" @@ -212,7 +212,7 @@ func appendRanges(tbl *model.TableInfo, tblID int64) ([]kv.KeyRange, error) { func BuildBackupRangeAndSchema( dom *domain.Domain, storage kv.Storage, - tableFilter *filter.Filter, + tableFilter filter.Filter, backupTS uint64, ) ([]rtree.Range, *Schemas, error) { info, err := dom.GetSnapshotInfoSchema(backupTS) @@ -233,7 +233,7 @@ func BuildBackupRangeAndSchema( randAlloc := autoid.NewAllocator(storage, dbInfo.ID, false, autoid.AutoRandomType) for _, tableInfo := range dbInfo.Tables { - if !tableFilter.Match(&filter.Table{Schema: dbInfo.Name.L, Name: tableInfo.Name.L}) { + if !tableFilter.MatchTable(dbInfo.Name.O, tableInfo.Name.O) { // Skip tables other than the given table. continue } diff --git a/pkg/backup/schema_test.go b/pkg/backup/schema_test.go index 98173dd55..5e2860d39 100644 --- a/pkg/backup/schema_test.go +++ b/pkg/backup/schema_test.go @@ -8,7 +8,7 @@ import ( "sync/atomic" . "github.com/pingcap/check" - "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb-tools/pkg/table-filter" "github.com/pingcap/tidb/util/testkit" "github.com/pingcap/tidb/util/testleak" @@ -56,9 +56,7 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) { tk := testkit.NewTestKit(c, s.mock.Storage) // Table t1 is not exist. - testFilter, err := filter.New(false, &filter.Rules{ - DoTables: []*filter.Table{{Schema: "test", Name: "t1"}}, - }) + testFilter, err := filter.Parse([]string{"test.t1"}) c.Assert(err, IsNil) _, backupSchemas, err := BuildBackupRangeAndSchema( s.mock.Domain, s.mock.Storage, testFilter, math.MaxUint64) @@ -66,9 +64,7 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) { c.Assert(backupSchemas, IsNil) // Database is not exist. - fooFilter, err := filter.New(false, &filter.Rules{ - DoTables: []*filter.Table{{Schema: "foo", Name: "t1"}}, - }) + fooFilter, err := filter.Parse([]string{"foo.t1"}) c.Assert(err, IsNil) _, backupSchemas, err = BuildBackupRangeAndSchema( s.mock.Domain, s.mock.Storage, fooFilter, math.MaxUint64) @@ -76,7 +72,7 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) { c.Assert(backupSchemas, IsNil) // Empty database. - noFilter, err := filter.New(false, &filter.Rules{}) + noFilter, err := filter.Parse([]string{"*.*"}) c.Assert(err, IsNil) _, backupSchemas, err = BuildBackupRangeAndSchema( s.mock.Domain, s.mock.Storage, noFilter, math.MaxUint64) diff --git a/pkg/task/backup.go b/pkg/task/backup.go index 68536ee1e..9f7d77814 100644 --- a/pkg/task/backup.go +++ b/pkg/task/backup.go @@ -14,7 +14,6 @@ import ( "github.com/pingcap/log" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" - "github.com/pingcap/tidb-tools/pkg/filter" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" @@ -104,10 +103,6 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig if err != nil { return err } - tableFilter, err := filter.New(cfg.CaseSensitive, &cfg.Filter) - if err != nil { - return err - } mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS, conn.SkipTiFlash, cfg.CheckRequirements) if err != nil { return err @@ -134,7 +129,7 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig isIncrementalBackup := cfg.LastBackupTS > 0 ranges, backupSchemas, err := backup.BuildBackupRangeAndSchema( - mgr.GetDomain(), mgr.GetTiKV(), tableFilter, backupTS) + mgr.GetDomain(), mgr.GetTiKV(), cfg.TableFilter, backupTS) if err != nil { return err } diff --git a/pkg/task/common.go b/pkg/task/common.go index 00dcd666e..d7cb5455e 100644 --- a/pkg/task/common.go +++ b/pkg/task/common.go @@ -6,14 +6,13 @@ import ( "context" "crypto/tls" "fmt" - "regexp" "strings" "github.com/gogo/protobuf/proto" "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/backup" pd "github.com/pingcap/pd/v3/client" - "github.com/pingcap/tidb-tools/pkg/filter" + filter "github.com/pingcap/tidb-tools/pkg/table-filter" "github.com/pingcap/tidb/store/tikv" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -47,6 +46,8 @@ const ( flagRemoveTiFlash = "remove-tiflash" flagConcurrency = "concurrency" flagChecksum = "checksum" + flagFilter = "filter" + flagCaseSensitive = "case-sensitive" flagCheckRequirement = "check-requirements" ) @@ -90,10 +91,20 @@ type Config struct { // LogProgress is true means the progress bar is printed to the log instead of stdout. LogProgress bool `json:"log-progress" toml:"log-progress"` - RemoveTiFlash bool `json:"remove-tiflash" toml:"remove-tiflash"` - CaseSensitive bool `json:"case-sensitive" toml:"case-sensitive"` - CheckRequirements bool `json:"check-requirements" toml:"check-requirements"` - Filter filter.Rules `json:"black-white-list" toml:"black-white-list"` + // CaseSensitive should not be used. + // + // Deprecated: This field is kept only to satisfy the cyclic dependency with TiDB. This field + // should be removed after TiDB upgrades the BR dependency. + CaseSensitive bool + // Filter should not be used, use TableFilter instead. + // + // Deprecated: This field is kept only to satisfy the cyclic dependency with TiDB. This field + // should be removed after TiDB upgrades the BR dependency. + Filter filter.MySQLReplicationRules + + TableFilter filter.Filter `json:"-" toml:"-"` + RemoveTiFlash bool `json:"remove-tiflash" toml:"remove-tiflash"` + CheckRequirements bool `json:"check-requirements" toml:"check-requirements"` } // DefineCommonFlags defines the flags common to all BRIE commands. @@ -125,19 +136,26 @@ func DefineCommonFlags(flags *pflag.FlagSet) { storage.DefineFlags(flags) } -// DefineDatabaseFlags defines the required --db flag. +// DefineDatabaseFlags defines the required --db flag for `db` subcommand. func DefineDatabaseFlags(command *cobra.Command) { command.Flags().String(flagDatabase, "", "database name") _ = command.MarkFlagRequired(flagDatabase) } -// DefineTableFlags defines the required --db and --table flags. +// DefineTableFlags defines the required --db and --table flags for `table` subcommand. func DefineTableFlags(command *cobra.Command) { DefineDatabaseFlags(command) command.Flags().StringP(flagTable, "t", "", "table name") _ = command.MarkFlagRequired(flagTable) } +// DefineFilterFlags defines the --filter and --case-sensitive flags for `full` subcommand. +func DefineFilterFlags(command *cobra.Command) { + flags := command.Flags() + flags.StringArrayP(flagFilter, "f", []string{"*.*"}, "select tables to process") + flags.Bool(flagCaseSensitive, false, "whether the table names used in --filter should be case-sensitive") +} + // ParseFromFlags parses the TLS config from the flag set. func (tls *TLSConfig) ParseFromFlags(flags *pflag.FlagSet) error { var err error @@ -199,20 +217,39 @@ func (cfg *Config) ParseFromFlags(flags *pflag.FlagSet) error { return errors.Trace(err) } - if dbFlag := flags.Lookup(flagDatabase); dbFlag != nil { - db := escapeFilterName(dbFlag.Value.String()) + var caseSensitive bool + if filterFlag := flags.Lookup(flagFilter); filterFlag != nil { + f, err := filter.Parse(filterFlag.Value.(pflag.SliceValue).GetSlice()) + if err != nil { + return err + } + cfg.TableFilter = f + caseSensitive, err = flags.GetBool(flagCaseSensitive) + if err != nil { + return errors.Trace(err) + } + } else if dbFlag := flags.Lookup(flagDatabase); dbFlag != nil { + db := dbFlag.Value.String() if len(db) == 0 { return errors.New("empty database name is not allowed") } if tblFlag := flags.Lookup(flagTable); tblFlag != nil { - tbl := escapeFilterName(tblFlag.Value.String()) + tbl := tblFlag.Value.String() if len(tbl) == 0 { return errors.New("empty table name is not allowed") } - cfg.Filter.DoTables = []*filter.Table{{Schema: db, Name: tbl}} + cfg.TableFilter = filter.NewTablesFilter(filter.Table{ + Schema: db, + Name: tbl, + }) } else { - cfg.Filter.DoDBs = []string{db} + cfg.TableFilter = filter.NewSchemasFilter(db) } + } else { + cfg.TableFilter, _ = filter.Parse([]string{"*.*"}) + } + if !caseSensitive { + cfg.TableFilter = filter.CaseInsensitive(cfg.TableFilter) } checkRequirements, err := flags.GetBool(flagCheckRequirement) if err != nil { @@ -299,10 +336,3 @@ func ReadBackupMeta( } return u, s, backupMeta, nil } - -func escapeFilterName(name string) string { - if !strings.HasPrefix(name, "~") { - return name - } - return "~^" + regexp.QuoteMeta(name) + "$" -} diff --git a/pkg/task/restore.go b/pkg/task/restore.go index 20676b460..843187f0a 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -11,7 +11,6 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/backup" "github.com/pingcap/log" - "github.com/pingcap/tidb-tools/pkg/filter" "github.com/pingcap/tidb/config" "github.com/spf13/pflag" "go.uber.org/multierr" @@ -147,9 +146,9 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf return errors.New("cannot do transactional restore from raw kv data") } - files, tables, dbs, err := filterRestoreFiles(client, cfg) - if err != nil { - return err + files, tables, dbs := filterRestoreFiles(client, cfg) + if len(dbs) == 0 && len(tables) != 0 { + return errors.New("invalid backup, contain tables but no databases") } var newTS uint64 @@ -160,9 +159,6 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf } } ddlJobs := restore.FilterDDLJobs(client.GetDDLJobs(), tables) - if err != nil { - return err - } // pre-set TiDB config for restore enableTiDBConfig() @@ -306,16 +302,11 @@ func dropToBlackhole( func filterRestoreFiles( client *restore.Client, cfg *RestoreConfig, -) (files []*backup.File, tables []*utils.Table, dbs []*utils.Database, err error) { - tableFilter, err := filter.New(cfg.CaseSensitive, &cfg.Filter) - if err != nil { - return nil, nil, nil, err - } - +) (files []*backup.File, tables []*utils.Table, dbs []*utils.Database) { for _, db := range client.GetDatabases() { createdDatabase := false for _, table := range db.Tables { - if !tableFilter.Match(&filter.Table{Schema: db.Info.Name.O, Name: table.Info.Name.O}) { + if !cfg.TableFilter.MatchTable(db.Info.Name.O, table.Info.Name.O) { continue } @@ -327,9 +318,6 @@ func filterRestoreFiles( tables = append(tables, table) } } - if len(dbs) == 0 && len(tables) != 0 { - err = errors.New("invalid backup, contain tables but no databases") - } return } diff --git a/tests/br_table_filter/run.sh b/tests/br_table_filter/run.sh new file mode 100755 index 000000000..f04b7b186 --- /dev/null +++ b/tests/br_table_filter/run.sh @@ -0,0 +1,120 @@ +#!/bin/sh +# +# Copyright 2020 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eux +DB="$TEST_NAME" + +run_sql "create schema $DB;" + +run_sql "create table $DB.one(c int);" +run_sql "create table $DB.two(c int);" +run_sql "create table $DB.three(c int);" +run_sql "create table $DB.four(c int);" +run_sql "create table $DB.FIVE(c int);" +run_sql "create table $DB.TEN(c int);" +run_sql 'create table '"$DB"'.`the,special,table`(c int);' + +run_sql "insert into $DB.one values (1);" +run_sql "insert into $DB.two values (2);" +run_sql "insert into $DB.three values (3);" +run_sql "insert into $DB.four values (4);" +run_sql "insert into $DB.FIVE values (5);" +run_sql "insert into $DB.TEN values (10);" +run_sql 'insert into '"$DB"'.`the,special,table` values (375);' + +echo 'Simple check' + +run_br backup full -f "$DB.*" -s "local://$TEST_DIR/$DB/full" --pd $PD_ADDR +run_sql "drop schema $DB;" +run_br restore full -s "local://$TEST_DIR/$DB/full" --pd $PD_ADDR + +run_sql "select c from $DB.one;" +run_sql "select c from $DB.two;" +run_sql "select c from $DB.three;" +run_sql "select c from $DB.four;" +run_sql "select c from $DB.FIVE;" +run_sql "select c from $DB.TEN;" +run_sql 'select c from '"$DB"'.`the,special,table`;' + +echo 'Filtered backup check' + +run_br backup full -f "$DB.t*" -s "local://$TEST_DIR/$DB/t" --pd $PD_ADDR +run_sql "drop schema $DB;" +run_br restore full -s "local://$TEST_DIR/$DB/t" --pd $PD_ADDR + +! run_sql "select c from $DB.one;" +run_sql "select c from $DB.two;" +run_sql "select c from $DB.three;" +! run_sql "select c from $DB.four;" +! run_sql "select c from $DB.FIVE;" +run_sql "select c from $DB.TEN;" +run_sql 'select c from '"$DB"'.`the,special,table`;' + +echo 'Filtered restore check' + +run_sql "drop schema $DB;" +run_br restore full -f "*.f*" -s "local://$TEST_DIR/$DB/full" --pd $PD_ADDR + +! run_sql "select c from $DB.one;" +! run_sql "select c from $DB.two;" +! run_sql "select c from $DB.three;" +run_sql "select c from $DB.four;" +run_sql "select c from $DB.FIVE;" +! run_sql "select c from $DB.TEN;" +! run_sql 'select c from '"$DB"'.`the,special,table`;' + +echo 'Multiple filters check' + +run_sql "drop schema $DB;" +run_br restore full -f '*.*' -f '!*.five' -f '!*.`the,special,table`' -s "local://$TEST_DIR/$DB/full" --pd $PD_ADDR + +run_sql "select c from $DB.one;" +run_sql "select c from $DB.two;" +run_sql "select c from $DB.three;" +run_sql "select c from $DB.four;" +! run_sql "select c from $DB.FIVE;" +run_sql "select c from $DB.TEN;" +! run_sql 'select c from '"$DB"'.`the,special,table`;' + +echo 'Case sensitive restore check' + +run_sql "drop schema $DB;" +run_br restore full --case-sensitive -f '*.t*' -s "local://$TEST_DIR/$DB/full" --pd $PD_ADDR + +! run_sql "select c from $DB.one;" +run_sql "select c from $DB.two;" +run_sql "select c from $DB.three;" +! run_sql "select c from $DB.four;" +! run_sql "select c from $DB.FIVE;" +! run_sql "select c from $DB.TEN;" +run_sql 'select c from '"$DB"'.`the,special,table`;' + +echo 'Case sensitive backup check' + +run_sql "drop schema $DB;" +run_br restore full --case-sensitive -s "local://$TEST_DIR/$DB/full" --pd $PD_ADDR +run_br backup full --case-sensitive -f "$DB.[oF]*" -s "local://$TEST_DIR/$DB/of" --pd $PD_ADDR +run_sql "drop schema $DB;" +run_br restore full --case-sensitive -s "local://$TEST_DIR/$DB/of" --pd $PD_ADDR + +run_sql "select c from $DB.one;" +! run_sql "select c from $DB.two;" +! run_sql "select c from $DB.three;" +! run_sql "select c from $DB.four;" +run_sql "select c from $DB.FIVE;" +! run_sql "select c from $DB.TEN;" +! run_sql 'select c from '"$DB"'.`the,special,table`;' + +run_sql "drop schema $DB;"