From 927fca9fa29972a9a9b4d8f447e453f223411059 Mon Sep 17 00:00:00 2001 From: luancheng Date: Tue, 11 Feb 2020 20:24:07 +0800 Subject: [PATCH 1/3] *: support tls --- go.mod | 4 +- go.sum | 6 + pkg/conn/conn.go | 43 +++++- pkg/restore/client.go | 19 ++- pkg/restore/import.go | 12 +- pkg/restore/split_client.go | 12 +- pkg/restore/util.go | 2 +- pkg/task/backup.go | 2 +- pkg/task/common.go | 48 ++++++- pkg/task/restore.go | 4 +- pkg/utils/tso.go | 18 ++- tests/_utils/run_services | 174 +++++++++++++++++++++++ tests/br_tls/certificates/ca.pem | 21 +++ tests/br_tls/certificates/client-key.pem | 27 ++++ tests/br_tls/certificates/client.pem | 21 +++ tests/br_tls/certificates/server-key.pem | 27 ++++ tests/br_tls/certificates/server.pem | 22 +++ tests/br_tls/config/pd.toml | 9 ++ tests/br_tls/config/tidb.toml | 14 ++ tests/br_tls/config/tikv.toml | 19 +++ tests/br_tls/run.sh | 67 +++++++++ tests/run.sh | 82 +---------- 22 files changed, 551 insertions(+), 102 deletions(-) create mode 100644 tests/_utils/run_services create mode 100644 tests/br_tls/certificates/ca.pem create mode 100644 tests/br_tls/certificates/client-key.pem create mode 100644 tests/br_tls/certificates/client.pem create mode 100644 tests/br_tls/certificates/server-key.pem create mode 100644 tests/br_tls/certificates/server.pem create mode 100644 tests/br_tls/config/pd.toml create mode 100644 tests/br_tls/config/tidb.toml create mode 100644 tests/br_tls/config/tikv.toml create mode 100755 tests/br_tls/run.sh diff --git a/go.mod b/go.mod index 02aaa51ef..70ca858a0 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/fsouza/fake-gcs-server v1.15.0 github.com/go-sql-driver/mysql v1.4.1 github.com/gogo/protobuf v1.3.1 - github.com/golang/snappy v0.0.1 // indirect github.com/google/btree v1.0.0 github.com/google/uuid v1.1.1 github.com/mattn/go-runewidth v0.0.7 // indirect @@ -31,6 +30,9 @@ require ( github.com/sirupsen/logrus v1.4.2 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.3 + github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285 // indirect + github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect + go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 go.opencensus.io v0.22.2 // indirect go.uber.org/zap v1.13.0 golang.org/x/net v0.0.0-20191011234655-491137f69257 // indirect diff --git a/go.sum b/go.sum index a58f8bf55..7c8ae23e9 100644 --- a/go.sum +++ b/go.sum @@ -264,10 +264,12 @@ github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= @@ -388,11 +390,15 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d h1:4J9HCZVpvDmj2tiKGSTUnb3Ok/9CEQb9oqu9LHKQQpc= github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= +github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285 h1:uSDYjYejelKyceA6DiCsngFof9jAyeaSyX9XC5a1a7Q= +github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJfDRtkanvQPiooDH8HvJ2FBh+iKT/OmiQQ= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo= github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= diff --git a/pkg/conn/conn.go b/pkg/conn/conn.go index 6ef1e87eb..249cc67c5 100644 --- a/pkg/conn/conn.go +++ b/pkg/conn/conn.go @@ -3,12 +3,14 @@ package conn import ( "bytes" "context" + "crypto/tls" "encoding/json" "fmt" "io" "io/ioutil" "net/http" "net/url" + "reflect" "strings" "sync" "sync/atomic" @@ -25,6 +27,7 @@ import ( "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/backoff" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" "github.com/pingcap/br/pkg/glue" @@ -44,6 +47,7 @@ type Mgr struct { addrs []string cli *http.Client } + tlsConf *tls.Config dom *domain.Domain storage tikv.Storage grpcClis struct { @@ -58,9 +62,6 @@ func pdRequest( ctx context.Context, addr string, prefix string, cli *http.Client, method string, body io.Reader) ([]byte, error) { - if addr != "" && !strings.HasPrefix("http", addr) { - addr = "http://" + addr - } u, err := url.Parse(addr) if err != nil { return nil, errors.Trace(err) @@ -88,12 +89,33 @@ func pdRequest( } // NewMgr creates a new Mgr. -func NewMgr(ctx context.Context, g glue.Glue, pdAddrs string, storage tikv.Storage) (*Mgr, error) { +func NewMgr( + ctx context.Context, + g glue.Glue, + pdAddrs string, + storage tikv.Storage, + tlsConf *tls.Config, + securityOption pd.SecurityOption) (*Mgr, error) { addrs := strings.Split(pdAddrs, ",") failure := errors.Errorf("pd address (%s) has wrong format", pdAddrs) cli := &http.Client{Timeout: 30 * time.Second} + if tlsConf != nil { + transport := reflect.New(reflect.ValueOf(http.DefaultTransport).Elem().Type()).Interface().(*http.Transport) + transport.TLSClientConfig = tlsConf + cli.Transport = transport + } + + processedAddrs := make([]string, 0, len(addrs)) for _, addr := range addrs { + if addr != "" && !strings.HasPrefix("http", addr) { + if tlsConf != nil { + addr = "https://" + addr + } else { + addr = "http://" + addr + } + } + processedAddrs = append(processedAddrs, addr) _, failure = pdRequest(ctx, addr, clusterVersionPrefix, cli, http.MethodGet, nil) // TODO need check cluster version >= 3.1 when br release if failure == nil { @@ -104,7 +126,7 @@ func NewMgr(ctx context.Context, g glue.Glue, pdAddrs string, storage tikv.Stora return nil, errors.Annotatef(failure, "pd address (%s) not available, please check network", pdAddrs) } - pdClient, err := pd.NewClient(addrs, pd.SecurityOption{}) + pdClient, err := pd.NewClient(addrs, securityOption) if err != nil { log.Error("fail to create pd client", zap.Error(err)) return nil, err @@ -140,8 +162,9 @@ func NewMgr(ctx context.Context, g glue.Glue, pdAddrs string, storage tikv.Stora pdClient: pdClient, storage: storage, dom: dom, + tlsConf: tlsConf, } - mgr.pdHTTP.addrs = addrs + mgr.pdHTTP.addrs = processedAddrs mgr.pdHTTP.cli = cli mgr.grpcClis.clis = make(map[uint64]*grpc.ClientConn) return mgr, nil @@ -217,6 +240,9 @@ func (mgr *Mgr) getGrpcConnLocked(ctx context.Context, storeID uint64) (*grpc.Cl return nil, errors.Trace(err) } opt := grpc.WithInsecure() + if mgr.tlsConf != nil { + opt = grpc.WithTransportCredentials(credentials.NewTLS(mgr.tlsConf)) + } ctx, cancel := context.WithTimeout(ctx, dialTimeout) keepAlive := 10 keepAliveTimeout := 3 @@ -269,6 +295,11 @@ func (mgr *Mgr) GetTiKV() tikv.Storage { return mgr.storage } +// GetTLSConfig returns the tls config +func (mgr *Mgr) GetTLSConfig() *tls.Config { + return mgr.tlsConf +} + // GetLockResolver gets the LockResolver. func (mgr *Mgr) GetLockResolver() *tikv.LockResolver { return mgr.storage.GetLockResolver() diff --git a/pkg/restore/client.go b/pkg/restore/client.go index a06617084..a7e5c4d08 100644 --- a/pkg/restore/client.go +++ b/pkg/restore/client.go @@ -2,6 +2,7 @@ package restore import ( "context" + "crypto/tls" "encoding/json" "math" "sort" @@ -20,6 +21,7 @@ import ( "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/backoff" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" "github.com/pingcap/br/pkg/checksum" @@ -41,6 +43,7 @@ type Client struct { fileImporter FileImporter workerPool *utils.WorkerPool tableWorkerPool *utils.WorkerPool + tlsConf *tls.Config databases map[string]*utils.Database ddlJobs []*model.Job @@ -57,6 +60,7 @@ func NewRestoreClient( g glue.Glue, pdClient pd.Client, store kv.Storage, + tlsConf *tls.Config, ) (*Client, error) { ctx, cancel := context.WithCancel(ctx) db, err := NewDB(g, store) @@ -71,6 +75,7 @@ func NewRestoreClient( pdClient: pdClient, tableWorkerPool: utils.NewWorkerPool(128, "table"), db: db, + tlsConf: tlsConf, }, nil } @@ -112,8 +117,8 @@ func (rc *Client) InitBackupMeta(backupMeta *backup.BackupMeta, backend *backup. rc.backupMeta = backupMeta log.Info("load backupmeta", zap.Int("databases", len(rc.databases)), zap.Int("jobs", len(rc.ddlJobs))) - metaClient := NewSplitClient(rc.pdClient) - importClient := NewImportClient(metaClient) + metaClient := NewSplitClient(rc.pdClient, rc.tlsConf) + importClient := NewImportClient(metaClient, rc.tlsConf) rc.fileImporter = NewFileImporter(rc.ctx, metaClient, importClient, backend, rc.rateLimit) return nil } @@ -128,6 +133,11 @@ func (rc *Client) EnableOnline() { rc.isOnline = true } +// GetTLSConfig returns the tls config +func (rc *Client) GetTLSConfig() *tls.Config { + return rc.tlsConf +} + // GetTS gets a new timestamp from PD func (rc *Client) GetTS(ctx context.Context) (uint64, error) { p, l, err := rc.pdClient.GetTS(ctx) @@ -145,7 +155,7 @@ func (rc *Client) ResetTS(pdAddrs []string) error { i := 0 return utils.WithRetry(rc.ctx, func() error { idx := i % len(pdAddrs) - return utils.ResetTS(pdAddrs[idx], restoreTS) + return utils.ResetTS(pdAddrs[idx], restoreTS, rc.tlsConf) }, newResetTSBackoffer()) } @@ -332,6 +342,9 @@ func (rc *Client) switchTiKVMode(ctx context.Context, mode import_sstpb.SwitchMo bfConf.MaxDelay = time.Second * 3 for _, store := range stores { opt := grpc.WithInsecure() + if rc.tlsConf != nil { + opt = grpc.WithTransportCredentials(credentials.NewTLS(rc.tlsConf)) + } gctx, cancel := context.WithTimeout(ctx, time.Second*5) keepAlive := 10 keepAliveTimeout := 3 diff --git a/pkg/restore/import.go b/pkg/restore/import.go index 01f8456ef..887ee3e88 100644 --- a/pkg/restore/import.go +++ b/pkg/restore/import.go @@ -2,6 +2,7 @@ package restore import ( "context" + "crypto/tls" "strings" "sync" "time" @@ -15,6 +16,7 @@ import ( "github.com/pingcap/pd/pkg/codec" "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" "github.com/pingcap/br/pkg/summary" "github.com/pingcap/br/pkg/utils" @@ -47,13 +49,15 @@ type importClient struct { mu sync.Mutex metaClient SplitClient clients map[uint64]import_sstpb.ImportSSTClient + tlsConf *tls.Config } // NewImportClient returns a new ImporterClient -func NewImportClient(metaClient SplitClient) ImporterClient { +func NewImportClient(metaClient SplitClient, tlsConf *tls.Config) ImporterClient { return &importClient{ metaClient: metaClient, clients: make(map[uint64]import_sstpb.ImportSSTClient), + tlsConf: tlsConf, } } @@ -107,7 +111,11 @@ func (ic *importClient) getImportClient( if err != nil { return nil, err } - conn, err := grpc.Dial(store.GetAddress(), grpc.WithInsecure()) + opt := grpc.WithInsecure() + if ic.tlsConf != nil { + opt = grpc.WithTransportCredentials(credentials.NewTLS(ic.tlsConf)) + } + conn, err := grpc.Dial(store.GetAddress(), opt) if err != nil { return nil, err } diff --git a/pkg/restore/split_client.go b/pkg/restore/split_client.go index 8a618a191..9ab3ed7f1 100644 --- a/pkg/restore/split_client.go +++ b/pkg/restore/split_client.go @@ -3,6 +3,7 @@ package restore import ( "bytes" "context" + "crypto/tls" "encoding/json" "fmt" "io/ioutil" @@ -20,6 +21,7 @@ import ( pd "github.com/pingcap/pd/client" "github.com/pingcap/pd/server/schedule/placement" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" ) // SplitClient is an external client used by RegionSplitter. @@ -58,13 +60,15 @@ type SplitClient interface { type pdClient struct { mu sync.Mutex client pd.Client + tlsConf *tls.Config storeCache map[uint64]*metapb.Store } // NewSplitClient returns a client used by RegionSplitter. -func NewSplitClient(client pd.Client) SplitClient { +func NewSplitClient(client pd.Client, tlsConf *tls.Config) SplitClient { return &pdClient{ client: client, + tlsConf: tlsConf, storeCache: make(map[uint64]*metapb.Store), } } @@ -199,7 +203,11 @@ func (c *pdClient) BatchSplitRegions( if err != nil { return nil, err } - conn, err := grpc.Dial(store.GetAddress(), grpc.WithInsecure()) + opt := grpc.WithInsecure() + if c.tlsConf != nil { + opt = grpc.WithTransportCredentials(credentials.NewTLS(c.tlsConf)) + } + conn, err := grpc.Dial(store.GetAddress(), opt) if err != nil { return nil, err } diff --git a/pkg/restore/util.go b/pkg/restore/util.go index 64ccfab19..7b70c9806 100644 --- a/pkg/restore/util.go +++ b/pkg/restore/util.go @@ -284,7 +284,7 @@ func SplitRanges( elapsed := time.Since(start) summary.CollectDuration("split region", elapsed) }() - splitter := NewRegionSplitter(NewSplitClient(client.GetPDClient())) + splitter := NewRegionSplitter(NewSplitClient(client.GetPDClient(), client.GetTLSConfig())) return splitter.Split(ctx, ranges, rewriteRules, func(keys [][]byte) { for range keys { updateCh <- struct{}{} diff --git a/pkg/task/backup.go b/pkg/task/backup.go index ead4c2351..31594a08f 100644 --- a/pkg/task/backup.go +++ b/pkg/task/backup.go @@ -74,7 +74,7 @@ func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig if err != nil { return err } - mgr, err := newMgr(ctx, g, cfg.PD) + mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS) if err != nil { return err } diff --git a/pkg/task/common.go b/pkg/task/common.go index c3f866492..e08acd807 100644 --- a/pkg/task/common.go +++ b/pkg/task/common.go @@ -2,6 +2,7 @@ package task import ( "context" + "crypto/tls" "fmt" "regexp" "strings" @@ -9,10 +10,13 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/backup" + pd "github.com/pingcap/pd/client" "github.com/pingcap/tidb-tools/pkg/filter" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/store/tikv" "github.com/spf13/cobra" "github.com/spf13/pflag" + "go.etcd.io/etcd/pkg/transport" "github.com/pingcap/br/pkg/conn" "github.com/pingcap/br/pkg/glue" @@ -50,6 +54,25 @@ type TLSConfig struct { Key string `json:"key" toml:"key"` } +// IsEnabled checks if TLS open or not +func (tls *TLSConfig) IsEnabled() bool { + return tls.CA != "" +} + +// ToTLSConfig generate tls.Config +func (tls *TLSConfig) ToTLSConfig() (*tls.Config, error) { + tlsInfo := transport.TLSInfo{ + CertFile: tls.Cert, + KeyFile: tls.Key, + TrustedCAFile: tls.CA, + } + tlsConfig, err := tlsInfo.ClientConfig() + if err != nil { + return nil, errors.Trace(err) + } + return tlsConfig, nil +} + // Config is the common configuration for all BRIE tasks. type Config struct { storage.BackendOptions @@ -179,18 +202,39 @@ func (cfg *Config) ParseFromFlags(flags *pflag.FlagSet) error { } // newMgr creates a new mgr at the given PD address. -func newMgr(ctx context.Context, g glue.Glue, pds []string) (*conn.Mgr, error) { +func newMgr(ctx context.Context, g glue.Glue, pds []string, tlsConfig TLSConfig) (*conn.Mgr, error) { + var ( + tlsConf *tls.Config + err error + ) pdAddress := strings.Join(pds, ",") if len(pdAddress) == 0 { return nil, errors.New("pd address can not be empty") } + securityOption := pd.SecurityOption{} + if tlsConfig.IsEnabled() { + conf := config.GetGlobalConfig() + conf.Security.ClusterSSLCA = tlsConfig.CA + conf.Security.ClusterSSLCert = tlsConfig.Cert + conf.Security.ClusterSSLKey = tlsConfig.Key + config.StoreGlobalConfig(conf) + + securityOption.CAPath = tlsConfig.CA + securityOption.CertPath = tlsConfig.Cert + securityOption.KeyPath = tlsConfig.Key + tlsConf, err = tlsConfig.ToTLSConfig() + if err != nil { + return nil, err + } + } + // Disable GC because TiDB enables GC already. store, err := tikv.Driver{}.Open(fmt.Sprintf("tikv://%s?disableGC=true", pdAddress)) if err != nil { return nil, err } - return conn.NewMgr(ctx, g, pdAddress, store.(tikv.Storage)) + return conn.NewMgr(ctx, g, pdAddress, store.(tikv.Storage), tlsConf, securityOption) } // GetStorage gets the storage backend from the config. diff --git a/pkg/task/restore.go b/pkg/task/restore.go index bb00d189d..ef1ac861f 100644 --- a/pkg/task/restore.go +++ b/pkg/task/restore.go @@ -60,13 +60,13 @@ func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf ctx, cancel := context.WithCancel(c) defer cancel() - mgr, err := newMgr(ctx, g, cfg.PD) + mgr, err := newMgr(ctx, g, cfg.PD, cfg.TLS) if err != nil { return err } defer mgr.Close() - client, err := restore.NewRestoreClient(ctx, g, mgr.GetPDClient(), mgr.GetTiKV()) + client, err := restore.NewRestoreClient(ctx, g, mgr.GetPDClient(), mgr.GetTiKV(), mgr.GetTLSConfig()) if err != nil { return err } diff --git a/pkg/utils/tso.go b/pkg/utils/tso.go index a4ca5f5b5..96e7885d7 100644 --- a/pkg/utils/tso.go +++ b/pkg/utils/tso.go @@ -2,10 +2,13 @@ package utils import ( "bytes" + "crypto/tls" "encoding/json" "fmt" "net/http" + "reflect" "strings" + "time" "github.com/pingcap/errors" ) @@ -15,16 +18,23 @@ const ( ) // ResetTS resets the timestamp of PD to a bigger value -func ResetTS(pdAddr string, ts uint64) error { +func ResetTS(pdAddr string, ts uint64, tlsConf *tls.Config) error { req, err := json.Marshal(struct { TSO string `json:"tso,omitempty"` }{TSO: fmt.Sprintf("%d", ts)}) if err != nil { return err } - // TODO: Support TLS - reqURL := "http://" + pdAddr + resetTSURL - resp, err := http.Post(reqURL, "application/json", strings.NewReader(string(req))) + cli := &http.Client{Timeout: 30 * time.Second} + prefix := "http://" + if tlsConf != nil { + prefix = "https://" + transport := reflect.New(reflect.ValueOf(http.DefaultTransport).Elem().Type()).Interface().(*http.Transport) + transport.TLSClientConfig = tlsConf + cli.Transport = transport + } + reqURL := prefix + pdAddr + resetTSURL + resp, err := cli.Post(reqURL, "application/json", strings.NewReader(string(req))) if err != nil { return errors.Trace(err) } diff --git a/tests/_utils/run_services b/tests/_utils/run_services new file mode 100644 index 000000000..9ae0999bd --- /dev/null +++ b/tests/_utils/run_services @@ -0,0 +1,174 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +TEST_DIR=/tmp/backup_restore_test + +PD_ADDR="127.0.0.1:2379" +TIDB_IP="127.0.0.1" +TIDB_PORT="4000" +TIDB_ADDR="127.0.0.1:4000" +TIDB_STATUS_ADDR="127.0.0.1:10080" +# actaul tikv_addr are TIKV_ADDR${i} +TIKV_ADDR="127.0.0.1:2016" +TIKV_STATUS_ADDR="127.0.0.1:2018" +TIKV_COUNT=4 + +stop_services() { + killall -9 tikv-server || true + killall -9 pd-server || true + killall -9 tidb-server || true + + find "$TEST_DIR" -maxdepth 1 -not -path "$TEST_DIR" -not -name "*.log" | xargs rm -r || true +} + +start_services() { + stop_services + + echo "Starting PD..." + bin/pd-server \ + --client-urls "http://$PD_ADDR" \ + --log-file "$TEST_DIR/pd.log" \ + --data-dir "$TEST_DIR/pd" & + # wait until PD is online... + i=0 + while ! curl -o /dev/null -sf "http://$PD_ADDR/pd/api/v1/version"; do + i=$((i+1)) + if [ "$i" -gt 10 ]; then + echo 'Failed to start PD' + exit 1 + fi + sleep 3 + done + + echo "Starting TiKV..." + for i in $(seq $TIKV_COUNT); do + bin/tikv-server \ + --pd "$PD_ADDR" \ + -A "$TIKV_ADDR$i" \ + --status-addr "$TIKV_STATUS_ADDR$i" \ + --log-file "$TEST_DIR/tikv${i}.log" \ + -C "tests/config/tikv.toml" \ + -s "$TEST_DIR/tikv${i}" & + done + sleep 1 + + echo "Starting TiDB..." + bin/tidb-server \ + -P 4000 \ + --status 10080 \ + --store tikv \ + --path "$PD_ADDR" \ + --config "tests/config/tidb.toml" \ + --log-file "$TEST_DIR/tidb.log" & + + echo "Verifying TiDB is started..." + i=0 + while ! curl -o /dev/null -sf "http://$TIDB_IP:10080/status"; do + i=$((i+1)) + if [ "$i" -gt 10 ]; then + echo 'Failed to start TiDB' + exit 1 + fi + sleep 3 + done + + i=0 + while ! curl "http://$PD_ADDR/pd/api/v1/cluster/status" -sf | grep -q "\"is_initialized\": true"; do + i=$((i+1)) + if [ "$i" -gt 10 ]; then + echo 'Failed to bootstrap cluster' + exit 1 + fi + sleep 3 + done +} + +start_services_withTLS() { + stop_services + + PD_CONFIG="$1/config/pd.toml" + TIDB_CONFIG="$1/config/tidb.toml" + TIKV_CONFIG="$1/config/tikv.toml" + + echo $PD_CONFIG + echo $TIDB_CONFIG + echo $TIKV_CONFIG + + echo "Starting PD..." + bin/pd-server \ + --client-urls "https://$PD_ADDR" \ + --log-file "$TEST_DIR/pd.log" \ + --config "$PD_CONFIG" \ + --data-dir "$TEST_DIR/pd" & + # wait until PD is online... + i=0 + while ! curl -k --cert $1/certificates/client.pem \ + --key $1/certificates/client-key.pem \ + -o /dev/null -sf "https://$PD_ADDR/pd/api/v1/version"; do + i=$((i+1)) + if [ "$i" -gt 10 ]; then + echo 'Failed to start PD' + exit 1 + fi + sleep 3 + done + + echo "Starting TiKV..." + for i in $(seq $TIKV_COUNT); do + bin/tikv-server \ + --pd "$PD_ADDR" \ + -A "$TIKV_ADDR$i" \ + --log-file "$TEST_DIR/tikv${i}.log" \ + -C "$TIKV_CONFIG" \ + -s "$TEST_DIR/tikv${i}" & + done + sleep 1 + + echo "Starting TiDB..." + bin/tidb-server \ + -P 4000 \ + --status 10080 \ + --store tikv \ + --config "$TIDB_CONFIG" \ + --path "$PD_ADDR" \ + --log-file "$TEST_DIR/tidb.log" & + + echo "Verifying TiDB is started..." + i=0 + while ! curl -k --cert $1/certificates/client.pem \ + --key $1/certificates/client-key.pem \ + -o /dev/null -sf "https://$TIDB_IP:10080/status"; do + i=$((i+1)) + if [ "$i" -gt 10 ]; then + echo 'Failed to start TiDB' + exit 1 + fi + sleep 3 + done + + i=0 + while ! curl -k --cert $1/certificates/client.pem \ + --key $1/certificates/client-key.pem \ + "https://$PD_ADDR/pd/api/v1/cluster/status" -sf | grep -q "\"is_initialized\": true"; do + i=$((i+1)) + if [ "$i" -gt 10 ]; then + echo 'Failed to bootstrap cluster' + exit 1 + fi + sleep 3 + done +} \ No newline at end of file diff --git a/tests/br_tls/certificates/ca.pem b/tests/br_tls/certificates/ca.pem new file mode 100644 index 000000000..49098d653 --- /dev/null +++ b/tests/br_tls/certificates/ca.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDgDCCAmigAwIBAgIUHWvlRJydvYTR0ot3b8f6IlSHcGUwDQYJKoZIhvcNAQEL +BQAwVzELMAkGA1UEBhMCQ04xEDAOBgNVBAgTB0JlaWppbmcxEDAOBgNVBAcTB0Jl +aWppbmcxEDAOBgNVBAoTB1BpbmdDQVAxEjAQBgNVBAMTCU15IG93biBDQTAgFw0y +MDAyMTgwNzQxMDBaGA8yMTIwMDEyNTA3NDEwMFowVzELMAkGA1UEBhMCQ04xEDAO +BgNVBAgTB0JlaWppbmcxEDAOBgNVBAcTB0JlaWppbmcxEDAOBgNVBAoTB1BpbmdD +QVAxEjAQBgNVBAMTCU15IG93biBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAOAdNtjanFhPaKJHQjr7+h/Cpps5bLc6S1vmgi/EIi9PKv3eyDgtlW1r +As2sjXRMHjcuZp2hHJ9r9FrMQD1rQQq5vJzQqM+eyWLc2tyZWXNWkZVvpjU4Hy5k +jZFLXoyHgAvps/LGu81F5Lk5CvLHswWTyGQUCFi1l/cYcQg6AExh2pO/WJu4hQhe +1mBBIKsJhZ5b5tWruLeI+YIjD1oo1ADMHYLK1BHON2fUmUHRGbrYKu4yCuyip3wn +rbVlpabn7l1JBMatCUJLHR6VWQ2MNjrOXAEUYm4xGEN+tUYyUOGl5mHFguLl3OIn +wj+1dT3WOr/NraPYlwVOnAd9GNbPJj0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG +MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJ0CEqxLwEpI6J2gYJRg15oWZrj/ +MA0GCSqGSIb3DQEBCwUAA4IBAQCf8xRf7q1xAaGrc9HCPvN4OFkxDwz1CifrvrLR +ZgIWGUdCHDW2D1IiWKZQWeJKC1otA5x0hrS5kEGfkLFhADEU4txwp70DQaBArPti +pSgheIEbaT0H3BUTYSgS3VL2HjxN5OVMN6jNG3rWyxnJpNOCsJhhJXPK50CRZ7fk +Dcodj6FfEM2bfp2bGkxyVtUch7eepfUVbslXa7jE7Y8M3cr9NoLUcSP6D1RJWkNd +dBQoUsb6Ckq27ozEKOgwuBVv4BrrbFN//+7WHP8Vy6sSMyd+dJLBi6wehJjQhIz6 +vqLWE81rSJuxZqjLpCkFdeEF+9SRjWegU0ZDM4V+YeX53BPC +-----END CERTIFICATE----- diff --git a/tests/br_tls/certificates/client-key.pem b/tests/br_tls/certificates/client-key.pem new file mode 100644 index 000000000..43b021796 --- /dev/null +++ b/tests/br_tls/certificates/client-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA06qb7HABWHrU4CvBUO/2hXGgobi/UlTqTrYGZoJqSrvhKCP6 +HOivZjyWaSMDIfrguN+0C+bd80/5XGMmwjDt8PuZ+Ef3jcJLuB1e+Kms0s5tiTng +6Z028PkSpGvKXjPebWu7zoxVpDcTGM6MmlZQqpOuIgwi+7WX/bIgIu9wooCvJEGq +hScG2wpvK3txnpsVA4eXXdFoH0R5mtqbxVFfhwKMDqga4pRwJStLTDiMrtUz+OKc +rMIrkH4ndhvm2UYTVvhHlkZ3ooeDYsu40NnvedwaE+Ii7EnmcSDF9PaCVrXSK9F/ +KNRXX/x67PMWCVqcNyGtRsCuDe7FnDfGpudVXwIDAQABAoIBAHAzW/v1U4FHe1hp +WUxCJ3eNSAzyFdja0mlu6+2i7B05gpz4lTiFz5RuQXzx5lM43a6iRpqYgsbbed+T +X5RIw5iehnuqCnvGpsSuLQ27Q7VrX30ChUrQ37LVFSC7Usak0B9IoIFYun0WBLV9 +p+KYJqKFLiU2McUj+bGtnoNmUVqRzXQosoQue/pS9OknZ3NU7FxiyI3o4ME8rDvv +9x4vc1zcqbGXTQB224kOT0xoYt8RTmIbHvkR6/yszAtHDBcdzhINVuf38lv9FvaN +FxymwsY4IKPumQZlOEzHvSnpHTrwBMFdXjqpX1VxQb3yznEK+01MHf/tYsiU57IS +WVQMTeECgYEA7Fk0w66LGgdeeWrNTSTBCBPTofDVmR7Tro6k++5XTRt552ZoVz2l +8Lfi/Px5hIyve9gnM7slWrQ72JIQ5xVYZHtic3iwVFuRAD/QVfWU/SNsRsSB/93M +3BEumwJA6vN/qvkZueos3LOxN8kExk6GD0wIl6HjTeJPbbPHqmk6Pr0CgYEA5UQI +skaj8QGpjG8Hc10FeJpYsZiZA7gJaNu4RPqBj+1RHu/eYrL2mouooZdJfIJTmlTz +4NJcfb+Dl6qwbHUQ3mddhauFu1/YRwmaR5QKjwaBdeZbly9ljsRISFpjtosc7IBA +/Bl83xtbCboMdm7cH49X2CgRQ1uVFWraye0MBEsCgYEA43vtHFdYjaIAHa9dkV3J +6aNjtF/gxzNznXSwecfrAU1r5PydezLcEDh94vCDacAbe4EOIm2Dw6zsWUQlvrW9 +0WEs3mWQmnFTvECvnrz0PT2mDus/EO4EKuDi0dG2eC4MeJywVVB/A6J09XOnA9Q6 +lmihcIkiBinIN5etm2kS5aUCgYBCdcRnmZ6woKC7uvvX72FEosmPQgMpVtIzeW4j +YNLqHAtmAnbe+a4PAukxXp/I3ibKGFJSG+j/8uJ8tthJuG3ZavFrbFtqA9C4VwpI +MZwV9fbVbJ+kZfL0veWOQ9Wf9xe9Xzh3XBQcwNtVKH+wXVamN3FpkcPfWM8Q1Fb0 +LilLnQKBgQCq7+YlSnQX0rbmPTXVVb+B12rbqLDnqA0EuaVGrdu9zPPT04e5fpHU +SD33ibaEyeOF+zLg8T53whDbLJ0tURhUk+BlLTjdd99NXtyGMlfDnIsCnAeJhY8f +Iki6LYbbP2uWV4/5IDy9XW7J42Pfl9QyEVXq+PfTyPPjXC9/J4GOuw== +-----END RSA PRIVATE KEY----- diff --git a/tests/br_tls/certificates/client.pem b/tests/br_tls/certificates/client.pem new file mode 100644 index 000000000..7dace2f9d --- /dev/null +++ b/tests/br_tls/certificates/client.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDfDCCAmSgAwIBAgIUaupI14PPUSshx7FmD7lsVPFahwAwDQYJKoZIhvcNAQEL +BQAwVzELMAkGA1UEBhMCQ04xEDAOBgNVBAgTB0JlaWppbmcxEDAOBgNVBAcTB0Jl +aWppbmcxEDAOBgNVBAoTB1BpbmdDQVAxEjAQBgNVBAMTCU15IG93biBDQTAgFw0y +MDAyMTgwNzQ4MDBaGA8yMTIwMDEyNTA3NDgwMFowETEPMA0GA1UEAxMGY2xpZW50 +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA06qb7HABWHrU4CvBUO/2 +hXGgobi/UlTqTrYGZoJqSrvhKCP6HOivZjyWaSMDIfrguN+0C+bd80/5XGMmwjDt +8PuZ+Ef3jcJLuB1e+Kms0s5tiTng6Z028PkSpGvKXjPebWu7zoxVpDcTGM6MmlZQ +qpOuIgwi+7WX/bIgIu9wooCvJEGqhScG2wpvK3txnpsVA4eXXdFoH0R5mtqbxVFf +hwKMDqga4pRwJStLTDiMrtUz+OKcrMIrkH4ndhvm2UYTVvhHlkZ3ooeDYsu40Nnv +edwaE+Ii7EnmcSDF9PaCVrXSK9F/KNRXX/x67PMWCVqcNyGtRsCuDe7FnDfGpudV +XwIDAQABo4GDMIGAMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcD +AjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBRqlq/slflqw/cdlE+xNcnmmxZwlTAf +BgNVHSMEGDAWgBSdAhKsS8BKSOidoGCUYNeaFma4/zALBgNVHREEBDACggAwDQYJ +KoZIhvcNAQELBQADggEBAMGC48O77wZHZRRxXIpTQDMUSpGTKks76l+s1N7sMLrG +DCQi/XFVfV8e/Z1qs224IyU1IGXXcdwK0Zfa9owUmVmiHE8lznv7m9m7j4BGOshc +pvnJaeuUluKR/QHzwpMsUKudoEyRjn09e0Jl0/IcsKh13rzgd458XR0ShCjxybo4 +nQ1aZb1wOPLG6tpUYsV+x2Coc6TgnJWJYlDbRfpIuj6y16T1kKuWzpm6VU3kbiJ9 +/nzDgauuJHIlXEWL9dBZcpzUibFswIQyGsK7c4AJrtY1OGx0/2nZIIjtGY3gtWyX +XGV9c4kM695gl5rJndB4IPl5GQeJBCNyIaVybh7Va70= +-----END CERTIFICATE----- diff --git a/tests/br_tls/certificates/server-key.pem b/tests/br_tls/certificates/server-key.pem new file mode 100644 index 000000000..2779d6ec6 --- /dev/null +++ b/tests/br_tls/certificates/server-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAq9mcQG/nSLREM2r7s2tCKCE/1KJQvV0xmkIglFD2VDDfYW+C +mBME5LNWbYR6L0yCVHU8B7aVnw1FsbiF4TpUY3w/r4mOGl7QbGivMYvRe6Nh2xUO +TvctwFyv2FvrtBX1rZ5/8QLbz1IFHOtTV7QUzLzWq3fSAiF1vhVsS3BUmh6QvWU8 +q9dylpmUQ22otSRXmirwEzFt9K+w3VK9Z6aac7e2XRurVPxbqgQUq2bblUhii8Fc +dCUA8NenlWp+H64mN2TzVaGb5Csr7SNS7AWDEPKfoo7W3H7bzKlmRVcPeRdftwti +SI1jfboxprya/nbTyBPE/yfLU/SYn/b89epziwIDAQABAoIBACPlI08OULgN90Tq +LsLuP3ZUY5nNgaHcKnU3JMj2FE3Hm5ElkpijOF1w3Dep+T+R8pMjnbNavuvnAMy7 +ZzOBVIknNcI7sDPv5AcQ4q8trkbt/I2fW0rBNIw+j/hYUuZdw+BNABpeZ31pe2nr ++Y+TLNkLBKfyMiqBxK88mE81mmZKblyvXCawW0A/iDDJ7fPNqoGF+y9ylTYaNRPk +aJGnaEZobJ4Lm5tSqW4gRX2ft6Hm67RkvVaopPFnlkvfusXUTFUqEVQCURRUqXbf +1ah2chUHxj22UdY9540H5yVNgEP3oR+uS/hbZqxKcJUTznUW5th3CyQPIKMlGlcB +p+zWlTECgYEAxlY4zGJw4QQwGYMKLyWCSHUgKYrKu2Ub2JKJFMTdsoj9H7DI+WHf +lQaO9NCOo2lt0ofYM1MzEpI5Cl/aMrPw+mwquBbxWdMHXK2eSsUQOVo9HtUjgK2t +J2AYFCfsYndo+hCj3ApMHgiY3sghTCXeycvT52bm11VeNVcs3pKxIYMCgYEA3dAJ +PwIfAB8t+6JCP2yYH4ExNjoMNYMdXqhz4vt3UGwgskRqTW6qdd9JvrRQ/JPvGpDy +T375h/+lLw0E4ljsnOPGSzbXNf4bYRHTwPOL+LqVM4Bg90hjclqphElHChxep1di +WcdArB0oae/l4M96z3GjfnXIUVOp8K6BUQCab1kCgYAFFAQUR5j4SfEpVg+WsXEq +hcUzCxixv5785pOX8opynctNWmtq5zSgTjCu2AAu8u4a69t/ROwT16aaO2YM0kqj +Ps3BNOUtFZgkqVVaOL13mnXiKjbkfo3majFzoqoMw13uuSpY4fKc+j9fxOQFXRrd +M9jTHfFfJhJpbzf44uyiHQKBgFIPwzvyVvG+l05/Ky83x9fv/frn4thxV45LmAQj +sHKqbjZFpWZcSOgu4aOSJlwrhsw3T84lVcAAzmXn1STAbVll01jEQz6QciSpacP6 +1pAAx240UqtptpD6BbkROxz8ffA/Hf3E/6Itb2QyAsP3PqI8kpYYkTG1WCvZA7Kq +HHiRAoGAXbUZ25LcrmyuxKWpbty8fck1tjKPvclQB35rOx6vgnfW6pcKMeebYvgq +nJka/QunEReOH/kGxAd/+ymvUBuFQCfFg3Aus+DtAuh9AkBr+cIyPjJqynnIT87J +MbkOw4uEhDJAtGUR9o1j83N1f05bnEwssXiXR0LZPylb9Qzc4tg= +-----END RSA PRIVATE KEY----- diff --git a/tests/br_tls/certificates/server.pem b/tests/br_tls/certificates/server.pem new file mode 100644 index 000000000..ea5ef2d5f --- /dev/null +++ b/tests/br_tls/certificates/server.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDjzCCAnegAwIBAgIUWBTDQm4xOYDxZBTkpCQouREtT8QwDQYJKoZIhvcNAQEL +BQAwVzELMAkGA1UEBhMCQ04xEDAOBgNVBAgTB0JlaWppbmcxEDAOBgNVBAcTB0Jl +aWppbmcxEDAOBgNVBAoTB1BpbmdDQVAxEjAQBgNVBAMTCU15IG93biBDQTAgFw0y +MDAyMTgwOTExMDBaGA8yMTIwMDEyNTA5MTEwMFowFjEUMBIGA1UEAxMLdGlkYi1z +ZXJ2ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCr2ZxAb+dItEQz +avuza0IoIT/UolC9XTGaQiCUUPZUMN9hb4KYEwTks1ZthHovTIJUdTwHtpWfDUWx +uIXhOlRjfD+viY4aXtBsaK8xi9F7o2HbFQ5O9y3AXK/YW+u0FfWtnn/xAtvPUgUc +61NXtBTMvNard9ICIXW+FWxLcFSaHpC9ZTyr13KWmZRDbai1JFeaKvATMW30r7Dd +Ur1npppzt7ZdG6tU/FuqBBSrZtuVSGKLwVx0JQDw16eVan4friY3ZPNVoZvkKyvt +I1LsBYMQ8p+ijtbcftvMqWZFVw95F1+3C2JIjWN9ujGmvJr+dtPIE8T/J8tT9Jif +9vz16nOLAgMBAAGjgZEwgY4wDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsG +AQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBRVB/Bvdzvh +6WQRWpc9SzcbXLz77zAfBgNVHSMEGDAWgBSdAhKsS8BKSOidoGCUYNeaFma4/zAP +BgNVHREECDAGhwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQAAqg5pgGQqORKRSdlY +wzVvzKaulpvjZfVMM6YiOUtmlU0CGWq7E3gLFzkvebpU0KsFlbyZ92h/2Fw5Ay2b +kxkCy18mJ4lGkvF0cU4UD3XheFMvD2QWWRX4WPpAhStofrWOXeyq3Div2+fQjMJd +kyeWUzPU7T467IWUHOWNsFAjfVHNsmG45qLGt+XQckHTvASX5IvN+5tkRUCW30vO +b3BdDQUFglGTUFU2epaZGTti0SYiRiY+9R3zFWX4uBcEBYhk9e/0BU8FqdWW5GjI +pFpH9t64CjKIdRQXpIn4cogK/GwyuRuDPV/RkMjrIqOi7pGejXwyDe9avHFVR6re +oowA +-----END CERTIFICATE----- diff --git a/tests/br_tls/config/pd.toml b/tests/br_tls/config/pd.toml new file mode 100644 index 000000000..69cb94b6f --- /dev/null +++ b/tests/br_tls/config/pd.toml @@ -0,0 +1,9 @@ +# config of pd + +[security] +# Path of file that contains list of trusted SSL CAs. if set, following four settings shouldn't be empty +cacert-path = "tests/br_tls/certificates/ca.pem" +# Path of file that contains X509 certificate in PEM format. +cert-path = "tests/br_tls/certificates/server.pem" +# Path of file that contains X509 key in PEM format. +key-path = "tests/br_tls/certificates/server-key.pem" diff --git a/tests/br_tls/config/tidb.toml b/tests/br_tls/config/tidb.toml new file mode 100644 index 000000000..48a783332 --- /dev/null +++ b/tests/br_tls/config/tidb.toml @@ -0,0 +1,14 @@ +# config of tidb + +# Schema lease duration +# There are lot of ddl in the tests, setting this +# to 360s to test whether BR is gracefully shutdown. +lease = "360s" + +[security] +# Path of file that contains list of trusted SSL CAs for connection with cluster components. +cluster-ssl-ca = "tests/br_tls/certificates/ca.pem" +# Path of file that contains X509 certificate in PEM format for connection with cluster components. +cluster-ssl-cert = "tests/br_tls/certificates/server.pem" +# Path of file that contains X509 key in PEM format for connection with cluster components. +cluster-ssl-key = "tests/br_tls/certificates/server-key.pem" \ No newline at end of file diff --git a/tests/br_tls/config/tikv.toml b/tests/br_tls/config/tikv.toml new file mode 100644 index 000000000..b4859a731 --- /dev/null +++ b/tests/br_tls/config/tikv.toml @@ -0,0 +1,19 @@ +# config of tikv + +[coprocessor] +region-max-keys = 20 +region-split-keys = 12 + +[rocksdb] +max-open-files = 4096 +[raftdb] +max-open-files = 4096 +[raftstore] +# true (default value) for high reliability, this can prevent data loss when power failure. +sync-log = false + +[security] +# set the path for certificates. Empty string means disabling secure connectoins. +ca-path = "tests/br_tls/certificates/ca.pem" +cert-path = "tests/br_tls/certificates/server.pem" +key-path = "tests/br_tls/certificates/server-key.pem" diff --git a/tests/br_tls/run.sh b/tests/br_tls/run.sh new file mode 100755 index 000000000..9c494b700 --- /dev/null +++ b/tests/br_tls/run.sh @@ -0,0 +1,67 @@ +#!/bin/sh +# +# Copyright 2019 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +source $cur/../_utils/run_services + +DB="$TEST_NAME" +TABLE="usertable1" +TABLE2="usertable2" + +echo "Restart cluster with tls" +start_services_withTLS "$cur" + +run_sql "DROP DATABASE IF EXISTS $DB;" +run_sql "CREATE DATABASE $DB;" + +run_sql "CREATE TABLE $DB.$TABLE( \ + YCSB_KEY varchar(64) NOT NULL, \ + FIELD0 varchar(1) DEFAULT NULL, \ + PRIMARY KEY (YCSB_KEY) \ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;" + +run_sql "INSERT INTO $DB.$TABLE VALUES (\"a\", \"b\");" +run_sql "INSERT INTO $DB.$TABLE VALUES (\"aa\", \"b\");" + +run_sql "CREATE TABLE $DB.$TABLE2( \ + YCSB_KEY varchar(64) NOT NULL, \ + FIELD0 varchar(1) DEFAULT NULL, \ + PRIMARY KEY (YCSB_KEY) \ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;" + +run_sql "INSERT INTO $DB.$TABLE2 VALUES (\"c\", \"d\");" + +# backup db +echo "backup start..." +run_br --pd $PD_ADDR backup db --db "$DB" -s "local://$TEST_DIR/$DB" --ratelimit 5 --concurrency 4 --ca $cur/certificates/ca.pem --cert $cur/certificates/client.pem --key $cur/certificates/client-key.pem + +run_sql "DROP DATABASE $DB;" + +# restore db +echo "restore start..." +run_br restore db --db $DB -s "local://$TEST_DIR/$DB" --pd $PD_ADDR --ca $cur/certificates/ca.pem --cert $cur/certificates/client.pem --key $cur/certificates/client-key.pem + +table_count=$(run_sql "use $DB; show tables;" | grep "Tables_in" | wc -l) +if [ "$table_count" -ne "2" ];then + echo "TEST: [$TEST_NAME] failed!" + exit 1 +fi + +run_sql "DROP DATABASE $DB;" + +echo "Restart service without tls" +start_services diff --git a/tests/run.sh b/tests/run.sh index a4edb762a..053013352 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -14,85 +14,11 @@ # limitations under the License. set -eu +cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +source $cur/_utils/run_services -TEST_DIR=/tmp/backup_restore_test - -PD_ADDR="127.0.0.1:2379" -TIDB_IP="127.0.0.1" -TIDB_PORT="4000" -TIDB_ADDR="127.0.0.1:4000" -TIDB_STATUS_ADDR="127.0.0.1:10080" -# actaul tikv_addr are TIKV_ADDR${i} -TIKV_ADDR="127.0.0.1:2016" -TIKV_STATUS_ADDR="127.0.0.1:2018" -TIKV_COUNT=4 - -stop_services() { - killall -9 tikv-server || true - killall -9 pd-server || true - killall -9 tidb-server || true - - find "$TEST_DIR" -maxdepth 1 -not -path "$TEST_DIR" -not -name "*.log" | xargs rm -r || true -} - -start_services() { - stop_services - - mkdir -p "$TEST_DIR" - rm -f "$TEST_DIR"/*.log - - echo "Starting PD..." - bin/pd-server \ - --client-urls "http://$PD_ADDR" \ - --log-file "$TEST_DIR/pd.log" \ - --data-dir "$TEST_DIR/pd" & - # wait until PD is online... - while ! curl -o /dev/null -sf "http://$PD_ADDR/pd/api/v1/version"; do - sleep 1 - done - - echo "Starting TiKV..." - for i in $(seq $TIKV_COUNT); do - bin/tikv-server \ - --pd "$PD_ADDR" \ - -A "$TIKV_ADDR$i" \ - --status-addr "$TIKV_STATUS_ADDR$i" \ - --log-file "$TEST_DIR/tikv${i}.log" \ - -C "tests/config/tikv.toml" \ - -s "$TEST_DIR/tikv${i}" & - done - sleep 1 - - echo "Starting TiDB..." - bin/tidb-server \ - -P 4000 \ - --status 10080 \ - --store tikv \ - --path "$PD_ADDR" \ - --config "tests/config/tidb.toml" \ - --log-file "$TEST_DIR/tidb.log" & - - echo "Verifying TiDB is started..." - i=0 - while ! curl -o /dev/null -sf "http://$TIDB_IP:10080/status"; do - i=$((i+1)) - if [ "$i" -gt 10 ]; then - echo 'Failed to start TiDB' - exit 1 - fi - sleep 3 - done - - i=0 - while ! curl "http://$PD_ADDR/pd/api/v1/cluster/status" -sf | grep -q "\"is_initialized\": true"; do - i=$((i+1)) - if [ "$i" -gt 10 ]; then - echo 'Failed to bootstrap cluster' - exit 1 - fi - sleep 3 - done -} +mkdir -p "$TEST_DIR" +rm -f "$TEST_DIR"/*.log trap stop_services EXIT start_services From 8e7df8eab75ddd3db45ebb1a708e09bbed897002 Mon Sep 17 00:00:00 2001 From: luancheng Date: Mon, 24 Feb 2020 17:24:16 +0800 Subject: [PATCH 2/3] move tikv.driver to glue --- pkg/glue/glue.go | 2 ++ pkg/gluetidb/glue.go | 15 +++++++++++++++ pkg/task/common.go | 9 +-------- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/pkg/glue/glue.go b/pkg/glue/glue.go index bc6fb214e..b680370aa 100644 --- a/pkg/glue/glue.go +++ b/pkg/glue/glue.go @@ -4,6 +4,7 @@ import ( "context" "github.com/pingcap/parser/model" + pd "github.com/pingcap/pd/client" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta/autoid" @@ -13,6 +14,7 @@ import ( type Glue interface { BootstrapSession(store kv.Storage) (*domain.Domain, error) CreateSession(store kv.Storage) (Session, error) + Open(path string, option pd.SecurityOption) (kv.Storage, error) } // Session is an abstraction of the session.Session interface. diff --git a/pkg/gluetidb/glue.go b/pkg/gluetidb/glue.go index 6b9f2f667..27ae01c37 100644 --- a/pkg/gluetidb/glue.go +++ b/pkg/gluetidb/glue.go @@ -5,11 +5,14 @@ import ( "context" "github.com/pingcap/parser/model" + pd "github.com/pingcap/pd/client" + "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/br/pkg/glue" ) @@ -35,6 +38,18 @@ func (Glue) CreateSession(store kv.Storage) (glue.Session, error) { return &tidbSession{se: se}, nil } +// Open implements glue.Glue +func (Glue) Open(path string, option pd.SecurityOption) (kv.Storage, error) { + if option.CAPath != "" { + conf := config.GetGlobalConfig() + conf.Security.ClusterSSLCA = option.CAPath + conf.Security.ClusterSSLCert = option.CertPath + conf.Security.ClusterSSLKey = option.KeyPath + config.StoreGlobalConfig(conf) + } + return tikv.Driver{}.Open(path) +} + // Execute implements glue.Session func (gs *tidbSession) Execute(ctx context.Context, sql string) error { _, err := gs.se.Execute(ctx, sql) diff --git a/pkg/task/common.go b/pkg/task/common.go index e08acd807..2de01b326 100644 --- a/pkg/task/common.go +++ b/pkg/task/common.go @@ -12,7 +12,6 @@ import ( "github.com/pingcap/kvproto/pkg/backup" pd "github.com/pingcap/pd/client" "github.com/pingcap/tidb-tools/pkg/filter" - "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/store/tikv" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -214,12 +213,6 @@ func newMgr(ctx context.Context, g glue.Glue, pds []string, tlsConfig TLSConfig) securityOption := pd.SecurityOption{} if tlsConfig.IsEnabled() { - conf := config.GetGlobalConfig() - conf.Security.ClusterSSLCA = tlsConfig.CA - conf.Security.ClusterSSLCert = tlsConfig.Cert - conf.Security.ClusterSSLKey = tlsConfig.Key - config.StoreGlobalConfig(conf) - securityOption.CAPath = tlsConfig.CA securityOption.CertPath = tlsConfig.Cert securityOption.KeyPath = tlsConfig.Key @@ -230,7 +223,7 @@ func newMgr(ctx context.Context, g glue.Glue, pds []string, tlsConfig TLSConfig) } // Disable GC because TiDB enables GC already. - store, err := tikv.Driver{}.Open(fmt.Sprintf("tikv://%s?disableGC=true", pdAddress)) + store, err := g.Open(fmt.Sprintf("tikv://%s?disableGC=true", pdAddress), securityOption) if err != nil { return nil, err } From ca37fa52ea23ce0f04e80923be8187516d9ec795 Mon Sep 17 00:00:00 2001 From: luancheng Date: Wed, 26 Feb 2020 18:17:04 +0800 Subject: [PATCH 3/3] fix comments --- pkg/conn/conn.go | 3 +-- pkg/utils/tso.go | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/conn/conn.go b/pkg/conn/conn.go index 249cc67c5..d1f7858f6 100644 --- a/pkg/conn/conn.go +++ b/pkg/conn/conn.go @@ -10,7 +10,6 @@ import ( "io/ioutil" "net/http" "net/url" - "reflect" "strings" "sync" "sync/atomic" @@ -101,7 +100,7 @@ func NewMgr( failure := errors.Errorf("pd address (%s) has wrong format", pdAddrs) cli := &http.Client{Timeout: 30 * time.Second} if tlsConf != nil { - transport := reflect.New(reflect.ValueOf(http.DefaultTransport).Elem().Type()).Interface().(*http.Transport) + transport := http.DefaultTransport.(*http.Transport).Clone() transport.TLSClientConfig = tlsConf cli.Transport = transport } diff --git a/pkg/utils/tso.go b/pkg/utils/tso.go index 96e7885d7..ec084b20d 100644 --- a/pkg/utils/tso.go +++ b/pkg/utils/tso.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "net/http" - "reflect" "strings" "time" @@ -29,7 +28,7 @@ func ResetTS(pdAddr string, ts uint64, tlsConf *tls.Config) error { prefix := "http://" if tlsConf != nil { prefix = "https://" - transport := reflect.New(reflect.ValueOf(http.DefaultTransport).Elem().Type()).Interface().(*http.Transport) + transport := http.DefaultTransport.(*http.Transport).Clone() transport.TLSClientConfig = tlsConf cli.Transport = transport }