*: merging tidb-lightning to release-4.0

.gitattributes

@@ -0,0 +1 @@
+*_generated.go linguist-generated=true

.github/ISSUE_TEMPLATE/bug-report.md

@@ -2,7 +2,7 @@
 name: "🐛 Bug Report"
 about: Something isn't working as expected
 title: ''
-labels: 'bug'
+labels: 'type/bug '
 ---
 
 Please answer these questions before submitting your issue. Thanks!
@@ -23,7 +23,23 @@ If possible, provide a recipe for reproducing the error.
 
 <!--
 br -V
+tidb-lightning -V
 tidb-server -V
 tikv-server -V
 pd-server -V
 -->
+
+5. Operation logs
+   - Please upload `br.log` for BR if possible
+   - Please upload `tidb-lightning.log` for TiDB-Lightning if possible
+   - Please upload `tikv-importer.log` from TiKV-Importer if possible
+   - Other interesting logs
+
+
+6. Configuration of the cluster and the task
+   - `tidb-lightning.toml` for TiDB-Lightning if possible
+   - `tikv-importer.toml` for TiKV-Importer if possible
+   - `topology.yml` if deployed by TiUP
+
+
+7. Screenshot/exported-PDF of Grafana dashboard or metrics' graph in Prometheus if possible

.github/ISSUE_TEMPLATE/feature-request.md

@@ -1,7 +1,7 @@
 ---
 name: "🚀 Feature Request"
 about: I have a suggestion
-labels: enhancement
+labels: 'type/feature-request'
 ---
 
 ## Feature Request

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,24 @@
+---
+name: "\U0001F914 Question"
+labels: "type/question"
+about: Usage question that isn't answered in docs or discussion
+
+---
+
+## Question
+
+
+<!--
+
+Before asking a question, make sure you have:
+
+- Searched existing Stack Overflow questions.
+- Googled your question.
+- Searched open and closed [GitHub issues](https://github.com/pingcap/br/issues?q=is%3Aissue)
+- Read the documentation:
+    * [BR documentation](https://docs.pingcap.com/tidb/stable/backup-and-restore-tool)
+    * [Lightning documentation](https://docs.pingcap.com/tidb/stable/tidb-lightning-overview)
+    * [BR 中文文档](https://docs.pingcap.com/zh/tidb/stable/backup-and-restore-tool)
+    * [Lightning 中文文档](https://docs.pingcap.com/zh/tidb/stable/tidb-lightning-overview)
+
+-->

.github/challenge-bot.yml

@@ -0,0 +1 @@
+defaultSigLabel: sig/migrate

.gitignore

@@ -1,15 +1,15 @@
-br
-bin/
-_tools/
-.idea/
-backupmeta
+/br
+/bin
+/.idea
 *.log
 *.ngo
-*.coverprofile
-coverage.txt
-docker/data/
-docker/logs/
+/docker/data/
+/docker/logs/
 *.swp
 .DS_Store
-go.mod
-go.sum
+/go.mod
+/go.sum
+
+# for the web interface
+web/node_modules/
+web/dist/

Makefile

@@ -1,66 +1,141 @@
 PROTOC ?= $(shell which protoc)
 PROTOS := $(shell find $(shell pwd) -type f -name '*.proto' -print)
-CWD := $(shell pwd)
-PACKAGES := go list ./...
+CWD    := $(shell pwd)
+TOOLS  := $(CWD)/tools/bin
+PACKAGES := go list ./... | grep -vE 'vendor|test|proto|diff|bin|fuzz'
 PACKAGE_DIRECTORIES := $(PACKAGES) | sed 's/github.com\/pingcap\/br\/*//'
+PACKAGE_FILES := $$(find . -name '*.go' -type f | grep -vE 'vendor|\.pb\.go|lightning/mock|res_vfsdata')
 CHECKER := awk '{ print } END { if (NR > 0) { exit 1 } }'
 
 BR_PKG := github.com/pingcap/br
 
-LDFLAGS += -X "$(BR_PKG)/pkg/utils.BRReleaseVersion=$(shell git describe --tags --dirty)"
+VERSION := v4.0.0-dev
+release_branch_regex := ^release-[0-9]\.[0-9].*$$
+ifneq ($(shell git rev-parse --abbrev-ref HEAD | egrep $(release_branch_regex)),)
+	# If we are in release branch, use tag version.
+	VERSION := $(shell git describe --tags --dirty)
+else ifneq ($(shell git status --porcelain),)
+	# Add -dirty if the working tree is dirty for non release branch.
+	VERSION := $(VERSION)-dirty
+endif
+
+LDFLAGS += -X "$(BR_PKG)/pkg/utils.BRReleaseVersion=$(VERSION)"
 LDFLAGS += -X "$(BR_PKG)/pkg/utils.BRBuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')"
 LDFLAGS += -X "$(BR_PKG)/pkg/utils.BRGitHash=$(shell git rev-parse HEAD)"
 LDFLAGS += -X "$(BR_PKG)/pkg/utils.BRGitBranch=$(shell git rev-parse --abbrev-ref HEAD)"
+# TODO: unify LDFLAGS
+LDFLAGS += -X "$(BR_PKG)/pkg/lightning/common.ReleaseVersion=$(VERSION)"
+LDFLAGS += -X "$(BR_PKG)/pkg/lightning/common.BuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')"
+LDFLAGS += -X "$(BR_PKG)/pkg/lightning/common.GitHash=$(shell git rev-parse HEAD)"
+LDFLAGS += -X "$(BR_PKG)/pkg/lightning/common.GitBranch=$(shell git rev-parse --abbrev-ref HEAD)"
+LDFLAGS += -X "$(BR_PKG)/pkg/lightning/common.GoVersion=$(shell go version)"
+
+LIGHTNING_BIN     := bin/tidb-lightning
+LIGHTNING_CTL_BIN := bin/tidb-lightning-ctl
+BR_BIN            := bin/br
+VFSGENDEV_BIN     := tools/bin/vfsgendev
+TEST_DIR          := /tmp/backup_restore_test
+
+path_to_add := $(addsuffix /bin,$(subst :,/bin:,$(GOPATH)))
+export PATH := $(path_to_add):$(PATH)
 
 GOBUILD := CGO_ENABLED=1 GO111MODULE=on go build -trimpath -ldflags '$(LDFLAGS)'
 GOTEST  := CGO_ENABLED=1 GO111MODULE=on go test -ldflags '$(LDFLAGS)'
 PREPARE_MOD := cp go.mod1 go.mod && cp go.sum1 go.sum
 FINISH_MOD := cp go.mod go.mod1 && cp go.sum go.sum1
 
+RACE_FLAG =
 ifeq ("$(WITH_RACE)", "1")
-	RACEFLAG = -race
+	RACE_FLAG = -race
+	GOBUILD   = CGO_ENABLED=1 GO111MODULE=on $(GO) build -ldflags '$(LDFLAGS)'
 endif
 
 all: build check test
 
 prepare:
 	$(PREPARE_MOD)
 
-build:
+finish-prepare:
+	$(FINISH_MOD)
+
+%_generated.go: %.rl
+	ragel -Z -G2 -o tmp_parser.go $<
+	@echo '// Code generated by ragel DO NOT EDIT.' | cat - tmp_parser.go | sed 's|//line |//.... |g' > $@
+	@rm tmp_parser.go
+
+data_parsers: tools pkg/lightning/mydump/parser_generated.go web
+	PATH="$(GOPATH)/bin":"$(PATH)":"$(TOOLS)" protoc -I. -I"$(GOPATH)/src" pkg/lightning/checkpoints/file_checkpoints.proto --gogofaster_out=.
+	$(TOOLS)/vfsgendev -source='"github.com/pingcap/br/pkg/lightning/web".Res' && mv res_vfsdata.go pkg/lightning/web/
+
+web:
+	cd web && npm install && npm run build
+
+build: br lightning lightning-ctl
+
+br:
+	$(PREPARE_MOD)
+	$(GOBUILD) $(RACEFLAG) -o $(BR_BIN) cmd/br/*.go
+
+lightning_for_web:
 	$(PREPARE_MOD)
-	$(GOBUILD) $(RACEFLAG) -o bin/br
+	$(GOBUILD) $(RACE_FLAG) -tags dev -o $(LIGHTNING_BIN) cmd/tidb-lightning/main.go
+
+lightning:
+	$(PREPARE_MOD)
+	$(GOBUILD) $(RACE_FLAG) -o $(LIGHTNING_BIN) cmd/tidb-lightning/main.go
+
+lightning-ctl:
+	$(PREPARE_MOD)
+	$(GOBUILD) $(RACE_FLAG) -o $(LIGHTNING_CTL_BIN) cmd/tidb-lightning-ctl/main.go
 
 build_for_integration_test:
 	$(PREPARE_MOD)
 	@make failpoint-enable
 	($(GOTEST) -c -cover -covermode=count \
 		-coverpkg=$(BR_PKG)/... \
-		-o bin/br.test && \
+		-o $(BR_BIN).test \
+		github.com/pingcap/br/cmd/br && \
+	$(GOTEST) -c -cover -covermode=count \
+		-coverpkg=$(BR_PKG)/... \
+		-o $(LIGHTNING_BIN).test \
+		github.com/pingcap/br/cmd/tidb-lightning && \
+	$(GOTEST) -c -cover -covermode=count \
+		-coverpkg=$(BR_PKG)/... \
+		-o $(LIGHTNING_CTL_BIN).test \
+		github.com/pingcap/br/cmd/tidb-lightning-ctl && \
 	$(GOBUILD) $(RACEFLAG) -o bin/locker tests/br_key_locked/*.go && \
 	$(GOBUILD) $(RACEFLAG) -o bin/gc tests/br_z_gc_safepoint/*.go && \
 	$(GOBUILD) $(RACEFLAG) -o bin/oauth tests/br_gcs/*.go && \
-	$(GOBUILD) $(RACEFLAG) -o bin/rawkv tests/br_rawkv/*.go) || (make failpoint-disable && exit 1)
+	$(GOBUILD) $(RACEFLAG) -o bin/rawkv tests/br_rawkv/*.go && \
+	$(GOBUILD) $(RACE_FLAG) -o bin/parquet_gen tests/lightning_checkpoint_parquet/*.go \
+	) || (make failpoint-disable && exit 1)
 	@make failpoint-disable
 
 test:
 	$(PREPARE_MOD)
 	@make failpoint-enable
-	$(GOTEST) $(RACEFLAG) -tags leak ./... || ( make failpoint-disable && exit 1 )
+	$(GOTEST) $(RACEFLAG) -tags leak $$($(PACKAGES)) || ( make failpoint-disable && exit 1 )
 	@make failpoint-disable
 
 testcover: tools
+	mkdir -p "$(TEST_DIR)"
 	$(PREPARE_MOD)
 	@make failpoint-enable
-	GO111MODULE=on tools/bin/overalls \
-		-project=$(BR_PKG) \
-		-covermode=count \
-		-ignore='.git,vendor,tests,_tools' \
-		-debug \
-		-- -coverpkg=./... || ( make failpoint-disable && exit 1 )
+	$(GOTEST) -cover -covermode=count -coverprofile="$(TEST_DIR)/cov.unit.out" $$($(PACKAGES)) || ( make failpoint-disable && exit 1 )
+	@make failpoint-disable
 
 integration_test: bins build build_for_integration_test
 	tests/run.sh
 
+coverage: tools
+	tools/bin/gocovmerge "$(TEST_DIR)"/cov.* | grep -vE ".*.pb.go|.*__failpoint_binding__.go" > "$(TEST_DIR)/all_cov.out"
+ifeq ("$(JenkinsCI)", "1")
+	tools/bin/goveralls -coverprofile=$(TEST_DIR)/all_cov.out -service=jenkins-ci -repotoken $(COVERALLS_TOKEN)
+else
+	go tool cover -html "$(TEST_DIR)/all_cov.out" -o "$(TEST_DIR)/all_cov.html"
+	grep -F '<option' "$(TEST_DIR)/all_cov.html"
+endif
+
 bins:
 	@which bin/tidb-server
 	@which bin/tikv-server
@@ -73,6 +148,7 @@ bins:
 	@which bin/libtiflash_proxy.so
 	@which bin/cdc
 	@which bin/fake-gcs-server
+	@which bin/tikv-importer
 	if [ ! -d bin/flash_cluster_manager ]; then echo "flash_cluster_manager not exist"; exit 1; fi
 
 tools:
@@ -88,9 +164,11 @@ check:
 static: export GO111MODULE=on
 static: prepare tools
 	@ # Not running vet and fmt through metalinter becauase it ends up looking at vendor
-	tools/bin/gofumports -w -d -format-only -local $(BR_PKG) $$($(PACKAGE_DIRECTORIES)) 2>&1 | $(CHECKER)
-	tools/bin/govet --shadow $$($(PACKAGE_DIRECTORIES)) 2>&1 | $(CHECKER)
+	tools/bin/gofumports -w -d -format-only -local $(BR_PKG) $(PACKAGE_FILES) 2>&1 | $(CHECKER)
+	# TODO: go vet lightning packages too.
+	tools/bin/govet --shadow $$($(PACKAGE_DIRECTORIES) | grep -v "lightning") 2>&1 | $(CHECKER)
 
+	# TODO: lint lightning packages too.
 	@# why some lints are disabled?
 	@#   gochecknoglobals - disabled because we do use quite a lot of globals
 	@#          goimports - executed above already, gofumports
@@ -130,17 +208,20 @@ static: prepare tools
 		--disable exhaustive \
 		--disable godot \
 		--disable gosec \
-		$$($(PACKAGE_DIRECTORIES))
+		$$($(PACKAGE_DIRECTORIES) | grep -v "lightning")
 	# pingcap/errors APIs are mixed with multiple patterns 'pkg/errors',
 	# 'juju/errors' and 'pingcap/parser'. To avoid confusion and mistake,
 	# we only allow a subset of APIs, that's "Normalize|Annotate|Trace|Cause".
+	# TODO: check lightning packages.
 	@# TODO: allow more APIs when we need to support "workaound".
-	grep -Rn --exclude="*_test.go" -E "(\t| )errors\.[A-Z]" cmd pkg | \
+	grep -Rn --include="*.go" --exclude="*_test.go" -E "(\t| )errors\.[A-Z]" \
+		$$($(PACKAGE_DIRECTORIES) | grep -vE "tests|lightning") | \
 		grep -vE "Normalize|Annotate|Trace|Cause|RedactLogEnabled" 2>&1 | $(CHECKER)
 
 lint: prepare tools
 	@echo "linting"
-	CGO_ENABLED=0 tools/bin/revive -formatter friendly -config revive.toml $$($(PACKAGES))
+	# TODO: lint lightning packages.
+	CGO_ENABLED=0 tools/bin/revive -formatter friendly -config revive.toml $$($(PACKAGES) | grep -v "lightning")
 
 tidy:
 	@echo "go mod tidy"
@@ -160,4 +241,4 @@ failpoint-enable: tools
 failpoint-disable: tools
 	tools/bin/failpoint-ctl disable
 
-.PHONY: tools
+.PHONY: tools web

cmd/backup.go → cmd/br/backup.go

@@ -1,6 +1,6 @@
 // Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
 
-package cmd
+package main
 
 import (
 	"github.com/pingcap/errors"

cmd/cmd.go → cmd/br/cmd.go

@@ -1,6 +1,6 @@
 // Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
 
-package cmd
+package main
 
 import (
 	"context"
@@ -151,6 +151,7 @@ func Init(cmd *cobra.Command) (err error) {
 		}
 
 		// Initialize the pprof server.
+		// TODO: Support TLS.
 		statusAddr, e := cmd.Flags().GetString(FlagStatusAddr)
 		if e != nil {
 			err = e

cmd/debug.go → cmd/br/debug.go

@@ -1,6 +1,6 @@
 // Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
 
-package cmd
+package main
 
 import (
 	"bytes"

main.go → cmd/br/main.go

@@ -10,8 +10,6 @@ import (
 	"github.com/pingcap/log"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
-
-	"github.com/pingcap/br/cmd"
 )
 
 func main() {
@@ -44,19 +42,20 @@ func main() {
 		TraverseChildren: true,
 		SilenceUsage:     true,
 	}
-	cmd.AddFlags(rootCmd)
-	cmd.SetDefaultContext(ctx)
+	AddFlags(rootCmd)
+	SetDefaultContext(ctx)
 	rootCmd.AddCommand(
-		cmd.NewDebugCommand(),
-		cmd.NewBackupCommand(),
-		cmd.NewRestoreCommand(),
+		NewDebugCommand(),
+		NewBackupCommand(),
+		NewRestoreCommand(),
 	)
 	// Ouputs cmd.Print to stdout.
 	rootCmd.SetOut(os.Stdout)
 
 	rootCmd.SetArgs(os.Args[1:])
 	if err := rootCmd.Execute(); err != nil {
+		cancel()
 		log.Error("br failed", zap.Error(err))
-		os.Exit(1)
+		os.Exit(1) // nolint:gocritic
 	}
 }