storage: use path in WalkDir and Open

pkg/storage/gcs.go

@@ -123,8 +123,8 @@ func (s *gcsStorage) FileExists(ctx context.Context, name string) (bool, error)
 	return true, nil
 }
 
-// Open a Reader by file name.
-func (s *gcsStorage) Open(ctx context.Context, name string) (ReadSeekCloser, error) {
+// Open a Reader by file path.
+func (s *gcsStorage) Open(ctx context.Context, path string) (ReadSeekCloser, error) {
 	// TODO, implement this if needed
 	panic("Unsupported Operation")
 }

pkg/storage/local.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	"io/ioutil"
 	"os"
-	"path"
 	"path/filepath"
 
 	"github.com/pingcap/errors"
@@ -20,20 +19,20 @@ type LocalStorage struct {
 }
 
 func (l *LocalStorage) Write(ctx context.Context, name string, data []byte) error {
-	filepath := path.Join(l.base, name)
-	return ioutil.WriteFile(filepath, data, 0644) // nolint:gosec
-	// the backupmeta file _is_ intended to be world-readable.
+	path := filepath.Join(l.base, name)
+	return ioutil.WriteFile(path, data, 0644) // nolint:gosec
+	// the backup meta file _is_ intended to be world-readable.
 }
 
 func (l *LocalStorage) Read(ctx context.Context, name string) ([]byte, error) {
-	filepath := path.Join(l.base, name)
-	return ioutil.ReadFile(filepath)
+	path := filepath.Join(l.base, name)
+	return ioutil.ReadFile(path)
 }
 
 // FileExists implement ExternalStorage.FileExists.
 func (l *LocalStorage) FileExists(ctx context.Context, name string) (bool, error) {
-	filepath := path.Join(l.base, name)
-	return pathExists(filepath)
+	path := filepath.Join(l.base, name)
+	return pathExists(path)
 }
 
 // WalkDir traverse all the files in a dir.
@@ -51,19 +50,21 @@ func (l *LocalStorage) WalkDir(ctx context.Context, dir string, listCount int64,
 		if f == nil || f.IsDir() {
 			return nil
 		}
-
-		return fn(f.Name(), f.Size())
+		// in mac osx, the path parameter is absolute path; in linux, the path is relative path to execution base dir,
+		// so use Rel to convert to relative path to l.base
+		path, _ = filepath.Rel(l.base, path)
+		return fn(path, f.Size())
 	})
 }
 
-// CreateUploader implenments ExternalStorage interface.
+// CreateUploader implements ExternalStorage interface.
 func (l *LocalStorage) CreateUploader(ctx context.Context, name string) (Uploader, error) {
 	panic("local storage not support multi-upload")
 }
 
-// Open a Reader by file name.
-func (l *LocalStorage) Open(ctx context.Context, name string) (ReadSeekCloser, error) {
-	return os.Open(path.Join(l.base, name))
+// Open a Reader by file path, path is a relative path to base path.
+func (l *LocalStorage) Open(ctx context.Context, path string) (ReadSeekCloser, error) {
+	return os.Open(filepath.Join(l.base, path))
 }
 
 func pathExists(_path string) (bool, error) {

pkg/storage/noop.go

@@ -23,8 +23,8 @@ func (*noopStorage) FileExists(ctx context.Context, name string) (bool, error) {
 	return false, nil
 }
 
-// Open a Reader by file name.
-func (*noopStorage) Open(ctx context.Context, name string) (ReadSeekCloser, error) {
+// Open a Reader by file path.
+func (*noopStorage) Open(ctx context.Context, path string) (ReadSeekCloser, error) {
 	return noopReader{}, nil
 }
 

pkg/storage/s3.go

@@ -396,23 +396,23 @@ func (rs *S3Storage) WalkDir(ctx context.Context, dir string, listCount int64, f
 	return nil
 }
 
-// Open a Reader by file name.
-func (rs *S3Storage) Open(ctx context.Context, name string) (ReadSeekCloser, error) {
-	reader, err := rs.open(ctx, name, 0, 0)
+// Open a Reader by file path.
+func (rs *S3Storage) Open(ctx context.Context, path string) (ReadSeekCloser, error) {
+	reader, err := rs.open(ctx, path, 0, 0)
 	if err != nil {
 		return nil, err
 	}
 	return &s3ObjectReader{
 		storage: rs,
-		name:    name,
+		name:    path,
 		reader:  reader,
 	}, nil
 }
 
-func (rs *S3Storage) open(ctx context.Context, name string, startOffset int64, endOffset int64) (io.ReadCloser, error) {
+func (rs *S3Storage) open(ctx context.Context, path string, startOffset int64, endOffset int64) (io.ReadCloser, error) {
 	input := &s3.GetObjectInput{
 		Bucket: aws.String(rs.options.Bucket),
-		Key:    aws.String(rs.options.Prefix + name),
+		Key:    aws.String(rs.options.Prefix + path),
 	}
 
 	var rangeOffset *string

pkg/storage/storage.go

@@ -33,8 +33,8 @@ type ExternalStorage interface {
 	Read(ctx context.Context, name string) ([]byte, error)
 	// FileExists return true if file exists
 	FileExists(ctx context.Context, name string) (bool, error)
-	// Open a Reader by file name.
-	Open(ctx context.Context, name string) (ReadSeekCloser, error)
+	// Open a Reader by file path. path is relative path to storage base path
+	Open(ctx context.Context, path string) (ReadSeekCloser, error)
 	// WalkDir traverse all the files in a dir.
 	//
 	// fn is the function called for each regular file visited by WalkDir.