From 2ab2f231c4453db187bcc61949444b8ed18af75f Mon Sep 17 00:00:00 2001 From: glorv Date: Wed, 9 Sep 2020 15:03:04 +0800 Subject: [PATCH 1/3] fix walk dir file size --- pkg/storage/local.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/storage/local.go b/pkg/storage/local.go index 25678d7cd..a96e4feb6 100644 --- a/pkg/storage/local.go +++ b/pkg/storage/local.go @@ -53,7 +53,17 @@ func (l *LocalStorage) WalkDir(ctx context.Context, opt *WalkOption, fn func(str // in mac osx, the path parameter is absolute path; in linux, the path is relative path to execution base dir, // so use Rel to convert to relative path to l.base path, _ = filepath.Rel(l.base, path) - return fn(path, f.Size()) + + size := f.Size() + // if not a regular file, we need to use os.stat to get the real file size + if !f.Mode().IsRegular() { + stat, err := os.Stat(path) + if err != nil { + return errors.Trace(err) + } + size = stat.Size() + } + return fn(path, size) }) } From 7515dc5a2cf04f204443126004e8b671d54e2f09 Mon Sep 17 00:00:00 2001 From: glorv Date: Wed, 9 Sep 2020 15:41:53 +0800 Subject: [PATCH 2/3] fix path and add a unit test --- pkg/storage/local.go | 2 +- pkg/storage/local_test.go | 61 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 pkg/storage/local_test.go diff --git a/pkg/storage/local.go b/pkg/storage/local.go index a96e4feb6..734eba5a0 100644 --- a/pkg/storage/local.go +++ b/pkg/storage/local.go @@ -57,7 +57,7 @@ func (l *LocalStorage) WalkDir(ctx context.Context, opt *WalkOption, fn func(str size := f.Size() // if not a regular file, we need to use os.stat to get the real file size if !f.Mode().IsRegular() { - stat, err := os.Stat(path) + stat, err := os.Stat(filepath.Join(l.base, path)) if err != nil { return errors.Trace(err) } diff --git a/pkg/storage/local_test.go b/pkg/storage/local_test.go new file mode 100644 index 000000000..530269d3e --- /dev/null +++ b/pkg/storage/local_test.go @@ -0,0 +1,61 @@ +package storage + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/pingcap/check" +) + +type testLocalSuite struct{} + +var _ = Suite(&testLocalSuite{}) + +func (r *testStorageSuite) TestWalkDirWithSoftLinkFile(c *C) { + dir1 := c.MkDir() + name1 := "test.warehouse.0.sql" + path1 := filepath.Join(dir1, name1) + f1, err := os.Create(path1) + c.Assert(err, IsNil) + + data := "/* whatever pragmas */;" + + "INSERT INTO `namespaced`.`table` (columns, more, columns) VALUES (1,-2, 3),\n(4,5., 6);" + + "INSERT `namespaced`.`table` (x,y,z) VALUES (7,8,9);" + + "insert another_table values (10,11e1,12, '(13)', '(', 14, ')');" + _, err = f1.Write([]byte(data)) + c.Assert(err, IsNil) + err = f1.Close() + c.Assert(err, IsNil) + + dir2 := c.MkDir() + name2 := "test.warehouse.1.sql" + f2, err := os.Create(filepath.Join(dir2, name2)) + c.Assert(err, IsNil) + _, err = f2.Write([]byte(data)) + c.Assert(err, IsNil) + err = f2.Close() + c.Assert(err, IsNil) + + err = os.Symlink(path1, filepath.Join(dir2, name1)) + c.Assert(err, IsNil) + + sb, err := ParseBackend(fmt.Sprintf("file://%s", dir2), &BackendOptions{}) + c.Assert(err, IsNil) + + store, err := Create(context.TODO(), sb, true) + c.Assert(err, IsNil) + + i := 0 + names := []string{name1, name2} + err = store.WalkDir(context.TODO(), &WalkOption{}, func(path string, size int64) error { + c.Assert(path, Equals, names[i]) + c.Assert(size, Equals, int64(len(data))) + i++ + + return nil + }) + c.Assert(err, IsNil) + c.Assert(i, Equals, 2) +} From 0461ee1976f349c512516f69112cf7ca550efd56 Mon Sep 17 00:00:00 2001 From: glorv Date: Wed, 9 Sep 2020 16:36:05 +0800 Subject: [PATCH 3/3] add license header --- pkg/storage/local_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/storage/local_test.go b/pkg/storage/local_test.go index 530269d3e..f608b5dfa 100644 --- a/pkg/storage/local_test.go +++ b/pkg/storage/local_test.go @@ -1,3 +1,5 @@ +// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. + package storage import (