From da7e9928e44a4d6b82ef686e03e82b2f1d23bc06 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Thu, 16 Jan 2020 15:51:02 -0800 Subject: [PATCH 1/7] Fix Symlinks not being copies across stages --- .../dockerfiles/Dockerfile_test_copy_symlink | 5 ++ pkg/commands/copy.go | 2 +- pkg/executor/build.go | 25 +++++-- pkg/snapshot/snapshot.go | 22 ++++-- pkg/util/fs_util.go | 70 +++++++++++++++---- 5 files changed, 97 insertions(+), 27 deletions(-) create mode 100644 integration/dockerfiles/Dockerfile_test_copy_symlink diff --git a/integration/dockerfiles/Dockerfile_test_copy_symlink b/integration/dockerfiles/Dockerfile_test_copy_symlink new file mode 100644 index 0000000000..efcff44125 --- /dev/null +++ b/integration/dockerfiles/Dockerfile_test_copy_symlink @@ -0,0 +1,5 @@ +FROM alpine:3.11 as t +RUN apk add gcc + +FROM scratch +COPY --from=t /usr/lib/libstdc++.so.6 /usr/lib/ \ No newline at end of file diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go index bac74584fa..2056b106cd 100644 --- a/pkg/commands/copy.go +++ b/pkg/commands/copy.go @@ -85,7 +85,7 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu return err } c.snapshotFiles = append(c.snapshotFiles, copiedFiles...) - } else if fi.Mode()&os.ModeSymlink != 0 { + } else if util.IsSymlink(fi) { // If file is a symlink, we want to create the same relative symlink exclude, err := util.CopySymlink(fullPath, destPath, c.buildcontext) if err != nil { diff --git a/pkg/executor/build.go b/pkg/executor/build.go index d1c17cf1d2..d55c0a875a 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -23,8 +23,6 @@ import ( "strconv" "time" - otiai10Cpy "github.com/otiai10/copy" - "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/moby/buildkit/frontend/dockerfile/instructions" @@ -565,6 +563,7 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { } } + // filesToSave, err := filesToSave(crossStageDependencies[index]) if err != nil { return nil, err @@ -574,8 +573,8 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { return nil, err } for _, p := range filesToSave { - logrus.Infof("Saving file %s for later use.", p) - otiai10Cpy.Copy(p, filepath.Join(dstDir, p)) + logrus.Infof("Saving file %s for later use", p) + util.CopyFileOrSymlink(p, dstDir) } // Delete the filesystem @@ -587,16 +586,28 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { return nil, err } +// fileToSave returns all the files matching the given pattern in deps. +// If a file is a symlink, it also returns the target file. +// It first returns all the target files and then symlinks so they can copied +// in that order to the kaniko workspace. func filesToSave(deps []string) ([]string, error) { - allFiles := []string{} + srcFiles := []string{} + symLinks := []string{} for _, src := range deps { srcs, err := filepath.Glob(src) if err != nil { return nil, err } - allFiles = append(allFiles, srcs...) + for _, f := range srcs { + if link, err := util.CanonicalizeLink(f); err == nil { + symLinks = append(symLinks, f) + srcFiles = append(srcFiles, link) + } else { + srcFiles = append(srcFiles, f) + } + } } - return allFiles, nil + return append(srcFiles, symLinks...), nil } func fetchExtraStages(stages []config.KanikoStage, opts *config.KanikoOptions) error { diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 067e347f0f..8029f9ae28 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -112,7 +112,6 @@ func (s *Snapshotter) TakeSnapshotFS() (string, error) { if err := writeToTar(t, filesToAdd, filesToWhiteOut); err != nil { return "", err } - return f.Name(), nil } @@ -170,7 +169,7 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { filesToAdd := []string{} for path := range memFs { if util.CheckWhitelist(path) { - logrus.Tracef("Not adding %s to layer, as it's whitelisted", path) + logrus.Infof("Not adding %s to layer, as it's whitelisted", path) continue } // Only add changed files. @@ -179,8 +178,12 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { return nil, nil, fmt.Errorf("could not check if file has changed %s %s", path, err) } if fileChanged { - logrus.Tracef("Adding %s to layer, because it was changed.", path) - filesToAdd = append(filesToAdd, path) + files, err := filesWithLinks(path) + if err != nil { + return nil, nil, err + } + logrus.Debug("Adding files %s to layer, because it was changed.", files) + filesToAdd = append(filesToAdd, files...) } } @@ -188,7 +191,6 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { filesToAdd = filesWithParentDirs(filesToAdd) sort.Strings(filesToAdd) - // Add files to the layered map for _, file := range filesToAdd { if err := s.l.Add(file); err != nil { @@ -236,3 +238,13 @@ func filesWithParentDirs(files []string) []string { return newFiles } + +func filesWithLinks(path string) ([]string, error) { + link, err := util.CanonicalizeLink(path) + if err == util.NotSymLink { + return []string{path}, nil + } else if err != nil { + return nil, err + } + return []string{path, link}, nil +} diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index a5decd5300..45bf611508 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -30,6 +30,8 @@ import ( "syscall" "time" + otiai10Cpy "github.com/otiai10/copy" + "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/docker/docker/builder/dockerignore" "github.com/docker/docker/pkg/fileutils" @@ -443,18 +445,8 @@ func FilepathExists(path string) bool { // CreateFile creates a file at path and copies over contents from the reader func CreateFile(path string, reader io.Reader, perm os.FileMode, uid uint32, gid uint32) error { // Create directory path if it doesn't exist - baseDir := filepath.Dir(path) - if info, err := os.Lstat(baseDir); os.IsNotExist(err) { - logrus.Tracef("baseDir %s for file %s does not exist. Creating.", baseDir, path) - if err := os.MkdirAll(baseDir, 0755); err != nil { - return err - } - } else { - switch mode := info.Mode(); { - case mode&os.ModeSymlink != 0: - logrus.Infof("destination cannot be a symlink %v", baseDir) - return errors.New("destination cannot be a symlink") - } + if err := createParentDirectory(path); err != nil { + return err } dest, err := os.Create(path) if err != nil { @@ -531,7 +523,7 @@ func CopyDir(src, dest, buildcontext string) ([]string, error) { if err := mkdirAllWithPermissions(destPath, mode, uid, gid); err != nil { return nil, err } - } else if fi.Mode()&os.ModeSymlink != 0 { + } else if IsSymlink(fi) { // If file is a symlink, we want to create the same relative symlink if _, err := CopySymlink(fullPath, destPath, buildcontext); err != nil { return nil, err @@ -553,7 +545,7 @@ func CopySymlink(src, dest, buildcontext string) (bool, error) { logrus.Debugf("%s found in .dockerignore, ignoring", src) return true, nil } - link, err := os.Readlink(src) + link, err := filepath.EvalSymlinks(src) if err != nil { return false, err } @@ -562,6 +554,9 @@ func CopySymlink(src, dest, buildcontext string) (bool, error) { return false, err } } + if err := createParentDirectory(dest); err != nil { + return false, err + } return false, os.Symlink(link, dest) } @@ -690,3 +685,50 @@ func CreateTargetTarfile(tarpath string) (*os.File, error) { return os.Create(tarpath) } + +// Returns true if a file is a symlink +func IsSymlink(fi os.FileInfo) bool { + return fi.Mode()&os.ModeSymlink != 0 +} + +var NotSymLink = fmt.Errorf("not a symlink") + +func CanonicalizeLink(path string) (string, error) { + fi, err := os.Lstat(path) + if err != nil { + return "", err + } + if !IsSymlink(fi) { + return "", NotSymLink + } + return filepath.EvalSymlinks(path) +} + +// otiai10Cpy.Copy in case the src file is a symlink, will copy the target +// file at destination instead of creating a symlink. See #915 for more details. +func CopyFileOrSymlink(src string, destDir string) error { + destFile := filepath.Join(destDir, src) + if fi, _ := os.Lstat(src); IsSymlink(fi) { + if link, err := os.Readlink(src); err != nil { + return err + } else { + return os.Symlink(link, destFile) + } + } else { + return otiai10Cpy.Copy(src, destFile) + } +} + +func createParentDirectory(path string) error { + baseDir := filepath.Dir(path) + if info, err := os.Lstat(baseDir); os.IsNotExist(err) { + logrus.Tracef("baseDir %s for file %s does not exist. Creating.", baseDir, path) + if err := os.MkdirAll(baseDir, 0755); err != nil { + return err + } + } else if IsSymlink(info) { + logrus.Infof("destination cannot be a symlink %v", baseDir) + return errors.New("destination cannot be a symlink") + } + return nil +} From f1f7297478c7f064840d6c96a06570abdf2a708f Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Thu, 16 Jan 2020 16:50:41 -0800 Subject: [PATCH 2/7] fix tests --- pkg/snapshot/snapshot.go | 4 ++-- pkg/util/fs_util.go | 13 ++++++------- pkg/util/fs_util_test.go | 3 +++ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 8029f9ae28..fe2eead8f9 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -182,7 +182,7 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { if err != nil { return nil, nil, err } - logrus.Debug("Adding files %s to layer, because it was changed.", files) + logrus.Debugf("Adding files %s to layer, because it was changed.", files) filesToAdd = append(filesToAdd, files...) } } @@ -241,7 +241,7 @@ func filesWithParentDirs(files []string) []string { func filesWithLinks(path string) ([]string, error) { link, err := util.CanonicalizeLink(path) - if err == util.NotSymLink { + if err == util.ErrNotSymLink { return []string{path}, nil } else if err != nil { return nil, err diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 45bf611508..b2c8e2ca00 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -691,7 +691,7 @@ func IsSymlink(fi os.FileInfo) bool { return fi.Mode()&os.ModeSymlink != 0 } -var NotSymLink = fmt.Errorf("not a symlink") +var ErrNotSymLink = fmt.Errorf("not a symlink") func CanonicalizeLink(path string) (string, error) { fi, err := os.Lstat(path) @@ -699,7 +699,7 @@ func CanonicalizeLink(path string) (string, error) { return "", err } if !IsSymlink(fi) { - return "", NotSymLink + return "", ErrNotSymLink } return filepath.EvalSymlinks(path) } @@ -709,14 +709,13 @@ func CanonicalizeLink(path string) (string, error) { func CopyFileOrSymlink(src string, destDir string) error { destFile := filepath.Join(destDir, src) if fi, _ := os.Lstat(src); IsSymlink(fi) { - if link, err := os.Readlink(src); err != nil { + link, err := os.Readlink(src) + if err != nil { return err - } else { - return os.Symlink(link, destFile) } - } else { - return otiai10Cpy.Copy(src, destFile) + return os.Symlink(link, destFile) } + return otiai10Cpy.Copy(src, destFile) } func createParentDirectory(path string) error { diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index e6b97063a0..6dfec95801 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -801,6 +801,9 @@ func TestCopySymlink(t *testing.T) { tc := tc t.Parallel() r, err := ioutil.TempDir("", "") + os.MkdirAll(filepath.Join(r, filepath.Dir(tc.linkTarget)), 0777) + tc.linkTarget = filepath.Join(r, tc.linkTarget) + ioutil.WriteFile(tc.linkTarget, nil, 0644) if err != nil { t.Fatal(err) } From 478205e5cad31c0983a9e6a721ef7c136c0a5096 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Wed, 22 Jan 2020 15:27:01 -0800 Subject: [PATCH 3/7] fix adding symlinks to FS which do not exists --- pkg/executor/build.go | 3 +-- pkg/snapshot/snapshot.go | 11 +++++++++-- pkg/util/fs_util.go | 25 ++++++++++++++++++++----- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/pkg/executor/build.go b/pkg/executor/build.go index d55c0a875a..cef20d233d 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -563,7 +563,6 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { } } - // filesToSave, err := filesToSave(crossStageDependencies[index]) if err != nil { return nil, err @@ -599,7 +598,7 @@ func filesToSave(deps []string) ([]string, error) { return nil, err } for _, f := range srcs { - if link, err := util.CanonicalizeLink(f); err == nil { + if link, err := util.EvalSymLink(f); err == nil { symLinks = append(symLinks, f) srcFiles = append(srcFiles, link) } else { diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index fe2eead8f9..e26413cc4b 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -19,6 +19,7 @@ package snapshot import ( "fmt" "io/ioutil" + "os" "path/filepath" "sort" "syscall" @@ -197,7 +198,6 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { return nil, nil, fmt.Errorf("unable to add file %s to layered map: %s", file, err) } } - return filesToAdd, filesToWhiteOut, nil } @@ -240,11 +240,18 @@ func filesWithParentDirs(files []string) []string { } func filesWithLinks(path string) ([]string, error) { - link, err := util.CanonicalizeLink(path) + link, err := util.GetSymLink(path) if err == util.ErrNotSymLink { return []string{path}, nil } else if err != nil { return nil, err } + // Add symlink if it exists in the FS + if !filepath.IsAbs(link) { + link = filepath.Join(filepath.Dir(path), link) + } + if _, err := os.Stat(link); err != nil { + return []string{path}, nil + } return []string{path, link}, nil } diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index b2c8e2ca00..2b5797bc15 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -506,6 +506,7 @@ func CopyDir(src, dest, buildcontext string) ([]string, error) { fullPath := filepath.Join(src, file) fi, err := os.Lstat(fullPath) if err != nil { + fmt.Println(" i am returning from here this", err) return nil, err } if excludeFile(fullPath, buildcontext) { @@ -545,7 +546,7 @@ func CopySymlink(src, dest, buildcontext string) (bool, error) { logrus.Debugf("%s found in .dockerignore, ignoring", src) return true, nil } - link, err := filepath.EvalSymlinks(src) + link, err := os.Readlink(src) if err != nil { return false, err } @@ -693,15 +694,29 @@ func IsSymlink(fi os.FileInfo) bool { var ErrNotSymLink = fmt.Errorf("not a symlink") -func CanonicalizeLink(path string) (string, error) { +func GetSymLink(path string) (string, error) { + if err := getSymlink(path); err != nil { + return "", err + } + return os.Readlink(path) +} + +func EvalSymLink(path string) (string, error) { + if err := getSymlink(path); err != nil { + return "", err + } + return filepath.EvalSymlinks(path) +} + +func getSymlink(path string) error { fi, err := os.Lstat(path) if err != nil { - return "", err + return err } if !IsSymlink(fi) { - return "", ErrNotSymLink + return ErrNotSymLink } - return filepath.EvalSymlinks(path) + return nil } // otiai10Cpy.Copy in case the src file is a symlink, will copy the target From b271eb8ab9560bf2049538619dcb0c16ab55c4a9 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Wed, 22 Jan 2020 15:39:03 -0800 Subject: [PATCH 4/7] fix linter --- pkg/util/fs_util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 2b5797bc15..0a75b08d21 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -695,8 +695,8 @@ func IsSymlink(fi os.FileInfo) bool { var ErrNotSymLink = fmt.Errorf("not a symlink") func GetSymLink(path string) (string, error) { - if err := getSymlink(path); err != nil { - return "", err + if err := getSymlink(path); err != nil { + return "", err } return os.Readlink(path) } From bb129e9c88b306692f5046edf6ce6219f0de61b3 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Wed, 22 Jan 2020 16:27:06 -0800 Subject: [PATCH 5/7] code review comments --- pkg/snapshot/snapshot.go | 4 ++-- pkg/util/fs_util.go | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index e26413cc4b..2a24aa1fb6 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -170,7 +170,7 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { filesToAdd := []string{} for path := range memFs { if util.CheckWhitelist(path) { - logrus.Infof("Not adding %s to layer, as it's whitelisted", path) + logrus.Tracef("Not adding %s to layer, as it's whitelisted", path) continue } // Only add changed files. @@ -183,7 +183,7 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { if err != nil { return nil, nil, err } - logrus.Debugf("Adding files %s to layer, because it was changed.", files) + logrus.Tracef("Adding files %s to layer, because it was changed.", files) filesToAdd = append(filesToAdd, files...) } } diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 0a75b08d21..0a0a4aebe4 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -719,8 +719,9 @@ func getSymlink(path string) error { return nil } -// otiai10Cpy.Copy in case the src file is a symlink, will copy the target -// file at destination instead of creating a symlink. See #915 for more details. +// For cross stage dependencies kaniko must persist the referenced path so that it can be used in +// the dependent stage. For symlinks we copy the target path because copying the symlink would +// result in a dead link func CopyFileOrSymlink(src string, destDir string) error { destFile := filepath.Join(destDir, src) if fi, _ := os.Lstat(src); IsSymlink(fi) { From 79d226ade0b5b758bb6f3bbf17a910532d6d3642 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Wed, 22 Jan 2020 16:31:45 -0800 Subject: [PATCH 6/7] refactor --- pkg/executor/build.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pkg/executor/build.go b/pkg/executor/build.go index cef20d233d..9d0a496b9a 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -587,11 +587,8 @@ func DoBuild(opts *config.KanikoOptions) (v1.Image, error) { // fileToSave returns all the files matching the given pattern in deps. // If a file is a symlink, it also returns the target file. -// It first returns all the target files and then symlinks so they can copied -// in that order to the kaniko workspace. func filesToSave(deps []string) ([]string, error) { srcFiles := []string{} - symLinks := []string{} for _, src := range deps { srcs, err := filepath.Glob(src) if err != nil { @@ -599,14 +596,12 @@ func filesToSave(deps []string) ([]string, error) { } for _, f := range srcs { if link, err := util.EvalSymLink(f); err == nil { - symLinks = append(symLinks, f) srcFiles = append(srcFiles, link) - } else { - srcFiles = append(srcFiles, f) } + srcFiles = append(srcFiles, f) } } - return append(srcFiles, symLinks...), nil + return srcFiles, nil } func fetchExtraStages(stages []config.KanikoStage, opts *config.KanikoOptions) error { From 3e5d0a6334f0047597958cc2409ae1c7092421e9 Mon Sep 17 00:00:00 2001 From: Tejal Desai Date: Thu, 23 Jan 2020 11:10:46 -0800 Subject: [PATCH 7/7] add unit tests --- .../dockerfiles/Dockerfile_test_copy_symlink | 7 +- pkg/snapshot/snapshot.go | 2 + pkg/snapshot/snapshot_test.go | 105 +++++++++++++++--- 3 files changed, 96 insertions(+), 18 deletions(-) diff --git a/integration/dockerfiles/Dockerfile_test_copy_symlink b/integration/dockerfiles/Dockerfile_test_copy_symlink index efcff44125..e517d8ce7a 100644 --- a/integration/dockerfiles/Dockerfile_test_copy_symlink +++ b/integration/dockerfiles/Dockerfile_test_copy_symlink @@ -1,5 +1,6 @@ -FROM alpine:3.11 as t -RUN apk add gcc +FROM busybox as t +RUN echo "hello" > /tmp/target +RUN ln -s /tmp/target /tmp/link FROM scratch -COPY --from=t /usr/lib/libstdc++.so.6 /usr/lib/ \ No newline at end of file +COPY --from=t /tmp/link /tmp \ No newline at end of file diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 2a24aa1fb6..3e37d5a611 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -179,6 +179,7 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) { return nil, nil, fmt.Errorf("could not check if file has changed %s %s", path, err) } if fileChanged { + // Get target file for symlinks so the symlink is not a dead link. files, err := filesWithLinks(path) if err != nil { return nil, nil, err @@ -239,6 +240,7 @@ func filesWithParentDirs(files []string) []string { return newFiles } +// filesWithLinks returns the symlink and the target path if its exists. func filesWithLinks(path string) ([]string, error) { link, err := util.GetSymLink(path) if err == util.ErrNotSymLink { diff --git a/pkg/snapshot/snapshot_test.go b/pkg/snapshot/snapshot_test.go index 798ae6c09e..8e3f16598e 100644 --- a/pkg/snapshot/snapshot_test.go +++ b/pkg/snapshot/snapshot_test.go @@ -31,7 +31,7 @@ import ( ) func TestSnapshotFSFileChange(t *testing.T) { - testDir, snapshotter, cleanup, err := setUpTestDir() + testDir, snapshotter, cleanup, err := setUpTest() testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/") defer cleanup() if err != nil { @@ -90,7 +90,7 @@ func TestSnapshotFSFileChange(t *testing.T) { } func TestSnapshotFSIsReproducible(t *testing.T) { - testDir, snapshotter, cleanup, err := setUpTestDir() + testDir, snapshotter, cleanup, err := setUpTest() defer cleanup() if err != nil { t.Fatal(err) @@ -129,7 +129,7 @@ func TestSnapshotFSIsReproducible(t *testing.T) { } func TestSnapshotFSChangePermissions(t *testing.T) { - testDir, snapshotter, cleanup, err := setUpTestDir() + testDir, snapshotter, cleanup, err := setUpTest() testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/") defer cleanup() if err != nil { @@ -180,7 +180,7 @@ func TestSnapshotFSChangePermissions(t *testing.T) { } func TestSnapshotFiles(t *testing.T) { - testDir, snapshotter, cleanup, err := setUpTestDir() + testDir, snapshotter, cleanup, err := setUpTest() testDirWithoutLeadingSlash := strings.TrimLeft(testDir, "/") defer cleanup() if err != nil { @@ -230,7 +230,7 @@ func TestSnapshotFiles(t *testing.T) { } func TestEmptySnapshotFS(t *testing.T) { - _, snapshotter, cleanup, err := setUpTestDir() + _, snapshotter, cleanup, err := setUpTest() if err != nil { t.Fatal(err) } @@ -253,29 +253,104 @@ func TestEmptySnapshotFS(t *testing.T) { } } -func setUpTestDir() (string, *Snapshotter, func(), error) { - testDir, err := ioutil.TempDir("", "") - if err != nil { - return "", nil, nil, errors.Wrap(err, "setting up temp dir") +func TestFileWithLinks(t *testing.T) { + + link := "baz/link" + tcs := []struct { + name string + path string + linkFileTarget string + expected []string + shouldErr bool + }{ + { + name: "given path is a symlink that points to a valid target", + path: link, + linkFileTarget: "file", + expected: []string{link, "baz/file"}, + }, + { + name: "given path is a symlink points to non existing path", + path: link, + linkFileTarget: "does-not-exists", + expected: []string{link}, + }, + { + name: "given path is a regular file", + path: "kaniko/file", + linkFileTarget: "file", + expected: []string{"kaniko/file"}, + }, } - snapshotPath, err := ioutil.TempDir("", "") - if err != nil { - return "", nil, nil, errors.Wrap(err, "setting up temp dir") + for _, tt := range tcs { + t.Run(tt.name, func(t *testing.T) { + testDir, cleanup, err := setUpTestDir() + if err != nil { + t.Fatal(err) + } + defer cleanup() + if err := setupSymlink(testDir, link, tt.linkFileTarget); err != nil { + t.Fatalf("could not set up symlink due to %s", err) + } + actual, err := filesWithLinks(filepath.Join(testDir, tt.path)) + if err != nil { + t.Fatalf("unexpected error %s", err) + } + sortAndCompareFilepaths(t, testDir, tt.expected, actual) + }) } +} - snapshotPathPrefix = snapshotPath +func setupSymlink(dir string, link string, target string) error { + return os.Symlink(target, filepath.Join(dir, link)) +} + +func sortAndCompareFilepaths(t *testing.T, testDir string, expected []string, actual []string) { + expectedFullPaths := make([]string, len(expected)) + for i, file := range expected { + expectedFullPaths[i] = filepath.Join(testDir, file) + } + sort.Strings(expectedFullPaths) + sort.Strings(actual) + testutil.CheckDeepEqual(t, expectedFullPaths, actual) +} +func setUpTestDir() (string, func(), error) { + testDir, err := ioutil.TempDir("", "") + if err != nil { + return "", nil, errors.Wrap(err, "setting up temp dir") + } files := map[string]string{ "foo": "baz1", "bar/bat": "baz2", "kaniko/file": "file", + "baz/file": "testfile", } // Set up initial files if err := testutil.SetupFiles(testDir, files); err != nil { - return "", nil, nil, errors.Wrap(err, "setting up file system") + return "", nil, errors.Wrap(err, "setting up file system") } + cleanup := func() { + os.RemoveAll(testDir) + } + + return testDir, cleanup, nil +} + +func setUpTest() (string, *Snapshotter, func(), error) { + testDir, dirCleanUp, err := setUpTestDir() + if err != nil { + return "", nil, nil, err + } + snapshotPath, err := ioutil.TempDir("", "") + if err != nil { + return "", nil, nil, errors.Wrap(err, "setting up temp dir") + } + + snapshotPathPrefix = snapshotPath + // Take the initial snapshot l := NewLayeredMap(util.Hasher(), util.CacheHasher()) snapshotter := NewSnapshotter(l, testDir) @@ -285,7 +360,7 @@ func setUpTestDir() (string, *Snapshotter, func(), error) { cleanup := func() { os.RemoveAll(snapshotPath) - os.RemoveAll(testDir) + dirCleanUp() } return testDir, snapshotter, cleanup, nil