From 39a522682b623905443dc9b62160dddaeac2ebff Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Mon, 28 Nov 2022 21:32:24 +0000 Subject: [PATCH] feat(import): add "copy" support for import Currently, "import:" directive only allows for temporarily mounting previously layers, and actual copying would have to be done via "run:" directive. But in cases where there is no shell in the current layer, there was no way to achieve this, greatly limiting usability for packaging golang binaries. Adding support using "dest:" option under "import:" directive. Thanks @smoser for the review and fix. Signed-off-by: Ramkumar Chinchani --- base.go | 6 ++- build.go | 4 +- cache.go | 13 ++++++ cmd/grab.go | 2 +- cmd/internal_go.go | 30 +++++++++++++ cmd/main.go | 2 +- doc/stacker_yaml.md | 11 +++++ grab.go | 31 ++++++++++++- import.go | 47 +++++++++++++++----- lib/dir.go | 63 ++++++++++++++++++++++++++- lib/file.go | 24 +++++++++-- network.go | 17 +++++++- overlay/overlay-dirs.go | 2 +- test/import.bats | 52 ++++++++++++++++++++++ types/layer.go | 96 +++++++++++++++++++++++++++++++++++------ 15 files changed, 365 insertions(+), 35 deletions(-) diff --git a/base.go b/base.go index 2eb45cb6..8515a1cf 100644 --- a/base.go +++ b/base.go @@ -31,6 +31,8 @@ type BaseLayerOpts struct { func GetBase(o BaseLayerOpts) error { switch o.Layer.From.Type { case types.BuiltLayer: + fallthrough + case types.ScratchLayer: return nil case types.TarLayer: cacheDir := path.Join(o.Config.StackerDir, "layer-bases") @@ -38,7 +40,7 @@ func GetBase(o BaseLayerOpts) error { return err } - _, err := acquireUrl(o.Config, o.Storage, o.Layer.From.Url, cacheDir, o.Progress, "") + _, err := acquireUrl(o.Config, o.Storage, o.Layer.From.Url, cacheDir, o.Progress, "", nil, -1, -1) return err /* now we can do all the containers/image types */ case types.OCILayer: @@ -74,6 +76,8 @@ func SetupRootfs(o BaseLayerOpts) error { } switch o.Layer.From.Type { + case types.ScratchLayer: + return o.Storage.SetupEmptyRootfs(o.Name) case types.TarLayer: err := o.Storage.SetupEmptyRootfs(o.Name) if err != nil { diff --git a/build.go b/build.go index 68f14b79..3fa8f9b3 100644 --- a/build.go +++ b/build.go @@ -352,10 +352,12 @@ func (b *Builder) build(s types.Storage, file string) error { return err } - if err := Import(opts.Config, s, name, l.Imports, opts.Progress); err != nil { + if err := Import(opts.Config, s, name, l.Imports, &l.OverlayDirs, opts.Progress); err != nil { return err } + log.Debugf("overlay-dirs, possibly modified after import: %v", l.OverlayDirs) + // Need to check if the image has bind mounts, if the image has bind mounts, // it needs to be rebuilt regardless of the build cache // The reason is that tracking build cache for bind mounted folders diff --git a/cache.go b/cache.go index c8ef60e7..e7d31233 100644 --- a/cache.go +++ b/cache.go @@ -207,6 +207,11 @@ func (c *BuildCache) Lookup(name string) (*CacheEntry, bool, error) { } for _, imp := range l.Imports { + if imp.Dest != "" { + // ignore imports which are copied + continue + } + cachedImport, ok := result.Imports[imp.Path] if !ok { log.Infof("cache miss because of new import: %s", imp.Path) @@ -401,6 +406,9 @@ func (c *BuildCache) getBaseHash(name string) (string, error) { } return fmt.Sprintf("%d", baseHash), nil + case types.ScratchLayer: + // no base hash to use + return "", nil case types.TarLayer: // use the hash of the input tarball cacheDir := path.Join(c.config.StackerDir, "layer-bases") @@ -457,6 +465,11 @@ func (c *BuildCache) Put(name string, manifests map[types.LayerType]ispec.Descri } for _, imp := range l.Imports { + if imp.Dest != "" { + // ignore imports which are copied + continue + } + fname := path.Base(imp.Path) importsDir := path.Join(c.config.StackerDir, "imports") diskPath := path.Join(importsDir, name, fname) diff --git a/cmd/grab.go b/cmd/grab.go index d416e44a..44b9ddf9 100644 --- a/cmd/grab.go +++ b/cmd/grab.go @@ -43,5 +43,5 @@ func doGrab(ctx *cli.Context) error { return err } - return stacker.Grab(config, s, name, parts[1], cwd) + return stacker.Grab(config, s, name, parts[1], cwd, nil, -1, -1) } diff --git a/cmd/internal_go.go b/cmd/internal_go.go index 74f8ee36..a3c9b457 100644 --- a/cmd/internal_go.go +++ b/cmd/internal_go.go @@ -24,6 +24,14 @@ var internalGoCmd = cli.Command{ Name: "cp", Action: doCP, }, + cli.Command{ + Name: "chmod", + Action: doChmod, + }, + cli.Command{ + Name: "chown", + Action: doChown, + }, cli.Command{ Name: "check-aa-profile", Action: doCheckAAProfile, @@ -109,6 +117,28 @@ func doCP(ctx *cli.Context) error { ) } +func doChmod(ctx *cli.Context) error { + if len(ctx.Args()) != 2 { + return errors.Errorf("wrong number of args") + } + + return lib.Chmod( + ctx.Args()[0], + ctx.Args()[1], + ) +} + +func doChown(ctx *cli.Context) error { + if len(ctx.Args()) != 2 { + return errors.Errorf("wrong number of args") + } + + return lib.Chown( + ctx.Args()[0], + ctx.Args()[1], + ) +} + func doCheckAAProfile(ctx *cli.Context) error { toCheck := ctx.Args()[0] command := fmt.Sprintf("changeprofile %s", toCheck) diff --git a/cmd/main.go b/cmd/main.go index 1ad21a70..0a77066f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -72,7 +72,7 @@ func shouldSkipInternalUserns(ctx *cli.Context) bool { } if len(args) >= 2 && args[0] == "internal-go" { - if args[1] == "atomfs" || args[1] == "cp" { + if args[1] == "atomfs" || args[1] == "cp" || args[1] == "chown" || args[1] == "chmod" { return true } } diff --git a/doc/stacker_yaml.md b/doc/stacker_yaml.md index 89217abd..4c913a11 100644 --- a/doc/stacker_yaml.md +++ b/doc/stacker_yaml.md @@ -110,6 +110,17 @@ import: - /path/to/file ``` +#### `import dest` + +The `import` directive also supports specifying the destination path (specified +by `dest`) in the resulting container image, where the source file (specified +by `path`) will be copyied to, for example: +``` +import: + - path: config.json + dest: / +``` + ### `overlay_dirs` This directive works only with OverlayFS backend storage. diff --git a/grab.go b/grab.go index 5a44dd15..b7fc2956 100644 --- a/grab.go +++ b/grab.go @@ -2,6 +2,7 @@ package stacker import ( "fmt" + "io/fs" "os" "path" @@ -10,7 +11,9 @@ import ( "stackerbuild.io/stacker/types" ) -func Grab(sc types.StackerConfig, storage types.Storage, name string, source string, targetDir string) error { +func Grab(sc types.StackerConfig, storage types.Storage, name string, source string, targetDir string, + mode *fs.FileMode, uid, gid int, +) error { c, err := container.New(sc, name) if err != nil { return err @@ -38,5 +41,29 @@ func Grab(sc types.StackerConfig, storage types.Storage, name string, source str return err } - return c.Execute(fmt.Sprintf("/static-stacker internal-go cp %s /stacker/%s", source, path.Base(source)), nil) + err = c.Execute(fmt.Sprintf("/static-stacker internal-go cp %s /stacker/%s", source, path.Base(source)), nil) + if err != nil { + return err + } + + if mode != nil { + err = c.Execute(fmt.Sprintf("/static-stacker internal-go chmod %s /stacker/%s", fmt.Sprintf("%o", *mode), path.Base(source)), nil) + if err != nil { + return err + } + } + + if uid > 0 { + owns := fmt.Sprintf("%d", uid) + if gid > 0 { + owns += fmt.Sprintf(":%d", gid) + } + + err = c.Execute(fmt.Sprintf("/static-stacker internal-go chown %s /stacker/%s", owns, path.Base(source)), nil) + if err != nil { + return err + } + } + + return nil } diff --git a/import.go b/import.go index 9c25da21..326fe98e 100644 --- a/import.go +++ b/import.go @@ -1,6 +1,7 @@ package stacker import ( + "io/fs" "os" "path" "strings" @@ -84,7 +85,7 @@ func verifyImportFileHash(imp string, hash string) error { return nil } -func importFile(imp string, cacheDir string, hash string) (string, error) { +func importFile(imp string, cacheDir string, hash string, mode *fs.FileMode, uid, gid int) (string, error) { e1, err := os.Lstat(imp) if err != nil { return "", errors.Wrapf(err, "couldn't stat import %s", imp) @@ -110,7 +111,7 @@ func importFile(imp string, cacheDir string, hash string) (string, error) { if needsCopy { log.Infof("copying %s", imp) - if err := lib.FileCopy(dest, imp); err != nil { + if err := lib.FileCopy(dest, imp, mode, uid, gid); err != nil { return "", errors.Wrapf(err, "couldn't copy import %s", imp) } } else { @@ -187,7 +188,7 @@ func importFile(imp string, cacheDir string, hash string) (string, error) { if err != nil { return "", err } - err = lib.FileCopy(destpath, srcpath) + err = lib.FileCopy(destpath, srcpath, mode, uid, gid) } if err != nil { return "", err @@ -213,7 +214,9 @@ func validateHash(hash string) error { return nil } -func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache string, progress bool, expectedHash string) (string, error) { +func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache string, progress bool, expectedHash string, + mode *fs.FileMode, uid, gid int, +) (string, error) { url, err := types.NewDockerishUrl(i) if err != nil { return "", err @@ -226,7 +229,7 @@ func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache st // It's just a path, let's copy it to .stacker. if url.Scheme == "" { - return importFile(i, cache, expectedHash) + return importFile(i, cache, expectedHash, mode, uid, gid) } else if url.Scheme == "http" || url.Scheme == "https" { // otherwise, we need to download it // first verify the hashes @@ -242,7 +245,7 @@ func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache st return "", errors.Errorf("The requested hash of %s import is different than the actual hash: %s != %s", i, expectedHash, remoteHash) } - return Download(cache, i, progress, expectedHash, remoteHash, remoteSize) + return Download(cache, i, progress, expectedHash, remoteHash, remoteSize, mode, uid, gid) } else if url.Scheme == "stacker" { // we always Grab() things from stacker://, because we need to // mount the container's rootfs to get them and don't @@ -254,7 +257,7 @@ func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache st return "", err } defer cleanup() - err = Grab(c, storage, snap, url.Path, cache) + err = Grab(c, storage, snap, url.Path, cache, mode, uid, gid) if err != nil { return "", err } @@ -271,7 +274,11 @@ func acquireUrl(c types.StackerConfig, storage types.Storage, i string, cache st } func CleanImportsDir(c types.StackerConfig, name string, imports types.Imports, cache *BuildCache) error { - dir := path.Join(c.StackerDir, "imports", name) + // remove all copied imports + dir := path.Join(c.StackerDir, "imports-copy", name) + _ = os.RemoveAll(dir) + + dir = path.Join(c.StackerDir, "imports", name) cacheEntry, cacheHit := cache.Cache[name] if !cacheHit { @@ -298,20 +305,40 @@ func CleanImportsDir(c types.StackerConfig, name string, imports types.Imports, return nil } -func Import(c types.StackerConfig, storage types.Storage, name string, imports types.Imports, progress bool) error { +// Import files from different sources to an ephemeral or permanent destination. +func Import(c types.StackerConfig, storage types.Storage, name string, imports types.Imports, overlayDirs *types.OverlayDirs, progress bool) error { dir := path.Join(c.StackerDir, "imports", name) if err := os.MkdirAll(dir, 0755); err != nil { return err } + cpdir := path.Join(c.StackerDir, "imports-copy", name) + + if err := os.MkdirAll(cpdir, 0755); err != nil { + return err + } + existing, err := os.ReadDir(dir) if err != nil { return errors.Wrapf(err, "couldn't read existing directory") } for _, i := range imports { - name, err := acquireUrl(c, storage, i.Path, dir, progress, i.Hash) + cache := dir + if i.Dest != "" { + tmpdir, err := os.MkdirTemp(cpdir, "") + if err != nil { + return errors.Wrapf(err, "couldn't create temp import copy directory") + } + + ovl := types.OverlayDir{Source: tmpdir, Dest: "/"} + *overlayDirs = append(*overlayDirs, ovl) + + cache = tmpdir + } + + name, err := acquireUrl(c, storage, i.Path, cache, progress, i.Hash, i.Mode, i.Uid, i.Gid) if err != nil { return err } diff --git a/lib/dir.go b/lib/dir.go index 95295e39..4b422d50 100644 --- a/lib/dir.go +++ b/lib/dir.go @@ -1,8 +1,11 @@ package lib import ( + "io/fs" "os" "path" + "strconv" + "strings" "github.com/pkg/errors" ) @@ -62,7 +65,7 @@ func DirCopy(dest string, source string) error { return err } } else { - if err = FileCopy(dstfp, srcfp); err != nil { + if err = FileCopy(dstfp, srcfp, nil, -1, -1); err != nil { return err } } @@ -80,6 +83,62 @@ func CopyThing(srcpath, destpath string) error { if srcInfo.IsDir() { return DirCopy(destpath, srcpath) } else { - return FileCopy(destpath, srcpath) + return FileCopy(destpath, srcpath, nil, -1, -1) } } + +// Chmod changes file permissions +func Chmod(mode, destpath string) error { + destInfo, err := os.Lstat(destpath) + if err != nil { + return errors.WithStack(err) + } + + if destInfo.IsDir() { + return errors.WithStack(os.ErrInvalid) + } + + if destInfo.Mode()&os.ModeSymlink != 0 { + return errors.WithStack(os.ErrInvalid) + } + + // read as an octal value + iperms, err := strconv.ParseUint(mode, 8, 32) + if err != nil { + return errors.WithStack(err) + } + + return os.Chmod(destpath, fs.FileMode(iperms)) +} + +// Chown changes file ownership +func Chown(owner, destpath string) error { + destInfo, err := os.Lstat(destpath) + if err != nil { + return errors.WithStack(err) + } + + if destInfo.IsDir() { + return errors.WithStack(os.ErrInvalid) + } + + owns := strings.Split(owner, ":") + if len(owns) > 2 { + return errors.WithStack(os.ErrInvalid) + } + + uid, err := strconv.ParseInt(owns[0], 10, 32) + if err != nil { + return errors.WithStack(err) + } + + gid := int64(-1) + if len(owns) > 1 { + gid, err = strconv.ParseInt(owns[1], 10, 32) + if err != nil { + return errors.WithStack(err) + } + } + + return os.Lchown(destpath, int(uid), int(gid)) +} diff --git a/lib/file.go b/lib/file.go index 761a1ade..b03333d0 100644 --- a/lib/file.go +++ b/lib/file.go @@ -2,6 +2,7 @@ package lib import ( "io" + "io/fs" "os" "path/filepath" "regexp" @@ -9,7 +10,7 @@ import ( "github.com/pkg/errors" ) -func FileCopy(dest string, source string) error { +func FileCopy(dest string, source string, mode *fs.FileMode, uid, gid int) error { os.RemoveAll(dest) linkFI, err := os.Lstat(source) @@ -24,7 +25,15 @@ func FileCopy(dest string, source string) error { return errors.Wrapf(err, "Coudn't read link %s", source) } - return os.Symlink(target, dest) + if err = os.Symlink(target, dest); err != nil { + return errors.Wrapf(err, "Couldn't symlink %s->%s", source, target) + } + + if err = os.Lchown(dest, uid, gid); err != nil { + return errors.Wrapf(err, "Couldn't set symlink ownership %s", dest) + } + + return nil } s, err := os.Open(source) @@ -44,11 +53,20 @@ func FileCopy(dest string, source string) error { } defer d.Close() - err = d.Chmod(fi.Mode()) + if mode != nil { + err = d.Chmod(*mode) + } else { + err = d.Chmod(fi.Mode()) + } if err != nil { return errors.Wrapf(err, "Coudn't chmod file %s", source) } + err = d.Chown(uid, gid) + if err != nil { + return errors.Wrapf(err, "Coudn't chown file %s", source) + } + _, err = io.Copy(d, s) return err } diff --git a/network.go b/network.go index 8bbc0765..b60a039b 100644 --- a/network.go +++ b/network.go @@ -2,6 +2,7 @@ package stacker import ( "io" + "io/fs" "net/http" "net/url" "os" @@ -16,7 +17,9 @@ import ( ) // download with caching support in the specified cache dir. -func Download(cacheDir string, url string, progress bool, expectedHash, remoteHash, remoteSize string) (string, error) { +func Download(cacheDir string, url string, progress bool, expectedHash, remoteHash, remoteSize string, + mode *fs.FileMode, uid, gid int, +) (string, error) { name := path.Join(cacheDir, path.Base(url)) if fi, err := os.Stat(name); err == nil { @@ -107,6 +110,18 @@ func Download(cacheDir string, url string, progress bool, expectedHash, remoteHa } } + if mode != nil { + err = out.Chmod(*mode) + if err != nil { + return "", errors.Wrapf(err, "Coudn't chmod file %s", name) + } + } + + err = out.Chown(uid, gid) + if err != nil { + return "", errors.Wrapf(err, "Coudn't chown file %s", source) + } + return name, err } diff --git a/overlay/overlay-dirs.go b/overlay/overlay-dirs.go index 171ff187..30524228 100644 --- a/overlay/overlay-dirs.go +++ b/overlay/overlay-dirs.go @@ -61,7 +61,7 @@ func generateOverlayDirLayer(name string, layerType types.LayerType, overlayDir } err = os.Symlink(contents, overlayPath(config, desc.Digest, "overlay")) - if err != nil { + if err != nil && !errors.Is(err, os.ErrExist) { return ispec.Descriptor{}, errors.Wrapf(err, "failed to create symlink") } diff --git a/test/import.bats b/test/import.bats index 034d4ae0..66284fe9 100644 --- a/test/import.bats +++ b/test/import.bats @@ -297,3 +297,55 @@ EOF stacker build } + +@test "different import types with perms" { + touch test_file + test_file_sha=$(sha test_file) || { stderr "failed sha $test_file"; return 1; } + touch test_file2 + cat > stacker.yaml <