From f7411d26cdc3068922eb18cebd78f818679d2ad5 Mon Sep 17 00:00:00 2001 From: jhowardmsft Date: Thu, 23 Apr 2015 15:55:36 -0700 Subject: [PATCH 01/19] Windows: mkdirall volume path aware Signed-off-by: jhowardmsft --- sequential/sequential_windows.go | 64 ++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 sequential/sequential_windows.go diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go new file mode 100644 index 00000000..90b50060 --- /dev/null +++ b/sequential/sequential_windows.go @@ -0,0 +1,64 @@ +// +build windows + +package system + +import ( + "os" + "regexp" + "syscall" +) + +// MkdirAll implementation that is volume path aware for Windows. +func MkdirAll(path string, perm os.FileMode) error { + if re := regexp.MustCompile(`^\\\\\?\\Volume{[a-z0-9-]+}$`); re.MatchString(path) { + return nil + } + + // The rest of this method is copied from os.MkdirAll and should be kept + // as-is to ensure compatibility. + + // Fast path: if we can tell whether path is a directory or file, stop with success or error. + dir, err := os.Stat(path) + if err == nil { + if dir.IsDir() { + return nil + } + return &os.PathError{ + Op: "mkdir", + Path: path, + Err: syscall.ENOTDIR, + } + } + + // Slow path: make sure parent exists and then call Mkdir for path. + i := len(path) + for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator. + i-- + } + + j := i + for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element. + j-- + } + + if j > 1 { + // Create parent + err = MkdirAll(path[0:j-1], perm) + if err != nil { + return err + } + } + + // Parent now exists; invoke Mkdir and use its result. + err = os.Mkdir(path, perm) + if err != nil { + // Handle arguments like "foo/." by + // double-checking that directory doesn't exist. + dir, err1 := os.Lstat(path) + if err1 == nil && dir.IsDir() { + return nil + } + return err + } + return nil +} From 20a92d057d5528926dd5296c068fa513d7033386 Mon Sep 17 00:00:00 2001 From: John Howard Date: Wed, 26 Aug 2015 12:37:01 -0700 Subject: [PATCH 02/19] Windows: Fix use of IsAbs check Signed-off-by: John Howard --- sequential/sequential_windows.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index 90b50060..16823d55 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -4,7 +4,9 @@ package system import ( "os" + "path/filepath" "regexp" + "strings" "syscall" ) @@ -62,3 +64,19 @@ func MkdirAll(path string, perm os.FileMode) error { } return nil } + +// IsAbs is a platform-specific wrapper for filepath.IsAbs. On Windows, +// golang filepath.IsAbs does not consider a path \windows\system32 as absolute +// as it doesn't start with a drive-letter/colon combination. However, in +// docker we need to verify things such as WORKDIR /windows/system32 in +// a Dockerfile (which gets translated to \windows\system32 when being processed +// by the daemon. This SHOULD be treated as absolute from a docker processing +// perspective. +func IsAbs(path string) bool { + if !filepath.IsAbs(path) { + if !strings.HasPrefix(path, string(os.PathSeparator)) { + return false + } + } + return true +} From 1798aa0655d5f3f7d3f7ebf9ee7ac3fb6323ecf3 Mon Sep 17 00:00:00 2001 From: John Howard Date: Fri, 4 Nov 2016 12:42:21 -0700 Subject: [PATCH 03/19] Windows: create daemon root with ACL Signed-off-by: John Howard --- sequential/sequential_windows.go | 61 +++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index 16823d55..97f1c7c8 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -8,15 +8,31 @@ import ( "regexp" "strings" "syscall" + "unsafe" + + winio "github.com/Microsoft/go-winio" ) +// MkdirAllWithACL is a wrapper for MkdirAll that creates a directory +// ACL'd for Builtin Administrators and Local System. +func MkdirAllWithACL(path string, perm os.FileMode) error { + return mkdirall(path, true) +} + // MkdirAll implementation that is volume path aware for Windows. -func MkdirAll(path string, perm os.FileMode) error { +func MkdirAll(path string, _ os.FileMode) error { + return mkdirall(path, false) +} + +// mkdirall is a custom version of os.MkdirAll modified for use on Windows +// so that it is both volume path aware, and can create a directory with +// a DACL. +func mkdirall(path string, adminAndLocalSystem bool) error { if re := regexp.MustCompile(`^\\\\\?\\Volume{[a-z0-9-]+}$`); re.MatchString(path) { return nil } - // The rest of this method is copied from os.MkdirAll and should be kept + // The rest of this method is largely copied from os.MkdirAll and should be kept // as-is to ensure compatibility. // Fast path: if we can tell whether path is a directory or file, stop with success or error. @@ -45,14 +61,19 @@ func MkdirAll(path string, perm os.FileMode) error { if j > 1 { // Create parent - err = MkdirAll(path[0:j-1], perm) + err = mkdirall(path[0:j-1], false) if err != nil { return err } } - // Parent now exists; invoke Mkdir and use its result. - err = os.Mkdir(path, perm) + // Parent now exists; invoke os.Mkdir or mkdirWithACL and use its result. + if adminAndLocalSystem { + err = mkdirWithACL(path) + } else { + err = os.Mkdir(path, 0) + } + if err != nil { // Handle arguments like "foo/." by // double-checking that directory doesn't exist. @@ -65,6 +86,36 @@ func MkdirAll(path string, perm os.FileMode) error { return nil } +// mkdirWithACL creates a new directory. If there is an error, it will be of +// type *PathError. . +// +// This is a modified and combined version of os.Mkdir and syscall.Mkdir +// in golang to cater for creating a directory am ACL permitting full +// access, with inheritance, to any subfolder/file for Built-in Administrators +// and Local System. +func mkdirWithACL(name string) error { + sa := syscall.SecurityAttributes{Length: 0} + sddl := "D:P(A;OICI;GA;;;BA)(A;OICI;GA;;;SY)" + sd, err := winio.SddlToSecurityDescriptor(sddl) + if err != nil { + return &os.PathError{"mkdir", name, err} + } + sa.Length = uint32(unsafe.Sizeof(sa)) + sa.InheritHandle = 1 + sa.SecurityDescriptor = uintptr(unsafe.Pointer(&sd[0])) + + namep, err := syscall.UTF16PtrFromString(name) + if err != nil { + return &os.PathError{"mkdir", name, err} + } + + e := syscall.CreateDirectory(namep, &sa) + if e != nil { + return &os.PathError{"mkdir", name, e} + } + return nil +} + // IsAbs is a platform-specific wrapper for filepath.IsAbs. On Windows, // golang filepath.IsAbs does not consider a path \windows\system32 as absolute // as it doesn't start with a drive-letter/colon combination. However, in From 82d88ed77bf75cdc570e0899976eda1fb80a7df4 Mon Sep 17 00:00:00 2001 From: John Howard Date: Tue, 1 Nov 2016 15:44:06 -0700 Subject: [PATCH 04/19] Windows: Use sequential file access Signed-off-by: John Howard --- sequential/sequential_windows.go | 103 +++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index 97f1c7c8..6fa5090d 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -131,3 +131,106 @@ func IsAbs(path string) bool { } return true } + +// The origin of the functions below here are the golang OS and syscall packages, +// slightly modified to only cope with files, not directories due to the +// specific use case. +// +// The alteration is to allow a file on Windows to be opened with +// FILE_FLAG_SEQUENTIAL_SCAN (particular for docker load), to avoid eating +// the standby list, particularly when accessing large files such as layer.tar. + +// CreateSequential creates the named file with mode 0666 (before umask), truncating +// it if it already exists. If successful, methods on the returned +// File can be used for I/O; the associated file descriptor has mode +// O_RDWR. +// If there is an error, it will be of type *PathError. +func CreateSequential(name string) (*os.File, error) { + return OpenFileSequential(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0) +} + +// OpenSequential opens the named file for reading. If successful, methods on +// the returned file can be used for reading; the associated file +// descriptor has mode O_RDONLY. +// If there is an error, it will be of type *PathError. +func OpenSequential(name string) (*os.File, error) { + return OpenFileSequential(name, os.O_RDONLY, 0) +} + +// OpenFileSequential is the generalized open call; most users will use Open +// or Create instead. +// If there is an error, it will be of type *PathError. +func OpenFileSequential(name string, flag int, _ os.FileMode) (*os.File, error) { + if name == "" { + return nil, &os.PathError{"open", name, syscall.ENOENT} + } + r, errf := syscallOpenFileSequential(name, flag, 0) + if errf == nil { + return r, nil + } + return nil, &os.PathError{"open", name, errf} +} + +func syscallOpenFileSequential(name string, flag int, _ os.FileMode) (file *os.File, err error) { + r, e := syscallOpenSequential(name, flag|syscall.O_CLOEXEC, 0) + if e != nil { + return nil, e + } + return os.NewFile(uintptr(r), name), nil +} + +func makeInheritSa() *syscall.SecurityAttributes { + var sa syscall.SecurityAttributes + sa.Length = uint32(unsafe.Sizeof(sa)) + sa.InheritHandle = 1 + return &sa +} + +func syscallOpenSequential(path string, mode int, _ uint32) (fd syscall.Handle, err error) { + if len(path) == 0 { + return syscall.InvalidHandle, syscall.ERROR_FILE_NOT_FOUND + } + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return syscall.InvalidHandle, err + } + var access uint32 + switch mode & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) { + case syscall.O_RDONLY: + access = syscall.GENERIC_READ + case syscall.O_WRONLY: + access = syscall.GENERIC_WRITE + case syscall.O_RDWR: + access = syscall.GENERIC_READ | syscall.GENERIC_WRITE + } + if mode&syscall.O_CREAT != 0 { + access |= syscall.GENERIC_WRITE + } + if mode&syscall.O_APPEND != 0 { + access &^= syscall.GENERIC_WRITE + access |= syscall.FILE_APPEND_DATA + } + sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE) + var sa *syscall.SecurityAttributes + if mode&syscall.O_CLOEXEC == 0 { + sa = makeInheritSa() + } + var createmode uint32 + switch { + case mode&(syscall.O_CREAT|syscall.O_EXCL) == (syscall.O_CREAT | syscall.O_EXCL): + createmode = syscall.CREATE_NEW + case mode&(syscall.O_CREAT|syscall.O_TRUNC) == (syscall.O_CREAT | syscall.O_TRUNC): + createmode = syscall.CREATE_ALWAYS + case mode&syscall.O_CREAT == syscall.O_CREAT: + createmode = syscall.OPEN_ALWAYS + case mode&syscall.O_TRUNC == syscall.O_TRUNC: + createmode = syscall.TRUNCATE_EXISTING + default: + createmode = syscall.OPEN_EXISTING + } + // Use FILE_FLAG_SEQUENTIAL_SCAN rather than FILE_ATTRIBUTE_NORMAL as implemented in golang. + //https://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx + const fileFlagSequentialScan = 0x08000000 // FILE_FLAG_SEQUENTIAL_SCAN + h, e := syscall.CreateFile(pathp, access, sharemode, sa, createmode, fileFlagSequentialScan, 0) + return h, e +} From a419ce520f978ad34ec24021f8f852d59c657d1f Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 18 Nov 2016 16:06:50 -0800 Subject: [PATCH 05/19] skip api/types/container/ (like golint) and fix one pkg Signed-off-by: Victor Vieux --- sequential/sequential_windows.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index 6fa5090d..6094f01f 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -98,7 +98,7 @@ func mkdirWithACL(name string) error { sddl := "D:P(A;OICI;GA;;;BA)(A;OICI;GA;;;SY)" sd, err := winio.SddlToSecurityDescriptor(sddl) if err != nil { - return &os.PathError{"mkdir", name, err} + return &os.PathError{Op: "mkdir", Path: name, Err: err} } sa.Length = uint32(unsafe.Sizeof(sa)) sa.InheritHandle = 1 @@ -106,12 +106,12 @@ func mkdirWithACL(name string) error { namep, err := syscall.UTF16PtrFromString(name) if err != nil { - return &os.PathError{"mkdir", name, err} + return &os.PathError{Op: "mkdir", Path: name, Err: err} } e := syscall.CreateDirectory(namep, &sa) if e != nil { - return &os.PathError{"mkdir", name, e} + return &os.PathError{Op: "mkdir", Path: name, Err: e} } return nil } @@ -162,13 +162,13 @@ func OpenSequential(name string) (*os.File, error) { // If there is an error, it will be of type *PathError. func OpenFileSequential(name string, flag int, _ os.FileMode) (*os.File, error) { if name == "" { - return nil, &os.PathError{"open", name, syscall.ENOENT} + return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT} } r, errf := syscallOpenFileSequential(name, flag, 0) if errf == nil { return r, nil } - return nil, &os.PathError{"open", name, errf} + return nil, &os.PathError{Op: "open", Path: name, Err: errf} } func syscallOpenFileSequential(name string, flag int, _ os.FileMode) (file *os.File, err error) { From aafb08f723fafdee4a618f39c4fb4fe9d0fe69d9 Mon Sep 17 00:00:00 2001 From: John Howard Date: Wed, 11 Jan 2017 15:55:43 -0800 Subject: [PATCH 06/19] Windows: Use sequential file access Signed-off-by: John Howard --- sequential/sequential_windows.go | 55 ++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index 6094f01f..626d2ad8 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -6,8 +6,11 @@ import ( "os" "path/filepath" "regexp" + "strconv" "strings" + "sync" "syscall" + "time" "unsafe" winio "github.com/Microsoft/go-winio" @@ -234,3 +237,55 @@ func syscallOpenSequential(path string, mode int, _ uint32) (fd syscall.Handle, h, e := syscall.CreateFile(pathp, access, sharemode, sa, createmode, fileFlagSequentialScan, 0) return h, e } + +// Helpers for TempFileSequential +var rand uint32 +var randmu sync.Mutex + +func reseed() uint32 { + return uint32(time.Now().UnixNano() + int64(os.Getpid())) +} +func nextSuffix() string { + randmu.Lock() + r := rand + if r == 0 { + r = reseed() + } + r = r*1664525 + 1013904223 // constants from Numerical Recipes + rand = r + randmu.Unlock() + return strconv.Itoa(int(1e9 + r%1e9))[1:] +} + +// TempFileSequential is a copy of ioutil.TempFile, modified to use sequential +// file access. Below is the original comment from golang: +// TempFile creates a new temporary file in the directory dir +// with a name beginning with prefix, opens the file for reading +// and writing, and returns the resulting *os.File. +// If dir is the empty string, TempFile uses the default directory +// for temporary files (see os.TempDir). +// Multiple programs calling TempFile simultaneously +// will not choose the same file. The caller can use f.Name() +// to find the pathname of the file. It is the caller's responsibility +// to remove the file when no longer needed. +func TempFileSequential(dir, prefix string) (f *os.File, err error) { + if dir == "" { + dir = os.TempDir() + } + + nconflict := 0 + for i := 0; i < 10000; i++ { + name := filepath.Join(dir, prefix+nextSuffix()) + f, err = OpenFileSequential(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + if os.IsExist(err) { + if nconflict++; nconflict > 10 { + randmu.Lock() + rand = reseed() + randmu.Unlock() + } + continue + } + break + } + return +} From b29161494456eebddfb34af803a7ea93b4acb62c Mon Sep 17 00:00:00 2001 From: John Howard Date: Thu, 1 Jun 2017 18:59:11 -0700 Subject: [PATCH 07/19] LCOW: Create layer folders with correct ACL Signed-off-by: John Howard --- sequential/sequential_windows.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index 626d2ad8..20117db9 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -16,21 +16,28 @@ import ( winio "github.com/Microsoft/go-winio" ) +const ( + // SddlAdministratorsLocalSystem is local administrators plus NT AUTHORITY\System + SddlAdministratorsLocalSystem = "D:P(A;OICI;GA;;;BA)(A;OICI;GA;;;SY)" + // SddlNtvmAdministratorsLocalSystem is NT VIRTUAL MACHINE\Virtual Machines plus local administrators plus NT AUTHORITY\System + SddlNtvmAdministratorsLocalSystem = "D:P(A;OICI;GA;;;S-1-5-83-0)(A;OICI;GA;;;BA)(A;OICI;GA;;;SY)" +) + // MkdirAllWithACL is a wrapper for MkdirAll that creates a directory -// ACL'd for Builtin Administrators and Local System. -func MkdirAllWithACL(path string, perm os.FileMode) error { - return mkdirall(path, true) +// with an appropriate SDDL defined ACL. +func MkdirAllWithACL(path string, perm os.FileMode, sddl string) error { + return mkdirall(path, true, sddl) } // MkdirAll implementation that is volume path aware for Windows. -func MkdirAll(path string, _ os.FileMode) error { - return mkdirall(path, false) +func MkdirAll(path string, _ os.FileMode, sddl string) error { + return mkdirall(path, false, sddl) } // mkdirall is a custom version of os.MkdirAll modified for use on Windows // so that it is both volume path aware, and can create a directory with // a DACL. -func mkdirall(path string, adminAndLocalSystem bool) error { +func mkdirall(path string, applyACL bool, sddl string) error { if re := regexp.MustCompile(`^\\\\\?\\Volume{[a-z0-9-]+}$`); re.MatchString(path) { return nil } @@ -64,15 +71,15 @@ func mkdirall(path string, adminAndLocalSystem bool) error { if j > 1 { // Create parent - err = mkdirall(path[0:j-1], false) + err = mkdirall(path[0:j-1], false, sddl) if err != nil { return err } } // Parent now exists; invoke os.Mkdir or mkdirWithACL and use its result. - if adminAndLocalSystem { - err = mkdirWithACL(path) + if applyACL { + err = mkdirWithACL(path, sddl) } else { err = os.Mkdir(path, 0) } @@ -96,9 +103,9 @@ func mkdirall(path string, adminAndLocalSystem bool) error { // in golang to cater for creating a directory am ACL permitting full // access, with inheritance, to any subfolder/file for Built-in Administrators // and Local System. -func mkdirWithACL(name string) error { +func mkdirWithACL(name string, sddl string) error { sa := syscall.SecurityAttributes{Length: 0} - sddl := "D:P(A;OICI;GA;;;BA)(A;OICI;GA;;;SY)" + sd, err := winio.SddlToSecurityDescriptor(sddl) if err != nil { return &os.PathError{Op: "mkdir", Path: name, Err: err} From b6bc91724a37b777aea68ec213dbfa0b168557d9 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Tue, 23 May 2017 10:22:32 -0400 Subject: [PATCH 08/19] [project] change syscall to /x/sys/unix|windows Changes most references of syscall to golang.org/x/sys/ Ones aren't changes include, Errno, Signal and SysProcAttr as they haven't been implemented in /x/sys/. Signed-off-by: Christopher Jones [s390x] switch utsname from unsigned to signed per https://github.com/golang/sys/commit/33267e036fd93fcd26ea95b7bdaf2d8306cb743c char in s390x in the /x/sys/unix package is now signed, so change the buildtags Signed-off-by: Christopher Jones --- sequential/sequential_windows.go | 80 ++++++++++++++++---------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index 20117db9..a61b53d0 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -14,6 +14,7 @@ import ( "unsafe" winio "github.com/Microsoft/go-winio" + "golang.org/x/sys/windows" ) const ( @@ -99,13 +100,12 @@ func mkdirall(path string, applyACL bool, sddl string) error { // mkdirWithACL creates a new directory. If there is an error, it will be of // type *PathError. . // -// This is a modified and combined version of os.Mkdir and syscall.Mkdir +// This is a modified and combined version of os.Mkdir and windows.Mkdir // in golang to cater for creating a directory am ACL permitting full // access, with inheritance, to any subfolder/file for Built-in Administrators // and Local System. func mkdirWithACL(name string, sddl string) error { - sa := syscall.SecurityAttributes{Length: 0} - + sa := windows.SecurityAttributes{Length: 0} sd, err := winio.SddlToSecurityDescriptor(sddl) if err != nil { return &os.PathError{Op: "mkdir", Path: name, Err: err} @@ -114,12 +114,12 @@ func mkdirWithACL(name string, sddl string) error { sa.InheritHandle = 1 sa.SecurityDescriptor = uintptr(unsafe.Pointer(&sd[0])) - namep, err := syscall.UTF16PtrFromString(name) + namep, err := windows.UTF16PtrFromString(name) if err != nil { return &os.PathError{Op: "mkdir", Path: name, Err: err} } - e := syscall.CreateDirectory(namep, &sa) + e := windows.CreateDirectory(namep, &sa) if e != nil { return &os.PathError{Op: "mkdir", Path: name, Err: e} } @@ -142,7 +142,7 @@ func IsAbs(path string) bool { return true } -// The origin of the functions below here are the golang OS and syscall packages, +// The origin of the functions below here are the golang OS and windows packages, // slightly modified to only cope with files, not directories due to the // specific use case. // @@ -174,74 +174,74 @@ func OpenFileSequential(name string, flag int, _ os.FileMode) (*os.File, error) if name == "" { return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT} } - r, errf := syscallOpenFileSequential(name, flag, 0) + r, errf := windowsOpenFileSequential(name, flag, 0) if errf == nil { return r, nil } return nil, &os.PathError{Op: "open", Path: name, Err: errf} } -func syscallOpenFileSequential(name string, flag int, _ os.FileMode) (file *os.File, err error) { - r, e := syscallOpenSequential(name, flag|syscall.O_CLOEXEC, 0) +func windowsOpenFileSequential(name string, flag int, _ os.FileMode) (file *os.File, err error) { + r, e := windowsOpenSequential(name, flag|windows.O_CLOEXEC, 0) if e != nil { return nil, e } return os.NewFile(uintptr(r), name), nil } -func makeInheritSa() *syscall.SecurityAttributes { - var sa syscall.SecurityAttributes +func makeInheritSa() *windows.SecurityAttributes { + var sa windows.SecurityAttributes sa.Length = uint32(unsafe.Sizeof(sa)) sa.InheritHandle = 1 return &sa } -func syscallOpenSequential(path string, mode int, _ uint32) (fd syscall.Handle, err error) { +func windowsOpenSequential(path string, mode int, _ uint32) (fd windows.Handle, err error) { if len(path) == 0 { - return syscall.InvalidHandle, syscall.ERROR_FILE_NOT_FOUND + return windows.InvalidHandle, windows.ERROR_FILE_NOT_FOUND } - pathp, err := syscall.UTF16PtrFromString(path) + pathp, err := windows.UTF16PtrFromString(path) if err != nil { - return syscall.InvalidHandle, err + return windows.InvalidHandle, err } var access uint32 - switch mode & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) { - case syscall.O_RDONLY: - access = syscall.GENERIC_READ - case syscall.O_WRONLY: - access = syscall.GENERIC_WRITE - case syscall.O_RDWR: - access = syscall.GENERIC_READ | syscall.GENERIC_WRITE + switch mode & (windows.O_RDONLY | windows.O_WRONLY | windows.O_RDWR) { + case windows.O_RDONLY: + access = windows.GENERIC_READ + case windows.O_WRONLY: + access = windows.GENERIC_WRITE + case windows.O_RDWR: + access = windows.GENERIC_READ | windows.GENERIC_WRITE } - if mode&syscall.O_CREAT != 0 { - access |= syscall.GENERIC_WRITE + if mode&windows.O_CREAT != 0 { + access |= windows.GENERIC_WRITE } - if mode&syscall.O_APPEND != 0 { - access &^= syscall.GENERIC_WRITE - access |= syscall.FILE_APPEND_DATA + if mode&windows.O_APPEND != 0 { + access &^= windows.GENERIC_WRITE + access |= windows.FILE_APPEND_DATA } - sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE) - var sa *syscall.SecurityAttributes - if mode&syscall.O_CLOEXEC == 0 { + sharemode := uint32(windows.FILE_SHARE_READ | windows.FILE_SHARE_WRITE) + var sa *windows.SecurityAttributes + if mode&windows.O_CLOEXEC == 0 { sa = makeInheritSa() } var createmode uint32 switch { - case mode&(syscall.O_CREAT|syscall.O_EXCL) == (syscall.O_CREAT | syscall.O_EXCL): - createmode = syscall.CREATE_NEW - case mode&(syscall.O_CREAT|syscall.O_TRUNC) == (syscall.O_CREAT | syscall.O_TRUNC): - createmode = syscall.CREATE_ALWAYS - case mode&syscall.O_CREAT == syscall.O_CREAT: - createmode = syscall.OPEN_ALWAYS - case mode&syscall.O_TRUNC == syscall.O_TRUNC: - createmode = syscall.TRUNCATE_EXISTING + case mode&(windows.O_CREAT|windows.O_EXCL) == (windows.O_CREAT | windows.O_EXCL): + createmode = windows.CREATE_NEW + case mode&(windows.O_CREAT|windows.O_TRUNC) == (windows.O_CREAT | windows.O_TRUNC): + createmode = windows.CREATE_ALWAYS + case mode&windows.O_CREAT == windows.O_CREAT: + createmode = windows.OPEN_ALWAYS + case mode&windows.O_TRUNC == windows.O_TRUNC: + createmode = windows.TRUNCATE_EXISTING default: - createmode = syscall.OPEN_EXISTING + createmode = windows.OPEN_EXISTING } // Use FILE_FLAG_SEQUENTIAL_SCAN rather than FILE_ATTRIBUTE_NORMAL as implemented in golang. //https://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx const fileFlagSequentialScan = 0x08000000 // FILE_FLAG_SEQUENTIAL_SCAN - h, e := syscall.CreateFile(pathp, access, sharemode, sa, createmode, fileFlagSequentialScan, 0) + h, e := windows.CreateFile(pathp, access, sharemode, sa, createmode, fileFlagSequentialScan, 0) return h, e } From c24df864604c4623cf8b02e2207ecf5bdb403030 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 18 Dec 2017 17:41:53 +0100 Subject: [PATCH 09/19] Remove redundant build-tags Files that are suffixed with `_linux.go` or `_windows.go` are already only built on Linux / Windows, so these build-tags were redundant. Signed-off-by: Sebastiaan van Stijn --- sequential/sequential_windows.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index a61b53d0..b1e46d9e 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -1,5 +1,3 @@ -// +build windows - package system import ( From 3511ac808c500c81deafc5f7be0184ef80ab9875 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 5 Feb 2018 16:05:59 -0500 Subject: [PATCH 10/19] Add canonical import comment Signed-off-by: Daniel Nephin --- sequential/sequential_windows.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index b1e46d9e..a1f6013f 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -1,4 +1,4 @@ -package system +package system // import "github.com/docker/docker/pkg/system" import ( "os" From 875002ff01dc54da4bd5c4cfe79f1b58e352e225 Mon Sep 17 00:00:00 2001 From: John Howard Date: Thu, 21 Mar 2019 09:20:48 -0700 Subject: [PATCH 11/19] LCOW: Add SIDs to layer.vhd at creation Signed-off-by: John Howard Some permissions corrections here. Also needs re-vendor of go-winio. - Create the layer folder directory as standard, not with SDDL. It will inherit permissions from the data-root correctly. - Apply the VM Group SID access to layer.vhd Permissions after this changes Data root: ``` PS C:\> icacls test test BUILTIN\Administrators:(OI)(CI)(F) NT AUTHORITY\SYSTEM:(OI)(CI)(F) ``` lcow subdirectory under dataroot ``` PS C:\> icacls test\lcow test\lcow BUILTIN\Administrators:(I)(OI)(CI)(F) NT AUTHORITY\SYSTEM:(I)(OI)(CI)(F) ``` layer.vhd in a layer folder for LCOW ``` .\test\lcow\c33923d21c9621fea2f990a8778f469ecdbdc57fd9ca682565d1fa86fadd5d95\layer.vhd NT VIRTUAL MACHINE\Virtual Machines:(R) BUILTIN\Administrators:(I)(F) NT AUTHORITY\SYSTEM:(I)(F) ``` And showing working ``` PS C:\> docker-ci-zap -folder=c:\test INFO: Zapped successfully PS C:\> docker run --rm alpine echo hello Unable to find image 'alpine:latest' locally latest: Pulling from library/alpine 8e402f1a9c57: Pull complete Digest: sha256:644fcb1a676b5165371437feaa922943aaf7afcfa8bfee4472f6860aad1ef2a0 Status: Downloaded newer image for alpine:latest hello ``` --- sequential/sequential_windows.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index a1f6013f..3049ff38 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -18,8 +18,6 @@ import ( const ( // SddlAdministratorsLocalSystem is local administrators plus NT AUTHORITY\System SddlAdministratorsLocalSystem = "D:P(A;OICI;GA;;;BA)(A;OICI;GA;;;SY)" - // SddlNtvmAdministratorsLocalSystem is NT VIRTUAL MACHINE\Virtual Machines plus local administrators plus NT AUTHORITY\System - SddlNtvmAdministratorsLocalSystem = "D:P(A;OICI;GA;;;S-1-5-83-0)(A;OICI;GA;;;BA)(A;OICI;GA;;;SY)" ) // MkdirAllWithACL is a wrapper for MkdirAll that creates a directory From 12d814f286bd76de96ca640201aabd52cfcd5356 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 8 Aug 2019 11:51:00 +0200 Subject: [PATCH 12/19] Allow system.MkDirAll() to be used as drop-in for os.MkDirAll() also renamed the non-windows variant of this file to be consistent with other files in this package Signed-off-by: Sebastiaan van Stijn --- sequential/sequential_unix.go | 67 ++++++++++++++++++++++++++++++++ sequential/sequential_windows.go | 7 ++-- 2 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 sequential/sequential_unix.go diff --git a/sequential/sequential_unix.go b/sequential/sequential_unix.go new file mode 100644 index 00000000..dcee3e9f --- /dev/null +++ b/sequential/sequential_unix.go @@ -0,0 +1,67 @@ +// +build !windows + +package system // import "github.com/docker/docker/pkg/system" + +import ( + "io/ioutil" + "os" + "path/filepath" +) + +// MkdirAllWithACL is a wrapper for os.MkdirAll on unix systems. +func MkdirAllWithACL(path string, perm os.FileMode, sddl string) error { + return os.MkdirAll(path, perm) +} + +// MkdirAll creates a directory named path along with any necessary parents, +// with permission specified by attribute perm for all dir created. +func MkdirAll(path string, perm os.FileMode) error { + return os.MkdirAll(path, perm) +} + +// IsAbs is a platform-specific wrapper for filepath.IsAbs. +func IsAbs(path string) bool { + return filepath.IsAbs(path) +} + +// The functions below here are wrappers for the equivalents in the os and ioutils packages. +// They are passthrough on Unix platforms, and only relevant on Windows. + +// CreateSequential creates the named file with mode 0666 (before umask), truncating +// it if it already exists. If successful, methods on the returned +// File can be used for I/O; the associated file descriptor has mode +// O_RDWR. +// If there is an error, it will be of type *PathError. +func CreateSequential(name string) (*os.File, error) { + return os.Create(name) +} + +// OpenSequential opens the named file for reading. If successful, methods on +// the returned file can be used for reading; the associated file +// descriptor has mode O_RDONLY. +// If there is an error, it will be of type *PathError. +func OpenSequential(name string) (*os.File, error) { + return os.Open(name) +} + +// OpenFileSequential is the generalized open call; most users will use Open +// or Create instead. It opens the named file with specified flag +// (O_RDONLY etc.) and perm, (0666 etc.) if applicable. If successful, +// methods on the returned File can be used for I/O. +// If there is an error, it will be of type *PathError. +func OpenFileSequential(name string, flag int, perm os.FileMode) (*os.File, error) { + return os.OpenFile(name, flag, perm) +} + +// TempFileSequential creates a new temporary file in the directory dir +// with a name beginning with prefix, opens the file for reading +// and writing, and returns the resulting *os.File. +// If dir is the empty string, TempFile uses the default directory +// for temporary files (see os.TempDir). +// Multiple programs calling TempFile simultaneously +// will not choose the same file. The caller can use f.Name() +// to find the pathname of the file. It is the caller's responsibility +// to remove the file when no longer needed. +func TempFileSequential(dir, prefix string) (f *os.File, err error) { + return ioutil.TempFile(dir, prefix) +} diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index 3049ff38..7cebd6ef 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -26,9 +26,10 @@ func MkdirAllWithACL(path string, perm os.FileMode, sddl string) error { return mkdirall(path, true, sddl) } -// MkdirAll implementation that is volume path aware for Windows. -func MkdirAll(path string, _ os.FileMode, sddl string) error { - return mkdirall(path, false, sddl) +// MkdirAll implementation that is volume path aware for Windows. It can be used +// as a drop-in replacement for os.MkdirAll() +func MkdirAll(path string, _ os.FileMode) error { + return mkdirall(path, false, "") } // mkdirall is a custom version of os.MkdirAll modified for use on Windows From 64d50040e562bd5c6c025f47190da925a8b55907 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Mon, 30 Sep 2019 17:07:47 +0200 Subject: [PATCH 13/19] Use newer x/sys/windows SecurityAttributes struct This struct now has a properly typed member, so use the properly typed functions with it. Also update the vendor directory and hope nothing explodes. Signed-off-by: Jason A. Donenfeld Signed-off-by: Sebastiaan van Stijn --- sequential/sequential_windows.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index 7cebd6ef..e95902f3 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -11,7 +11,6 @@ import ( "time" "unsafe" - winio "github.com/Microsoft/go-winio" "golang.org/x/sys/windows" ) @@ -103,13 +102,13 @@ func mkdirall(path string, applyACL bool, sddl string) error { // and Local System. func mkdirWithACL(name string, sddl string) error { sa := windows.SecurityAttributes{Length: 0} - sd, err := winio.SddlToSecurityDescriptor(sddl) + sd, err := windows.SecurityDescriptorFromString(sddl) if err != nil { return &os.PathError{Op: "mkdir", Path: name, Err: err} } sa.Length = uint32(unsafe.Sizeof(sa)) sa.InheritHandle = 1 - sa.SecurityDescriptor = uintptr(unsafe.Pointer(&sd[0])) + sa.SecurityDescriptor = sd namep, err := windows.UTF16PtrFromString(name) if err != nil { From 845ff9d3db9041268b141b627ca7b544443d5656 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 27 Nov 2019 15:38:17 +0100 Subject: [PATCH 14/19] pkg/system: normalize comment formatting Signed-off-by: Sebastiaan van Stijn --- sequential/sequential_windows.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index e95902f3..e1d134a5 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -235,7 +235,7 @@ func windowsOpenSequential(path string, mode int, _ uint32) (fd windows.Handle, createmode = windows.OPEN_EXISTING } // Use FILE_FLAG_SEQUENTIAL_SCAN rather than FILE_ATTRIBUTE_NORMAL as implemented in golang. - //https://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx + // https://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx const fileFlagSequentialScan = 0x08000000 // FILE_FLAG_SEQUENTIAL_SCAN h, e := windows.CreateFile(pathp, access, sharemode, sa, createmode, fileFlagSequentialScan, 0) return h, e From 88b47d5f16bde0a9a915f34876fd9ca483c3009c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 9 Mar 2020 16:35:12 +0100 Subject: [PATCH 15/19] pkg/system: simplify IsAbs() Signed-off-by: Sebastiaan van Stijn --- sequential/sequential_windows.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index e1d134a5..b4646277 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -130,12 +130,10 @@ func mkdirWithACL(name string, sddl string) error { // by the daemon. This SHOULD be treated as absolute from a docker processing // perspective. func IsAbs(path string) bool { - if !filepath.IsAbs(path) { - if !strings.HasPrefix(path, string(os.PathSeparator)) { - return false - } + if filepath.IsAbs(path) || strings.HasPrefix(path, string(os.PathSeparator)) { + return true } - return true + return false } // The origin of the functions below here are the golang OS and windows packages, From de65c12788f81fa2fe181c190cc0bae49632ce14 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 23 Aug 2021 15:14:53 +0200 Subject: [PATCH 16/19] Update to Go 1.17.0, and gofmt with Go 1.17 Signed-off-by: Sebastiaan van Stijn --- sequential/sequential_unix.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sequential/sequential_unix.go b/sequential/sequential_unix.go index dcee3e9f..186d9d9a 100644 --- a/sequential/sequential_unix.go +++ b/sequential/sequential_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package system // import "github.com/docker/docker/pkg/system" From 8b36fc0028d59a0d10eeb92e150863d2f926eec1 Mon Sep 17 00:00:00 2001 From: Eng Zer Jun Date: Tue, 24 Aug 2021 18:10:50 +0800 Subject: [PATCH 17/19] refactor: move from io/ioutil to io and os package The io/ioutil package has been deprecated in Go 1.16. This commit replaces the existing io/ioutil functions with their new definitions in io and os packages. Signed-off-by: Eng Zer Jun --- sequential/sequential_unix.go | 3 +-- sequential/sequential_windows.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sequential/sequential_unix.go b/sequential/sequential_unix.go index 186d9d9a..8b991201 100644 --- a/sequential/sequential_unix.go +++ b/sequential/sequential_unix.go @@ -4,7 +4,6 @@ package system // import "github.com/docker/docker/pkg/system" import ( - "io/ioutil" "os" "path/filepath" ) @@ -64,5 +63,5 @@ func OpenFileSequential(name string, flag int, perm os.FileMode) (*os.File, erro // to find the pathname of the file. It is the caller's responsibility // to remove the file when no longer needed. func TempFileSequential(dir, prefix string) (f *os.File, err error) { - return ioutil.TempFile(dir, prefix) + return os.CreateTemp(dir, prefix) } diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index b4646277..8f79dc8f 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -258,7 +258,7 @@ func nextSuffix() string { return strconv.Itoa(int(1e9 + r%1e9))[1:] } -// TempFileSequential is a copy of ioutil.TempFile, modified to use sequential +// TempFileSequential is a copy of os.CreateTemp, modified to use sequential // file access. Below is the original comment from golang: // TempFile creates a new temporary file in the directory dir // with a name beginning with prefix, opens the file for reading From 172bbe5bf3be92bd32efac1565b7e702f6695332 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 27 Aug 2022 13:13:02 +0200 Subject: [PATCH 18/19] sequential: initialize module, and remove unrelated functions Signed-off-by: Sebastiaan van Stijn --- Makefile | 2 +- sequential/doc.go | 15 ++++ sequential/go.mod | 5 ++ sequential/go.sum | 2 + sequential/sequential_unix.go | 26 +----- sequential/sequential_windows.go | 137 +------------------------------ 6 files changed, 28 insertions(+), 159 deletions(-) create mode 100644 sequential/doc.go create mode 100644 sequential/go.mod create mode 100644 sequential/go.sum diff --git a/Makefile b/Makefile index 389e09df..80c6ee9d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PACKAGES ?= mountinfo mount signal symlink +PACKAGES ?= mountinfo mount sequential signal symlink BINDIR ?= _build/bin CROSS ?= linux/arm linux/arm64 linux/ppc64le linux/s390x \ freebsd/amd64 openbsd/amd64 darwin/amd64 darwin/arm64 windows/amd64 diff --git a/sequential/doc.go b/sequential/doc.go new file mode 100644 index 00000000..af281750 --- /dev/null +++ b/sequential/doc.go @@ -0,0 +1,15 @@ +// Package sequential provides a set of functions for managing sequential +// files on Windows. +// +// The origin of these functions are the golang OS and windows packages, +// slightly modified to only cope with files, not directories due to the +// specific use case. +// +// The alteration is to allow a file on Windows to be opened with +// FILE_FLAG_SEQUENTIAL_SCAN (particular for docker load), to avoid eating +// the standby list, particularly when accessing large files such as layer.tar. +// +// For non-Windows platforms, the package provides wrappers for the equivalents +// in the os packages. They are passthrough on Unix platforms, and only relevant +// on Windows. +package sequential diff --git a/sequential/go.mod b/sequential/go.mod new file mode 100644 index 00000000..59c2412a --- /dev/null +++ b/sequential/go.mod @@ -0,0 +1,5 @@ +module github.com/moby/sys/sequential + +go 1.17 + +require golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a diff --git a/sequential/go.sum b/sequential/go.sum new file mode 100644 index 00000000..af14a66e --- /dev/null +++ b/sequential/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/sequential/sequential_unix.go b/sequential/sequential_unix.go index 8b991201..d7ac7f44 100644 --- a/sequential/sequential_unix.go +++ b/sequential/sequential_unix.go @@ -1,31 +1,9 @@ //go:build !windows // +build !windows -package system // import "github.com/docker/docker/pkg/system" +package sequential -import ( - "os" - "path/filepath" -) - -// MkdirAllWithACL is a wrapper for os.MkdirAll on unix systems. -func MkdirAllWithACL(path string, perm os.FileMode, sddl string) error { - return os.MkdirAll(path, perm) -} - -// MkdirAll creates a directory named path along with any necessary parents, -// with permission specified by attribute perm for all dir created. -func MkdirAll(path string, perm os.FileMode) error { - return os.MkdirAll(path, perm) -} - -// IsAbs is a platform-specific wrapper for filepath.IsAbs. -func IsAbs(path string) bool { - return filepath.IsAbs(path) -} - -// The functions below here are wrappers for the equivalents in the os and ioutils packages. -// They are passthrough on Unix platforms, and only relevant on Windows. +import "os" // CreateSequential creates the named file with mode 0666 (before umask), truncating // it if it already exists. If successful, methods on the returned diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index 8f79dc8f..4fda979c 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -1,11 +1,9 @@ -package system // import "github.com/docker/docker/pkg/system" +package sequential import ( "os" "path/filepath" - "regexp" "strconv" - "strings" "sync" "syscall" "time" @@ -14,136 +12,6 @@ import ( "golang.org/x/sys/windows" ) -const ( - // SddlAdministratorsLocalSystem is local administrators plus NT AUTHORITY\System - SddlAdministratorsLocalSystem = "D:P(A;OICI;GA;;;BA)(A;OICI;GA;;;SY)" -) - -// MkdirAllWithACL is a wrapper for MkdirAll that creates a directory -// with an appropriate SDDL defined ACL. -func MkdirAllWithACL(path string, perm os.FileMode, sddl string) error { - return mkdirall(path, true, sddl) -} - -// MkdirAll implementation that is volume path aware for Windows. It can be used -// as a drop-in replacement for os.MkdirAll() -func MkdirAll(path string, _ os.FileMode) error { - return mkdirall(path, false, "") -} - -// mkdirall is a custom version of os.MkdirAll modified for use on Windows -// so that it is both volume path aware, and can create a directory with -// a DACL. -func mkdirall(path string, applyACL bool, sddl string) error { - if re := regexp.MustCompile(`^\\\\\?\\Volume{[a-z0-9-]+}$`); re.MatchString(path) { - return nil - } - - // The rest of this method is largely copied from os.MkdirAll and should be kept - // as-is to ensure compatibility. - - // Fast path: if we can tell whether path is a directory or file, stop with success or error. - dir, err := os.Stat(path) - if err == nil { - if dir.IsDir() { - return nil - } - return &os.PathError{ - Op: "mkdir", - Path: path, - Err: syscall.ENOTDIR, - } - } - - // Slow path: make sure parent exists and then call Mkdir for path. - i := len(path) - for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator. - i-- - } - - j := i - for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element. - j-- - } - - if j > 1 { - // Create parent - err = mkdirall(path[0:j-1], false, sddl) - if err != nil { - return err - } - } - - // Parent now exists; invoke os.Mkdir or mkdirWithACL and use its result. - if applyACL { - err = mkdirWithACL(path, sddl) - } else { - err = os.Mkdir(path, 0) - } - - if err != nil { - // Handle arguments like "foo/." by - // double-checking that directory doesn't exist. - dir, err1 := os.Lstat(path) - if err1 == nil && dir.IsDir() { - return nil - } - return err - } - return nil -} - -// mkdirWithACL creates a new directory. If there is an error, it will be of -// type *PathError. . -// -// This is a modified and combined version of os.Mkdir and windows.Mkdir -// in golang to cater for creating a directory am ACL permitting full -// access, with inheritance, to any subfolder/file for Built-in Administrators -// and Local System. -func mkdirWithACL(name string, sddl string) error { - sa := windows.SecurityAttributes{Length: 0} - sd, err := windows.SecurityDescriptorFromString(sddl) - if err != nil { - return &os.PathError{Op: "mkdir", Path: name, Err: err} - } - sa.Length = uint32(unsafe.Sizeof(sa)) - sa.InheritHandle = 1 - sa.SecurityDescriptor = sd - - namep, err := windows.UTF16PtrFromString(name) - if err != nil { - return &os.PathError{Op: "mkdir", Path: name, Err: err} - } - - e := windows.CreateDirectory(namep, &sa) - if e != nil { - return &os.PathError{Op: "mkdir", Path: name, Err: e} - } - return nil -} - -// IsAbs is a platform-specific wrapper for filepath.IsAbs. On Windows, -// golang filepath.IsAbs does not consider a path \windows\system32 as absolute -// as it doesn't start with a drive-letter/colon combination. However, in -// docker we need to verify things such as WORKDIR /windows/system32 in -// a Dockerfile (which gets translated to \windows\system32 when being processed -// by the daemon. This SHOULD be treated as absolute from a docker processing -// perspective. -func IsAbs(path string) bool { - if filepath.IsAbs(path) || strings.HasPrefix(path, string(os.PathSeparator)) { - return true - } - return false -} - -// The origin of the functions below here are the golang OS and windows packages, -// slightly modified to only cope with files, not directories due to the -// specific use case. -// -// The alteration is to allow a file on Windows to be opened with -// FILE_FLAG_SEQUENTIAL_SCAN (particular for docker load), to avoid eating -// the standby list, particularly when accessing large files such as layer.tar. - // CreateSequential creates the named file with mode 0666 (before umask), truncating // it if it already exists. If successful, methods on the returned // File can be used for I/O; the associated file descriptor has mode @@ -246,6 +114,7 @@ var randmu sync.Mutex func reseed() uint32 { return uint32(time.Now().UnixNano() + int64(os.Getpid())) } + func nextSuffix() string { randmu.Lock() r := rand @@ -277,7 +146,7 @@ func TempFileSequential(dir, prefix string) (f *os.File, err error) { nconflict := 0 for i := 0; i < 10000; i++ { name := filepath.Join(dir, prefix+nextSuffix()) - f, err = OpenFileSequential(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + f, err = OpenFileSequential(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) if os.IsExist(err) { if nconflict++; nconflict > 10 { randmu.Lock() From a88a50063514d68279421a926b3bf4a496f4c047 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 27 Aug 2022 13:22:06 +0200 Subject: [PATCH 19/19] sequential: rename funcs to match their "os" equivalent Rename the functions so that they can act as a drop-in replacement for their "os" equivalents. Signed-off-by: Sebastiaan van Stijn --- sequential/sequential_unix.go | 16 +++++++------- sequential/sequential_windows.go | 36 ++++++++++++++++---------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/sequential/sequential_unix.go b/sequential/sequential_unix.go index d7ac7f44..a3c7340e 100644 --- a/sequential/sequential_unix.go +++ b/sequential/sequential_unix.go @@ -5,33 +5,33 @@ package sequential import "os" -// CreateSequential creates the named file with mode 0666 (before umask), truncating +// Create creates the named file with mode 0666 (before umask), truncating // it if it already exists. If successful, methods on the returned // File can be used for I/O; the associated file descriptor has mode // O_RDWR. // If there is an error, it will be of type *PathError. -func CreateSequential(name string) (*os.File, error) { +func Create(name string) (*os.File, error) { return os.Create(name) } -// OpenSequential opens the named file for reading. If successful, methods on +// Open opens the named file for reading. If successful, methods on // the returned file can be used for reading; the associated file // descriptor has mode O_RDONLY. // If there is an error, it will be of type *PathError. -func OpenSequential(name string) (*os.File, error) { +func Open(name string) (*os.File, error) { return os.Open(name) } -// OpenFileSequential is the generalized open call; most users will use Open +// OpenFile is the generalized open call; most users will use Open // or Create instead. It opens the named file with specified flag // (O_RDONLY etc.) and perm, (0666 etc.) if applicable. If successful, // methods on the returned File can be used for I/O. // If there is an error, it will be of type *PathError. -func OpenFileSequential(name string, flag int, perm os.FileMode) (*os.File, error) { +func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { return os.OpenFile(name, flag, perm) } -// TempFileSequential creates a new temporary file in the directory dir +// CreateTemp creates a new temporary file in the directory dir // with a name beginning with prefix, opens the file for reading // and writing, and returns the resulting *os.File. // If dir is the empty string, TempFile uses the default directory @@ -40,6 +40,6 @@ func OpenFileSequential(name string, flag int, perm os.FileMode) (*os.File, erro // will not choose the same file. The caller can use f.Name() // to find the pathname of the file. It is the caller's responsibility // to remove the file when no longer needed. -func TempFileSequential(dir, prefix string) (f *os.File, err error) { +func CreateTemp(dir, prefix string) (f *os.File, err error) { return os.CreateTemp(dir, prefix) } diff --git a/sequential/sequential_windows.go b/sequential/sequential_windows.go index 4fda979c..3f7f0d83 100644 --- a/sequential/sequential_windows.go +++ b/sequential/sequential_windows.go @@ -12,39 +12,39 @@ import ( "golang.org/x/sys/windows" ) -// CreateSequential creates the named file with mode 0666 (before umask), truncating +// Create creates the named file with mode 0666 (before umask), truncating // it if it already exists. If successful, methods on the returned // File can be used for I/O; the associated file descriptor has mode // O_RDWR. // If there is an error, it will be of type *PathError. -func CreateSequential(name string) (*os.File, error) { - return OpenFileSequential(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0) +func Create(name string) (*os.File, error) { + return OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0) } -// OpenSequential opens the named file for reading. If successful, methods on +// Open opens the named file for reading. If successful, methods on // the returned file can be used for reading; the associated file // descriptor has mode O_RDONLY. // If there is an error, it will be of type *PathError. -func OpenSequential(name string) (*os.File, error) { - return OpenFileSequential(name, os.O_RDONLY, 0) +func Open(name string) (*os.File, error) { + return OpenFile(name, os.O_RDONLY, 0) } -// OpenFileSequential is the generalized open call; most users will use Open +// OpenFile is the generalized open call; most users will use Open // or Create instead. // If there is an error, it will be of type *PathError. -func OpenFileSequential(name string, flag int, _ os.FileMode) (*os.File, error) { +func OpenFile(name string, flag int, _ os.FileMode) (*os.File, error) { if name == "" { return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT} } - r, errf := windowsOpenFileSequential(name, flag, 0) - if errf == nil { + r, err := openFileSequential(name, flag, 0) + if err == nil { return r, nil } - return nil, &os.PathError{Op: "open", Path: name, Err: errf} + return nil, &os.PathError{Op: "open", Path: name, Err: err} } -func windowsOpenFileSequential(name string, flag int, _ os.FileMode) (file *os.File, err error) { - r, e := windowsOpenSequential(name, flag|windows.O_CLOEXEC, 0) +func openFileSequential(name string, flag int, _ os.FileMode) (file *os.File, err error) { + r, e := openSequential(name, flag|windows.O_CLOEXEC, 0) if e != nil { return nil, e } @@ -58,7 +58,7 @@ func makeInheritSa() *windows.SecurityAttributes { return &sa } -func windowsOpenSequential(path string, mode int, _ uint32) (fd windows.Handle, err error) { +func openSequential(path string, mode int, _ uint32) (fd windows.Handle, err error) { if len(path) == 0 { return windows.InvalidHandle, windows.ERROR_FILE_NOT_FOUND } @@ -107,7 +107,7 @@ func windowsOpenSequential(path string, mode int, _ uint32) (fd windows.Handle, return h, e } -// Helpers for TempFileSequential +// Helpers for CreateTemp var rand uint32 var randmu sync.Mutex @@ -127,7 +127,7 @@ func nextSuffix() string { return strconv.Itoa(int(1e9 + r%1e9))[1:] } -// TempFileSequential is a copy of os.CreateTemp, modified to use sequential +// CreateTemp is a copy of os.CreateTemp, modified to use sequential // file access. Below is the original comment from golang: // TempFile creates a new temporary file in the directory dir // with a name beginning with prefix, opens the file for reading @@ -138,7 +138,7 @@ func nextSuffix() string { // will not choose the same file. The caller can use f.Name() // to find the pathname of the file. It is the caller's responsibility // to remove the file when no longer needed. -func TempFileSequential(dir, prefix string) (f *os.File, err error) { +func CreateTemp(dir, prefix string) (f *os.File, err error) { if dir == "" { dir = os.TempDir() } @@ -146,7 +146,7 @@ func TempFileSequential(dir, prefix string) (f *os.File, err error) { nconflict := 0 for i := 0; i < 10000; i++ { name := filepath.Join(dir, prefix+nextSuffix()) - f, err = OpenFileSequential(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) + f, err = OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) if os.IsExist(err) { if nconflict++; nconflict > 10 { randmu.Lock()