From 2c7d469dd25599b1ee65d45edba400689d162d8d Mon Sep 17 00:00:00 2001 From: Daniel Canter Date: Mon, 14 Feb 2022 17:09:45 -0800 Subject: [PATCH 1/2] Linux GCS: Log disk info on ENOSPC errors We've had quite a few times where creating the upper or work directories in the guest fails with ENOSPC but we don't have any view into what the mount looks like at the time we get this. This change just catches any ENOSPC errors when creating an overlayfs mount, calls statfs and logs the disk space and inode info for the mount the failed directory is on. This should make investigating these types of issues much easier. This may be followed up with a change to delete the upper and work directories for a container, as this becomes troublesome with the model for sharing a scratch volume. Signed-off-by: Daniel Canter --- internal/guest/storage/overlay/overlay.go | 44 +++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/internal/guest/storage/overlay/overlay.go b/internal/guest/storage/overlay/overlay.go index b370ab888c..7ec76fd90c 100644 --- a/internal/guest/storage/overlay/overlay.go +++ b/internal/guest/storage/overlay/overlay.go @@ -5,12 +5,17 @@ package overlay import ( "context" + "fmt" "os" + "path/filepath" "strings" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/memory" "github.com/Microsoft/hcsshim/internal/oc" "github.com/Microsoft/hcsshim/pkg/securitypolicy" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "go.opencensus.io/trace" "golang.org/x/sys/unix" ) @@ -22,6 +27,36 @@ var ( unixMount = unix.Mount ) +// processErrNoSpace logs disk space and inode information for `path` that we encountered the ENOSPC error on. This can be used to get a better +// view of whats going on on the disk at the time of the error. +func processErrNoSpace(ctx context.Context, path string, err error) { + st := &unix.Statfs_t{} + // Pass in filepath.Dir() of the path as if we got an error while creating the directory it definitely doesn't exist. Take its parent + // which should be on the same drive. + if statErr := unix.Statfs(filepath.Dir(path), st); statErr != nil { + log.G(ctx).WithError(statErr).WithField("path", filepath.Dir(path)).Warn("failed to get disk information for ENOSPC error") + return + } + + all := st.Blocks * uint64(st.Bsize) + available := st.Bavail * uint64(st.Bsize) + free := st.Bfree * uint64(st.Bsize) + used := all - free + + toGigabyteStr := func(val uint64) string { + return fmt.Sprintf("%.1f", float64(val)/float64(memory.GigaByte)) + } + + log.G(ctx).WithFields(logrus.Fields{ + "available-disk-space-GB": toGigabyteStr(available), + "free-disk-space-GB": toGigabyteStr(free), + "used-disk-space-GB": toGigabyteStr(used), + "total-inodes": st.Files, + "free-inodes": st.Ffree, + "path": path, + }).WithError(err).Warn("got ENOSPC, gathering diagnostics") +} + // MountLayer first enforces the security policy for the container's layer paths // and then calls Mount to mount the layer paths as an overlayfs func MountLayer(ctx context.Context, layerPaths []string, upperdirPath, workdirPath, rootfsPath string, readonly bool, containerId string, securityPolicy securitypolicy.SecurityPolicyEnforcer) (err error) { @@ -58,6 +93,15 @@ func Mount(ctx context.Context, basePaths []string, upperdirPath, workdirPath, t trace.StringAttribute("target", target), trace.BoolAttribute("readonly", readonly)) + // If we got an ENOSPC error on creating any directories, log disk space and inode info for the mount that the directory belongs to get a better + // view of the where the problem lies. + defer func() { + var perr *os.PathError + if errors.As(err, &perr) && perr.Err == unix.ENOSPC { + processErrNoSpace(ctx, perr.Path, err) + } + }() + if target == "" { return errors.New("cannot have empty target") } From 77a4c0f15e86a81e5aeb39bf44d6c295dfda65cb Mon Sep 17 00:00:00 2001 From: Daniel Canter Date: Tue, 15 Feb 2022 10:26:52 -0800 Subject: [PATCH 2/2] PR Feedback: GB -> GiB Signed-off-by: Daniel Canter --- internal/guest/storage/overlay/overlay.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/guest/storage/overlay/overlay.go b/internal/guest/storage/overlay/overlay.go index 7ec76fd90c..93c7fa88d7 100644 --- a/internal/guest/storage/overlay/overlay.go +++ b/internal/guest/storage/overlay/overlay.go @@ -48,12 +48,12 @@ func processErrNoSpace(ctx context.Context, path string, err error) { } log.G(ctx).WithFields(logrus.Fields{ - "available-disk-space-GB": toGigabyteStr(available), - "free-disk-space-GB": toGigabyteStr(free), - "used-disk-space-GB": toGigabyteStr(used), - "total-inodes": st.Files, - "free-inodes": st.Ffree, - "path": path, + "available-disk-space-GiB": toGigabyteStr(available), + "free-disk-space-GiB": toGigabyteStr(free), + "used-disk-space-GiB": toGigabyteStr(used), + "total-inodes": st.Files, + "free-inodes": st.Ffree, + "path": path, }).WithError(err).Warn("got ENOSPC, gathering diagnostics") }