From 3c9e4ff7d9e290b385698a20c2934155166ccb89 Mon Sep 17 00:00:00 2001 From: Daniel Canter Date: Fri, 24 Sep 2021 02:11:47 -0700 Subject: [PATCH 1/2] Add WCOW sandbox mount support This change adds sandbox mount like support for WCOW. Sandboxmounts are our LCOW solution similar to a k8s empty dir mount. We create a directory that will house various other subdirectories that a user can specify by supplying a sandbox:// prefix for the host path of a mount. In the usual case the hostpath of a mount is in the context of the physical host (e.g. I want C:\path on my machine mapped to C:\containerpath in my container), however for sandbox mounts the host path is treated as relative to this directory we have made in the VM. The root directory for these sandbox mounts I defined as C:\SandboxMounts Example: "mounts": [ { "container_path": "C:\\test", "host_path": "sandbox:///test" } ] The above will make a directory at C:\SandboxMounts\test in the Utility VM that will be mapped to C:\\test in the container. If another container in the same pod supplied the same mount then you would end up "sharing" this directory with the other container, meaning you would both see anything placed/modified in this directory. The backing storage for these mounts will be the UVMs scratch disk, which currently is always 10GB (8.5 actually usable) as that's whats hardcoded in HCS for the call we use that generates the vhd. For some reason the expand vhd function from HCS doesn't function for the VM scratch disk which needs some investigation :( Signed-off-by: Daniel Canter --- internal/hcsoci/hcsdoc_wcow.go | 3 ++ internal/hcsoci/resources_wcow.go | 35 +++++++++++++++++++ .../hcsshim/internal/hcsoci/hcsdoc_wcow.go | 3 ++ .../hcsshim/internal/hcsoci/resources_wcow.go | 35 +++++++++++++++++++ 4 files changed, 76 insertions(+) diff --git a/internal/hcsoci/hcsdoc_wcow.go b/internal/hcsoci/hcsdoc_wcow.go index f27b5da27e..0482b07773 100644 --- a/internal/hcsoci/hcsdoc_wcow.go +++ b/internal/hcsoci/hcsdoc_wcow.go @@ -75,6 +75,9 @@ func createMountsConfig(ctx context.Context, coi *createOptionsInternal) (*mount return nil, err } mdv2.HostPath = uvmPath + } else if strings.HasPrefix(mount.Source, "sandbox://") { + // Convert to the path in the guest that was asked for. + mdv2.HostPath = convertToWCOWSandboxMountPath(mount.Source) } else { // vsmb mount uvmPath, err := coi.HostingSystem.GetVSMBUvmPath(ctx, mount.Source, readOnly) diff --git a/internal/hcsoci/resources_wcow.go b/internal/hcsoci/resources_wcow.go index adfaee786a..0a2fef6fc8 100644 --- a/internal/hcsoci/resources_wcow.go +++ b/internal/hcsoci/resources_wcow.go @@ -5,12 +5,14 @@ package hcsoci // Contains functions relating to a WCOW container, as opposed to a utility VM import ( + "bytes" "context" "fmt" "os" "path/filepath" "strings" + "github.com/Microsoft/hcsshim/internal/cmd" "github.com/Microsoft/hcsshim/internal/credentials" "github.com/Microsoft/hcsshim/internal/devices" "github.com/Microsoft/hcsshim/internal/layers" @@ -23,6 +25,8 @@ import ( "github.com/pkg/errors" ) +const wcowSandboxMountPath = "C:\\SandboxMounts" + func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, isSandbox bool) error { if coi.Spec == nil || coi.Spec.Windows == nil || coi.Spec.Windows.LayerFolders == nil { return errors.New("field 'Spec.Windows.Layerfolders' is not populated") @@ -179,6 +183,32 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R return errors.Wrapf(err, "adding SCSI EVD mount failed %+v", mount) } r.Add(scsiMount) + } else if strings.HasPrefix(mount.Source, "sandbox://") { + // Mounts that map to a path in the UVM are specified with a 'sandbox://' prefix. + // + // Example: sandbox:///a/dirInUvm destination:C:\\dirInContainer. + // + // so first convert to a path in the sandboxmounts path itself. + sandboxPath := convertToWCOWSandboxMountPath(mount.Source) + + // Now we need to exec a process in the vm that will make these directories as theres + // no functionality in the Windows gcs to create an arbitrary directory. + // + // Create the directory, but also run dir afterwards regardless of if mkdir succeeded to handle the case where the directory already exists + // e.g. from a previous container specifying the same mount (and thus creating the same directory). + b := &bytes.Buffer{} + stderr, err := cmd.CreatePipeAndListen(b, false) + if err != nil { + return err + } + req := &cmd.CmdProcessRequest{ + Args: []string{"cmd", "/c", "mkdir", sandboxPath, "&", "dir", sandboxPath}, + Stderr: stderr, + } + exitCode, err := cmd.ExecInUvm(ctx, coi.HostingSystem, req) + if err != nil { + return errors.Wrapf(err, "failed to create sandbox mount directory in utility VM with exit code %d %q", exitCode, b.String()) + } } else { if uvm.IsPipe(mount.Source) { pipe, err := coi.HostingSystem.AddPipe(ctx, mount.Source) @@ -201,3 +231,8 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R return nil } + +func convertToWCOWSandboxMountPath(source string) string { + subPath := strings.TrimPrefix(source, "sandbox://") + return filepath.Join(wcowSandboxMountPath, subPath) +} diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go index f27b5da27e..0482b07773 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go @@ -75,6 +75,9 @@ func createMountsConfig(ctx context.Context, coi *createOptionsInternal) (*mount return nil, err } mdv2.HostPath = uvmPath + } else if strings.HasPrefix(mount.Source, "sandbox://") { + // Convert to the path in the guest that was asked for. + mdv2.HostPath = convertToWCOWSandboxMountPath(mount.Source) } else { // vsmb mount uvmPath, err := coi.HostingSystem.GetVSMBUvmPath(ctx, mount.Source, readOnly) diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go index adfaee786a..0a2fef6fc8 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go @@ -5,12 +5,14 @@ package hcsoci // Contains functions relating to a WCOW container, as opposed to a utility VM import ( + "bytes" "context" "fmt" "os" "path/filepath" "strings" + "github.com/Microsoft/hcsshim/internal/cmd" "github.com/Microsoft/hcsshim/internal/credentials" "github.com/Microsoft/hcsshim/internal/devices" "github.com/Microsoft/hcsshim/internal/layers" @@ -23,6 +25,8 @@ import ( "github.com/pkg/errors" ) +const wcowSandboxMountPath = "C:\\SandboxMounts" + func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, isSandbox bool) error { if coi.Spec == nil || coi.Spec.Windows == nil || coi.Spec.Windows.LayerFolders == nil { return errors.New("field 'Spec.Windows.Layerfolders' is not populated") @@ -179,6 +183,32 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R return errors.Wrapf(err, "adding SCSI EVD mount failed %+v", mount) } r.Add(scsiMount) + } else if strings.HasPrefix(mount.Source, "sandbox://") { + // Mounts that map to a path in the UVM are specified with a 'sandbox://' prefix. + // + // Example: sandbox:///a/dirInUvm destination:C:\\dirInContainer. + // + // so first convert to a path in the sandboxmounts path itself. + sandboxPath := convertToWCOWSandboxMountPath(mount.Source) + + // Now we need to exec a process in the vm that will make these directories as theres + // no functionality in the Windows gcs to create an arbitrary directory. + // + // Create the directory, but also run dir afterwards regardless of if mkdir succeeded to handle the case where the directory already exists + // e.g. from a previous container specifying the same mount (and thus creating the same directory). + b := &bytes.Buffer{} + stderr, err := cmd.CreatePipeAndListen(b, false) + if err != nil { + return err + } + req := &cmd.CmdProcessRequest{ + Args: []string{"cmd", "/c", "mkdir", sandboxPath, "&", "dir", sandboxPath}, + Stderr: stderr, + } + exitCode, err := cmd.ExecInUvm(ctx, coi.HostingSystem, req) + if err != nil { + return errors.Wrapf(err, "failed to create sandbox mount directory in utility VM with exit code %d %q", exitCode, b.String()) + } } else { if uvm.IsPipe(mount.Source) { pipe, err := coi.HostingSystem.AddPipe(ctx, mount.Source) @@ -201,3 +231,8 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R return nil } + +func convertToWCOWSandboxMountPath(source string) string { + subPath := strings.TrimPrefix(source, "sandbox://") + return filepath.Join(wcowSandboxMountPath, subPath) +} From 92004da684484c39ade5aa7d2c0b556d56e77fa4 Mon Sep 17 00:00:00 2001 From: Daniel Canter Date: Fri, 24 Sep 2021 02:37:59 -0700 Subject: [PATCH 2/2] Add WCOW sandbox mount tests This change adds two cri-containerd tests to test WCOW sandbox mounts. One test verifies the general functionality by having two containers supply the same sandbox:// mount and validating that each container can see whats in the mount. Another tests verifies that if we don't supply the sandbox:/// mount for another container in the pod, it doesn't have access to the mount. Signed-off-by: Daniel Canter --- test/cri-containerd/runpodsandbox_test.go | 124 ++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/test/cri-containerd/runpodsandbox_test.go b/test/cri-containerd/runpodsandbox_test.go index f5090f43ec..fec2650778 100644 --- a/test/cri-containerd/runpodsandbox_test.go +++ b/test/cri-containerd/runpodsandbox_test.go @@ -853,6 +853,130 @@ func Test_RunPodSandbox_Mount_SandboxDir_LCOW(t *testing.T) { //TODO: Parse the output of the exec command to make sure the uvm mount was successful } +func Test_RunPodSandbox_Mount_SandboxDir_WCOW(t *testing.T) { + requireFeatures(t, featureWCOWHypervisor) + + pullRequiredImages(t, []string{imageWindowsNanoserver}) + + client := newTestRuntimeClient(t) + ctx := context.Background() + + sbRequest := getRunPodSandboxRequest(t, wcowHypervisorRuntimeHandler, nil) + podID := runPodSandbox(t, client, ctx, sbRequest) + defer removePodSandbox(t, client, ctx, podID) + defer stopPodSandbox(t, client, ctx, podID) + + command := []string{ + "cmd", + "/c", + "ping", + "-t", + "127.0.0.1", + } + + mounts := []*runtime.Mount{ + { + HostPath: "sandbox:///test", + ContainerPath: "C:\\test", + }, + } + // Create 2 containers with sandbox mounts and verify both can write and see the others files + container1Name := t.Name() + "-Container-" + "1" + container1Id := createContainerInSandbox(t, client, ctx, podID, container1Name, imageWindowsNanoserver, command, nil, mounts, sbRequest.Config) + defer removeContainer(t, client, ctx, container1Id) + + startContainer(t, client, ctx, container1Id) + defer stopContainer(t, client, ctx, container1Id) + + execEcho := []string{ + "cmd", + "/c", + "echo", + `"test"`, + ">", + "C:\\test\\test.txt", + } + _, errorMsg, exitCode := execContainer(t, client, ctx, container1Id, execEcho) + if exitCode != 0 { + t.Fatalf("Exec into container failed with: %v and exit code: %d, %s", errorMsg, exitCode, container1Id) + } + + container2Name := t.Name() + "-Container-" + "2" + container2Id := createContainerInSandbox(t, client, ctx, podID, container2Name, imageWindowsNanoserver, command, nil, mounts, sbRequest.Config) + defer removeContainer(t, client, ctx, container2Id) + + startContainer(t, client, ctx, container2Id) + defer stopContainer(t, client, ctx, container2Id) + + // Test that we can see the file made in the first container in the second one. + execDir := []string{ + "cmd", + "/c", + "dir", + "C:\\test\\test.txt", + } + _, errorMsg, exitCode = execContainer(t, client, ctx, container2Id, execDir) + if exitCode != 0 { + t.Fatalf("Exec into container failed with: %v and exit code: %d, %s", errorMsg, exitCode, container2Id) + } +} + +func Test_RunPodSandbox_Mount_SandboxDir_NoShare_WCOW(t *testing.T) { + requireFeatures(t, featureWCOWHypervisor) + + pullRequiredImages(t, []string{imageWindowsNanoserver}) + + client := newTestRuntimeClient(t) + ctx := context.Background() + + sbRequest := getRunPodSandboxRequest(t, wcowHypervisorRuntimeHandler, nil) + podID := runPodSandbox(t, client, ctx, sbRequest) + defer removePodSandbox(t, client, ctx, podID) + defer stopPodSandbox(t, client, ctx, podID) + + command := []string{ + "cmd", + "/c", + "ping", + "-t", + "127.0.0.1", + } + + mounts := []*runtime.Mount{ + { + HostPath: "sandbox:///test", + ContainerPath: "C:\\test", + }, + } + // This test case is making sure that the sandbox mount doesn't show up in another container if not + // explicitly asked for. Make first container with the mount and another shortly after without. + container1Name := t.Name() + "-Container-" + "1" + container1Id := createContainerInSandbox(t, client, ctx, podID, container1Name, imageWindowsNanoserver, command, nil, mounts, sbRequest.Config) + defer removeContainer(t, client, ctx, container1Id) + + startContainer(t, client, ctx, container1Id) + defer stopContainer(t, client, ctx, container1Id) + + container2Name := t.Name() + "-Container-" + "2" + container2Id := createContainerInSandbox(t, client, ctx, podID, container2Name, imageWindowsNanoserver, command, nil, nil, sbRequest.Config) + defer removeContainer(t, client, ctx, container2Id) + + startContainer(t, client, ctx, container2Id) + defer stopContainer(t, client, ctx, container2Id) + + // Test that we can't see the file made in the first container in the second one. + execDir := []string{ + "cmd", + "/c", + "dir", + "C:\\test\\", + } + output, _, exitCode := execContainer(t, client, ctx, container2Id, execDir) + if exitCode == 0 { + t.Fatalf("Found directory in second container when not expected: %s", output) + } +} + func Test_RunPodSandbox_CPUGroup(t *testing.T) { testutilities.RequiresBuild(t, 20124) ctx := context.Background()