From 7ee3448700c307b3a6268bc3f1bc8138c15c200a Mon Sep 17 00:00:00 2001 From: Maksim An Date: Thu, 9 Dec 2021 12:31:48 -0800 Subject: [PATCH 1/3] Add new gcs hooks, add expected mounts to security policy Introduce a new `wait-paths` binary, which polls file system until requested paths are available or a timeout is reached. Security policy has been updated to have ExpectedMounts entries, which will be used in conjunction with "wait-paths" hook for synchronization purposes. Refactor oci-hook logic into its own internal package and update existing code to use that package. Copy runc HookName and constants definitions to break dependency on runc Introduce ExpectedMounts as part of security policy language and the logic to enforce the policy, which resolves the expected mounts in the UVM and adds a wait-paths hook to the spec. Signed-off-by: Maksim An --- Makefile | 4 +- cmd/gcs/main.go | 16 ++- cmd/hooks/wait-paths/main.go | 72 ++++++++++ internal/devices/drivers.go | 3 +- internal/guest/runtime/hcsv2/nvidia_utils.go | 45 +++--- .../guest/runtime/hcsv2/sandbox_container.go | 10 +- internal/guest/runtime/hcsv2/spec.go | 22 +-- .../runtime/hcsv2/standalone_container.go | 8 +- internal/guest/runtime/hcsv2/uvm.go | 8 +- .../guest/runtime/hcsv2/workload_container.go | 18 +-- .../mountmonitoringsecuritypolicyenforcer.go | 6 + internal/guestpath/paths.go | 24 ++++ internal/hcsoci/create.go | 5 +- internal/hcsoci/devices.go | 10 +- internal/hcsoci/hcsdoc_wcow.go | 8 +- internal/hcsoci/resources_lcow.go | 44 +++--- internal/hcsoci/resources_wcow.go | 12 +- internal/hooks/spec.go | 53 +++++++ internal/layers/layers.go | 14 +- internal/tools/securitypolicy/README.md | 26 +++- .../tools/securitypolicy/helpers/helpers.go | 1 + internal/tools/securitypolicy/main.go | 1 - internal/uvm/constants.go | 13 -- pkg/annotations/annotations.go | 2 +- pkg/securitypolicy/securitypolicy.go | 53 +++++-- pkg/securitypolicy/securitypolicyenforcer.go | 132 +++++++++++++++++- test/cri-containerd/container_test.go | 7 +- test/cri-containerd/policy_test.go | 1 + .../github.com/Microsoft/hcsshim/Makefile | 4 +- .../hcsshim/internal/devices/drivers.go | 3 +- .../hcsshim/internal/guestpath/paths.go | 24 ++++ .../hcsshim/internal/hcsoci/create.go | 5 +- .../hcsshim/internal/hcsoci/devices.go | 10 +- .../hcsshim/internal/hcsoci/hcsdoc_wcow.go | 8 +- .../hcsshim/internal/hcsoci/resources_lcow.go | 44 +++--- .../hcsshim/internal/hcsoci/resources_wcow.go | 12 +- .../Microsoft/hcsshim/internal/hooks/spec.go | 53 +++++++ .../hcsshim/internal/layers/layers.go | 14 +- .../tools/securitypolicy/helpers/helpers.go | 1 + .../hcsshim/internal/uvm/constants.go | 13 -- .../hcsshim/pkg/annotations/annotations.go | 2 +- .../pkg/securitypolicy/securitypolicy.go | 53 +++++-- .../securitypolicy/securitypolicyenforcer.go | 132 +++++++++++++++++- test/vendor/modules.txt | 2 + 44 files changed, 782 insertions(+), 216 deletions(-) create mode 100644 cmd/hooks/wait-paths/main.go create mode 100644 internal/guestpath/paths.go create mode 100644 internal/hooks/spec.go create mode 100644 test/vendor/github.com/Microsoft/hcsshim/internal/guestpath/paths.go create mode 100644 test/vendor/github.com/Microsoft/hcsshim/internal/hooks/spec.go diff --git a/Makefile b/Makefile index ac00def635..c09b1e192a 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ clean: test: cd $(SRCROOT) && go test -v ./internal/guest/... -out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools Makefile +out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools bin/cmd/hooks/wait-paths Makefile @mkdir -p out rm -rf rootfs mkdir -p rootfs/bin/ @@ -40,6 +40,7 @@ out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools Makefile cp bin/vsockexec rootfs/bin/ cp bin/cmd/gcs rootfs/bin/ cp bin/cmd/gcstools rootfs/bin/ + cp bin/cmd/hooks/wait-paths rootfs/bin/ for tool in $(GCS_TOOLS); do ln -s gcstools rootfs/bin/$$tool; done git -C $(SRCROOT) rev-parse HEAD > rootfs/gcs.commit && \ git -C $(SRCROOT) rev-parse --abbrev-ref HEAD > rootfs/gcs.branch @@ -60,6 +61,7 @@ out/initrd.img: $(BASE) out/delta.tar.gz $(SRCROOT)/hack/catcpio.sh -include deps/cmd/gcs.gomake -include deps/cmd/gcstools.gomake +-include deps/cmd/hooks/wait-paths.gomake # Implicit rule for includes that define Go targets. %.gomake: $(SRCROOT)/Makefile diff --git a/cmd/gcs/main.go b/cmd/gcs/main.go index 0926ef4d4b..13f3201fe7 100644 --- a/cmd/gcs/main.go +++ b/cmd/gcs/main.go @@ -14,20 +14,22 @@ import ( "syscall" "time" + "github.com/containerd/cgroups" + cgroupstats "github.com/containerd/cgroups/stats/v1" + oci "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "go.opencensus.io/trace" + "github.com/Microsoft/hcsshim/internal/guest/bridge" "github.com/Microsoft/hcsshim/internal/guest/kmsg" "github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2" "github.com/Microsoft/hcsshim/internal/guest/runtime/runc" "github.com/Microsoft/hcsshim/internal/guest/transport" + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oc" "github.com/cenkalti/backoff/v4" - "github.com/containerd/cgroups" - cgroupstats "github.com/containerd/cgroups/stats/v1" - oci "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "go.opencensus.io/trace" ) func memoryLogFormat(metrics *cgroupstats.Metrics) logrus.Fields { @@ -229,7 +231,7 @@ func main() { log.SetScrubbing(*scrubLogs) - baseLogPath := "/run/gcs/c" + baseLogPath := guestpath.LCOWRootPrefixInUVM logrus.Info("GCS started") diff --git a/cmd/hooks/wait-paths/main.go b/cmd/hooks/wait-paths/main.go new file mode 100644 index 0000000000..c3c49ccf9a --- /dev/null +++ b/cmd/hooks/wait-paths/main.go @@ -0,0 +1,72 @@ +// +build linux + +package main + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +const ( + pathsFlag = "paths" + timeoutFlag = "timeout" +) + +// This is a hook that waits for a specific path to appear. +// The hook has required list of comma-separated paths and a default timeout in seconds. + +func main() { + app := cli.NewApp() + app.Name = "wait-paths" + app.Usage = "Provide a list paths and an optional timeout" + app.Flags = []cli.Flag{ + cli.StringFlag{ + Name: pathsFlag + ",p", + Usage: "Comma-separated list of paths that should become available", + Required: true, + }, + cli.IntFlag{ + Name: timeoutFlag + ",t", + Usage: "Timeout in seconds", + Value: 30, + }, + } + app.Action = run + if err := app.Run(os.Args); err != nil { + logrus.Fatalf("%s\n", err) + } + os.Exit(0) +} + +func run(cCtx *cli.Context) error { + timeout := cCtx.GlobalInt(timeoutFlag) + paths := strings.Split(cCtx.GlobalString(pathsFlag), ",") + + waitCtx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) + defer cancel() + + for _, path := range paths { + for { + if _, err := os.Stat(path); err != nil { + if !os.IsNotExist(err) { + return err + } + select { + case <-waitCtx.Done(): + return fmt.Errorf("timeout while waiting for path %q to appear", path) + default: + time.Sleep(time.Millisecond * 10) + continue + } + } + break + } + } + return nil +} diff --git a/internal/devices/drivers.go b/internal/devices/drivers.go index 4efe677e22..0dae33c963 100644 --- a/internal/devices/drivers.go +++ b/internal/devices/drivers.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/Microsoft/hcsshim/internal/cmd" + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" @@ -45,7 +46,7 @@ func InstallKernelDriver(ctx context.Context, vm *uvm.UtilityVM, driver string) } return closer, execPnPInstallDriver(ctx, vm, uvmPath) } - uvmPathForShare := fmt.Sprintf(uvm.LCOWGlobalMountPrefix, vm.UVMMountCounter()) + uvmPathForShare := fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, vm.UVMMountCounter()) scsiCloser, err := vm.AddSCSI(ctx, driver, uvmPathForShare, true, false, []string{}, uvm.VMAccessTypeIndividual) if err != nil { return closer, fmt.Errorf("failed to add SCSI disk to utility VM for path %+v: %s", driver, err) diff --git a/internal/guest/runtime/hcsv2/nvidia_utils.go b/internal/guest/runtime/hcsv2/nvidia_utils.go index 84cae4718b..735bb4931d 100644 --- a/internal/guest/runtime/hcsv2/nvidia_utils.go +++ b/internal/guest/runtime/hcsv2/nvidia_utils.go @@ -10,17 +10,16 @@ import ( "os/exec" "strings" + oci "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/Microsoft/hcsshim/cmd/gcstools/generichook" "github.com/Microsoft/hcsshim/internal/guest/storage/pci" + "github.com/Microsoft/hcsshim/internal/guestpath" + "github.com/Microsoft/hcsshim/internal/hooks" "github.com/Microsoft/hcsshim/pkg/annotations" - oci "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" ) -// path that the shim mounts the nvidia gpu vhd to in the uvm -// this MUST match the path mapped to in the shim -const lcowNvidiaMountPath = "/run/nvidia" - const nvidiaDebugFilePath = "/nvidia-container.log" const nvidiaToolBinary = "nvidia-container-cli" @@ -70,35 +69,33 @@ func addNvidiaDevicePreHook(ctx context.Context, spec *oci.Spec) error { // add template for pid argument to be injected later by the generic hook binary args = append(args, "--no-cgroups", "--pid={{pid}}", spec.Root.Path) - if spec.Hooks == nil { - spec.Hooks = &oci.Hooks{} - } - hookLogDebugFileEnvOpt := fmt.Sprintf("%s=%s", generichook.LogDebugFileEnvKey, nvidiaDebugFilePath) hookEnv := append(updateEnvWithNvidiaVariables(), hookLogDebugFileEnvOpt) - nvidiaHook := oci.Hook{ - Path: genericHookPath, - Args: args, - Env: hookEnv, - } - - spec.Hooks.Prestart = append(spec.Hooks.Prestart, nvidiaHook) - return nil + nvidiaHook := hooks.NewOCIHook(genericHookPath, args, hookEnv) + return hooks.AddOCIHook(spec, hooks.Prestart, nvidiaHook) } // updateEnvWithNvidiaVariables creates an env with the nvidia gpu vhd in PATH and insecure mode set func updateEnvWithNvidiaVariables() []string { + nvidiaBin := fmt.Sprintf("%s/bin", guestpath.LCOWNvidiaMountPath) + env := updatePathEnv(nvidiaBin) + // NVC_INSECURE_MODE allows us to run nvidia-container-cli without seccomp + // we don't currently use seccomp in the uvm, so avoid using it here for now as well + env = append(env, "NVC_INSECURE_MODE=1") + return env +} + +// updatePathEnv adds specified `dirs` to PATH variable and returns the result environment variables. +func updatePathEnv(dirs ...string) []string { pathPrefix := "PATH=" - nvidiaBin := fmt.Sprintf("%s/bin", lcowNvidiaMountPath) + additionalDirs := strings.Join(dirs, ":") env := os.Environ() for i, v := range env { if strings.HasPrefix(v, pathPrefix) { - newPath := fmt.Sprintf("%s:%s", v, nvidiaBin) + newPath := fmt.Sprintf("%s:%s", v, additionalDirs) env[i] = newPath + return env } } - // NVC_INSECURE_MODE allows us to run nvidia-container-cli without seccomp - // we don't currently use seccomp in the uvm, so avoid using it here for now as well - env = append(env, "NVC_INSECURE_MODE=1") - return env + return append(env, fmt.Sprintf("PATH=%s", additionalDirs)) } diff --git a/internal/guest/runtime/hcsv2/sandbox_container.go b/internal/guest/runtime/hcsv2/sandbox_container.go index ddfbace177..c76b11d3ce 100644 --- a/internal/guest/runtime/hcsv2/sandbox_container.go +++ b/internal/guest/runtime/hcsv2/sandbox_container.go @@ -10,16 +10,18 @@ import ( "path/filepath" "strings" - "github.com/Microsoft/hcsshim/internal/guest/network" - "github.com/Microsoft/hcsshim/internal/oc" - "github.com/Microsoft/hcsshim/pkg/annotations" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "go.opencensus.io/trace" + + "github.com/Microsoft/hcsshim/internal/guest/network" + "github.com/Microsoft/hcsshim/internal/guestpath" + "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/pkg/annotations" ) func getSandboxRootDir(id string) string { - return filepath.Join("/run/gcs/c", id) + return filepath.Join(guestpath.LCOWRootPrefixInUVM, id) } func getSandboxHugePageMountsDir(id string) string { diff --git a/internal/guest/runtime/hcsv2/spec.go b/internal/guest/runtime/hcsv2/spec.go index a0be478804..5241ba6431 100644 --- a/internal/guest/runtime/hcsv2/spec.go +++ b/internal/guest/runtime/hcsv2/spec.go @@ -10,12 +10,14 @@ import ( "strconv" "strings" - "github.com/Microsoft/hcsshim/internal/log" - "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/opencontainers/runc/libcontainer/devices" "github.com/opencontainers/runc/libcontainer/user" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" + + "github.com/Microsoft/hcsshim/internal/hooks" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/pkg/annotations" ) // getNetworkNamespaceID returns the `ToLower` of @@ -257,17 +259,7 @@ func applyAnnotationsToSpec(ctx context.Context, spec *oci.Spec) error { } // Helper function to create an oci prestart hook to run ldconfig -func addLDConfigHook(ctx context.Context, spec *oci.Spec, args, env []string) error { - if spec.Hooks == nil { - spec.Hooks = &oci.Hooks{} - } - - ldConfigHook := oci.Hook{ - Path: "/sbin/ldconfig", - Args: args, - Env: env, - } - - spec.Hooks.Prestart = append(spec.Hooks.Prestart, ldConfigHook) - return nil +func addLDConfigHook(_ context.Context, spec *oci.Spec, args, env []string) error { + ldConfigHook := hooks.NewOCIHook("/sbin/ldconfig", args, env) + return hooks.AddOCIHook(spec, hooks.Prestart, ldConfigHook) } diff --git a/internal/guest/runtime/hcsv2/standalone_container.go b/internal/guest/runtime/hcsv2/standalone_container.go index 8ea0275cac..89189313e8 100644 --- a/internal/guest/runtime/hcsv2/standalone_container.go +++ b/internal/guest/runtime/hcsv2/standalone_container.go @@ -10,15 +10,17 @@ import ( "path/filepath" "strings" - "github.com/Microsoft/hcsshim/internal/guest/network" - "github.com/Microsoft/hcsshim/internal/oc" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "go.opencensus.io/trace" + + "github.com/Microsoft/hcsshim/internal/guest/network" + "github.com/Microsoft/hcsshim/internal/guestpath" + "github.com/Microsoft/hcsshim/internal/oc" ) func getStandaloneRootDir(id string) string { - return filepath.Join("/run/gcs/c", id) + return filepath.Join(guestpath.LCOWRootPrefixInUVM, id) } func getStandaloneHostnamePath(id string) string { diff --git a/internal/guest/runtime/hcsv2/uvm.go b/internal/guest/runtime/hcsv2/uvm.go index 7de89dc9e3..e72464c0d0 100644 --- a/internal/guest/runtime/hcsv2/uvm.go +++ b/internal/guest/runtime/hcsv2/uvm.go @@ -16,7 +16,7 @@ import ( "syscall" "time" - shellwords "github.com/mattn/go-shellwords" + "github.com/mattn/go-shellwords" "github.com/pkg/errors" "github.com/Microsoft/hcsshim/internal/guest/gcserr" @@ -225,12 +225,16 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM // We append the variable after the security policy enforcing logic completes so as to bypass it; the // security policy variable cannot be included in the security policy as its value is not available // security policy construction time. - if policyEnforcer, ok := (h.securityPolicyEnforcer).(*securitypolicy.StandardSecurityPolicyEnforcer); ok { secPolicyEnv := fmt.Sprintf("SECURITY_POLICY=%s", policyEnforcer.EncodedSecurityPolicy) settings.OCISpecification.Process.Env = append(settings.OCISpecification.Process.Env, secPolicyEnv) } + // Sandbox mount paths need to be resolved in the spec before expected mounts policy can be enforced. + if err = h.securityPolicyEnforcer.EnforceExpectedMountsPolicy(id, settings.OCISpecification); err != nil { + return nil, errors.Wrapf(err, "container creation denied due to policy") + } + // Create the BundlePath if err := os.MkdirAll(settings.OCIBundlePath, 0700); err != nil { return nil, errors.Wrapf(err, "failed to create OCIBundlePath: '%s'", settings.OCIBundlePath) diff --git a/internal/guest/runtime/hcsv2/workload_container.go b/internal/guest/runtime/hcsv2/workload_container.go index d271c162ef..b9679d9036 100644 --- a/internal/guest/runtime/hcsv2/workload_container.go +++ b/internal/guest/runtime/hcsv2/workload_container.go @@ -9,16 +9,18 @@ import ( "path/filepath" "strings" - "github.com/Microsoft/hcsshim/internal/oc" - "github.com/Microsoft/hcsshim/pkg/annotations" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "go.opencensus.io/trace" "golang.org/x/sys/unix" + + "github.com/Microsoft/hcsshim/internal/guestpath" + "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/pkg/annotations" ) func getWorkloadRootDir(id string) string { - return filepath.Join("/run/gcs/c", id) + return filepath.Join(guestpath.LCOWRootPrefixInUVM, id) } // os.MkdirAll combines the given permissions with the running process's @@ -32,11 +34,10 @@ func mkdirAllModePerm(target string) error { } func updateSandboxMounts(sbid string, spec *oci.Spec) error { - sandboxMountPrefix := "sandbox://" for i, m := range spec.Mounts { - if strings.HasPrefix(m.Source, sandboxMountPrefix) { + if strings.HasPrefix(m.Source, guestpath.SandboxMountPrefix) { mountsDir := getSandboxMountsDir(sbid) - subPath := strings.TrimPrefix(m.Source, sandboxMountPrefix) + subPath := strings.TrimPrefix(m.Source, guestpath.SandboxMountPrefix) sandboxSource := filepath.Join(mountsDir, subPath) // filepath.Join cleans the resulting path before returning so it would resolve the relative path if one was given. @@ -59,11 +60,10 @@ func updateSandboxMounts(sbid string, spec *oci.Spec) error { } func updateHugePageMounts(sbid string, spec *oci.Spec) error { - mountPrefix := "hugepages://" for i, m := range spec.Mounts { - if strings.HasPrefix(m.Source, mountPrefix) { + if strings.HasPrefix(m.Source, guestpath.HugePagesMountPrefix) { mountsDir := getSandboxHugePageMountsDir(sbid) - subPath := strings.TrimPrefix(m.Source, mountPrefix) + subPath := strings.TrimPrefix(m.Source, guestpath.HugePagesMountPrefix) pageSize := strings.Split(subPath, string(os.PathSeparator))[0] hugePageMountSource := filepath.Join(mountsDir, subPath) diff --git a/internal/guest/storage/test/policy/mountmonitoringsecuritypolicyenforcer.go b/internal/guest/storage/test/policy/mountmonitoringsecuritypolicyenforcer.go index bb06d465a8..aa34077fcb 100644 --- a/internal/guest/storage/test/policy/mountmonitoringsecuritypolicyenforcer.go +++ b/internal/guest/storage/test/policy/mountmonitoringsecuritypolicyenforcer.go @@ -1,6 +1,8 @@ package policy import ( + oci "github.com/opencontainers/runtime-spec/specs-go" + "github.com/Microsoft/hcsshim/pkg/securitypolicy" ) @@ -32,3 +34,7 @@ func (p *MountMonitoringSecurityPolicyEnforcer) EnforceOverlayMountPolicy(contai func (p *MountMonitoringSecurityPolicyEnforcer) EnforceCreateContainerPolicy(_ string, _ []string, _ []string, _ string) (err error) { return nil } + +func (p *MountMonitoringSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(_ string, _ *oci.Spec) error { + return nil +} diff --git a/internal/guestpath/paths.go b/internal/guestpath/paths.go new file mode 100644 index 0000000000..62bbfeb636 --- /dev/null +++ b/internal/guestpath/paths.go @@ -0,0 +1,24 @@ +package guestpath + +const ( + // LCOWNvidiaMountPath is the path format in LCOW UVM where nvidia tools are mounted + // keep this value in sync with opengcs + LCOWNvidiaMountPath = "/run/nvidia" + // LCOWRootPrefixInUVM is the path inside UVM where LCOW container's root file system will be mounted + LCOWRootPrefixInUVM = "/run/gcs/c" + // WCOWRootPrefixInUVM is the path inside UVM where WCOW container's root file system will be mounted + WCOWRootPrefixInUVM = `C:\c` + // SandboxMountPrefix is mount prefix used in container spec to mark a sandbox-mount + SandboxMountPrefix = "sandbox://" + // HugePagesMountPrefix is mount prefix used in container spec to mark a huge-pages mount + HugePagesMountPrefix = "hugepages://" + // LCOWMountPathPrefixFmt is the path format in the LCOW UVM where non global mounts, such + // as Plan9 mounts are added + LCOWMountPathPrefixFmt = "/mounts/m%d" + // LCOWGlobalMountPrefixFmt is the path format in the LCOW UVM where global mounts are added + LCOWGlobalMountPrefixFmt = "/run/mounts/m%d" + // WCOWGlobalMountPrefixFmt is the path prefix format in the WCOW UVM where mounts are added + WCOWGlobalMountPrefixFmt = "C:\\mounts\\m%d" + // RootfsPath is part of the container's rootfs path + RootfsPath = "rootfs" +) diff --git a/internal/hcsoci/create.go b/internal/hcsoci/create.go index 16821a1861..058530aac1 100644 --- a/internal/hcsoci/create.go +++ b/internal/hcsoci/create.go @@ -14,6 +14,7 @@ import ( "github.com/Microsoft/go-winio/pkg/guid" "github.com/Microsoft/hcsshim/internal/clone" "github.com/Microsoft/hcsshim/internal/cow" + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/hcs" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" @@ -27,8 +28,8 @@ import ( ) var ( - lcowRootInUVM = "/run/gcs/c/%s" - wcowRootInUVM = `C:\c\%s` + lcowRootInUVM = guestpath.LCOWRootPrefixInUVM + "/%s" + wcowRootInUVM = guestpath.WCOWRootPrefixInUVM + "/%s" ) // CreateOptions are the set of fields used to call CreateContainer(). diff --git a/internal/hcsoci/devices.go b/internal/hcsoci/devices.go index 42d11aac52..4f17a715d6 100644 --- a/internal/hcsoci/devices.go +++ b/internal/hcsoci/devices.go @@ -9,7 +9,11 @@ import ( "path/filepath" "strconv" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/Microsoft/hcsshim/internal/devices" + "github.com/Microsoft/hcsshim/internal/guestpath" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oci" @@ -17,8 +21,6 @@ import ( "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" ) const deviceUtilExeName = "device-util.exe" @@ -222,14 +224,14 @@ func handleAssignedDevicesLCOW( scsiMount, err := vm.AddSCSI( ctx, gpuSupportVhdPath, - uvm.LCOWNvidiaMountPath, + guestpath.LCOWNvidiaMountPath, true, false, options, uvm.VMAccessTypeNoop, ) if err != nil { - return resultDevs, closers, errors.Wrapf(err, "failed to add scsi device %s in the UVM %s at %s", gpuSupportVhdPath, vm.ID(), uvm.LCOWNvidiaMountPath) + return resultDevs, closers, errors.Wrapf(err, "failed to add scsi device %s in the UVM %s at %s", gpuSupportVhdPath, vm.ID(), guestpath.LCOWNvidiaMountPath) } closers = append(closers, scsiMount) } diff --git a/internal/hcsoci/hcsdoc_wcow.go b/internal/hcsoci/hcsdoc_wcow.go index 688d901609..b3080399a6 100644 --- a/internal/hcsoci/hcsdoc_wcow.go +++ b/internal/hcsoci/hcsdoc_wcow.go @@ -11,6 +11,10 @@ import ( "regexp" "strings" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" + + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/hcs/schema1" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/layers" @@ -22,8 +26,6 @@ import ( "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/sirupsen/logrus" ) // A simple wrapper struct around the container mount configs that should be added to the @@ -78,7 +80,7 @@ func createMountsConfig(ctx context.Context, coi *createOptionsInternal) (*mount return nil, err } mdv2.HostPath = uvmPath - } else if strings.HasPrefix(mount.Source, "sandbox://") { + } else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) { // Convert to the path in the guest that was asked for. mdv2.HostPath = convertToWCOWSandboxMountPath(mount.Source) } else { diff --git a/internal/hcsoci/resources_lcow.go b/internal/hcsoci/resources_lcow.go index 085854a4c7..d438738c28 100644 --- a/internal/hcsoci/resources_lcow.go +++ b/internal/hcsoci/resources_lcow.go @@ -13,12 +13,14 @@ import ( "path/filepath" "strings" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" ) func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, isSandbox bool) error { @@ -39,7 +41,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * // This is the "Plan 9" root filesystem. // TODO: We need a test for this. Ask @jstarks how you can even lay this out on Windows. hostPath := coi.Spec.Root.Path - uvmPathForContainersFileSystem := path.Join(r.ContainerRootInUVM(), uvm.RootfsPath) + uvmPathForContainersFileSystem := path.Join(r.ContainerRootInUVM(), guestpath.RootfsPath) share, err := coi.HostingSystem.AddPlan9(ctx, hostPath, uvmPathForContainersFileSystem, coi.Spec.Root.Readonly, false, nil) if err != nil { return errors.Wrap(err, "adding plan9 root") @@ -65,7 +67,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * if coi.HostingSystem != nil { hostPath := mount.Source - uvmPathForShare := path.Join(containerRootInUVM, fmt.Sprintf(uvm.LCOWMountPathPrefix, i)) + uvmPathForShare := path.Join(containerRootInUVM, fmt.Sprintf(guestpath.LCOWMountPathPrefixFmt, i)) uvmPathForFile := uvmPathForShare readOnly := false @@ -79,7 +81,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * l := log.G(ctx).WithField("mount", fmt.Sprintf("%+v", mount)) if mount.Type == "physical-disk" { l.Debug("hcsshim::allocateLinuxResources Hot-adding SCSI physical disk for OCI mount") - uvmPathForShare = fmt.Sprintf(uvm.LCOWGlobalMountPrefix, coi.HostingSystem.UVMMountCounter()) + uvmPathForShare = fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, coi.HostingSystem.UVMMountCounter()) scsiMount, err := coi.HostingSystem.AddSCSIPhysicalDisk(ctx, hostPath, uvmPathForShare, readOnly, mount.Options) if err != nil { return errors.Wrapf(err, "adding SCSI physical disk mount %+v", mount) @@ -90,7 +92,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * coi.Spec.Mounts[i].Type = "none" } else if mount.Type == "virtual-disk" { l.Debug("hcsshim::allocateLinuxResources Hot-adding SCSI virtual disk for OCI mount") - uvmPathForShare = fmt.Sprintf(uvm.LCOWGlobalMountPrefix, coi.HostingSystem.UVMMountCounter()) + uvmPathForShare = fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, coi.HostingSystem.UVMMountCounter()) // if the scsi device is already attached then we take the uvm path that the function below returns // that is where it was previously mounted in UVM @@ -110,15 +112,19 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * uvmPathForFile = scsiMount.UVMPath r.Add(scsiMount) coi.Spec.Mounts[i].Type = "none" - } else if strings.HasPrefix(mount.Source, "sandbox://") { + } else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) { // Mounts that map to a path in UVM are specified with 'sandbox://' prefix. // example: sandbox:///a/dirInUvm destination:/b/dirInContainer uvmPathForFile = mount.Source - } else if strings.HasPrefix(mount.Source, "hugepages://") { + } else if strings.HasPrefix(mount.Source, guestpath.HugePagesMountPrefix) { // currently we only support 2M hugepage size - hugePageSubDirs := strings.Split(strings.TrimPrefix(mount.Source, "hugepages://"), "/") + hugePageSubDirs := strings.Split(strings.TrimPrefix(mount.Source, guestpath.HugePagesMountPrefix), "/") if len(hugePageSubDirs) < 2 { - return errors.Errorf(`%s mount path is invalid, expected format: hugepages:///`, mount.Source) + return errors.Errorf( + `%s mount path is invalid, expected format: %s/`, + mount.Source, + guestpath.HugePagesMountPrefix, + ) } // hugepages:// should be followed by pagesize @@ -155,15 +161,17 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * } } - if coi.HostingSystem != nil { - if coi.hasWindowsAssignedDevices() { - windowsDevices, closers, err := handleAssignedDevicesLCOW(ctx, coi.HostingSystem, coi.Spec.Annotations, coi.Spec.Windows.Devices) - if err != nil { - return err - } - r.Add(closers...) - coi.Spec.Windows.Devices = windowsDevices + if coi.HostingSystem == nil { + return nil + } + + if coi.hasWindowsAssignedDevices() { + windowsDevices, closers, err := handleAssignedDevicesLCOW(ctx, coi.HostingSystem, coi.Spec.Annotations, coi.Spec.Windows.Devices) + if err != nil { + return err } + r.Add(closers...) + coi.Spec.Windows.Devices = windowsDevices } return nil } diff --git a/internal/hcsoci/resources_wcow.go b/internal/hcsoci/resources_wcow.go index 1150ca5c3c..fa22c8047e 100644 --- a/internal/hcsoci/resources_wcow.go +++ b/internal/hcsoci/resources_wcow.go @@ -13,16 +13,18 @@ import ( "path/filepath" "strings" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/Microsoft/hcsshim/internal/cmd" "github.com/Microsoft/hcsshim/internal/credentials" + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/schemaversion" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/internal/wclayer" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" ) const wcowSandboxMountPath = "C:\\SandboxMounts" @@ -140,7 +142,7 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R } if coi.HostingSystem != nil && schemaversion.IsV21(coi.actualSchemaVersion) { - uvmPath := fmt.Sprintf(uvm.WCOWGlobalMountPrefix, coi.HostingSystem.UVMMountCounter()) + uvmPath := fmt.Sprintf(guestpath.WCOWGlobalMountPrefixFmt, coi.HostingSystem.UVMMountCounter()) readOnly := false for _, o := range mount.Options { if strings.ToLower(o) == "ro" { @@ -178,7 +180,7 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R return errors.Wrapf(err, "adding SCSI EVD mount failed %+v", mount) } r.Add(scsiMount) - } else if strings.HasPrefix(mount.Source, "sandbox://") { + } else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) { // Mounts that map to a path in the UVM are specified with a 'sandbox://' prefix. // // Example: sandbox:///a/dirInUvm destination:C:\\dirInContainer. @@ -228,6 +230,6 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R } func convertToWCOWSandboxMountPath(source string) string { - subPath := strings.TrimPrefix(source, "sandbox://") + subPath := strings.TrimPrefix(source, guestpath.SandboxMountPrefix) return filepath.Join(wcowSandboxMountPath, subPath) } diff --git a/internal/hooks/spec.go b/internal/hooks/spec.go new file mode 100644 index 0000000000..51ba3aa592 --- /dev/null +++ b/internal/hooks/spec.go @@ -0,0 +1,53 @@ +package hooks + +import ( + "fmt" + + oci "github.com/opencontainers/runtime-spec/specs-go" +) + +// Note: The below type definition as well as constants have been copied from +// https://github.com/opencontainers/runc/blob/master/libcontainer/configs/config.go. +// This is done to not introduce a direct dependency on runc, which would complicate +// integration with windows. +type HookName string + +const ( + + // Prestart commands are executed after the container namespaces are created, + // but before the user supplied command is executed from init. + // Note: This hook is now deprecated + // Prestart commands are called in the Runtime namespace. + Prestart HookName = "prestart" + + // CreateRuntime commands MUST be called as part of the create operation after + // the runtime environment has been created but before the pivot_root has been executed. + // CreateRuntime is called immediately after the deprecated Prestart hook. + // CreateRuntime commands are called in the Runtime Namespace. + CreateRuntime HookName = "createRuntime" +) + +// NewOCIHook creates a new oci.Hook with given parameters +func NewOCIHook(path string, args, env []string) oci.Hook { + return oci.Hook{ + Path: path, + Args: args, + Env: env, + } +} + +// AddOCIHook adds oci.Hook of the given hook name to spec +func AddOCIHook(spec *oci.Spec, hn HookName, hk oci.Hook) error { + if spec.Hooks == nil { + spec.Hooks = &oci.Hooks{} + } + switch hn { + case Prestart: + spec.Hooks.Prestart = append(spec.Hooks.Prestart, hk) + case CreateRuntime: + spec.Hooks.CreateRuntime = append(spec.Hooks.CreateRuntime, hk) + default: + return fmt.Errorf("hook %q is not supported", hn) + } + return nil +} diff --git a/internal/layers/layers.go b/internal/layers/layers.go index 573ad72aa7..0334076365 100644 --- a/internal/layers/layers.go +++ b/internal/layers/layers.go @@ -11,15 +11,17 @@ import ( "path/filepath" "time" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" + + "github.com/Microsoft/hcsshim/internal/guestpath" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/hcserror" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/ospath" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/internal/wclayer" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "golang.org/x/sys/windows" ) // ImageLayers contains all the layers for an image. @@ -261,7 +263,7 @@ func MountContainerLayers(ctx context.Context, containerID string, layerFolders err = vm.CombineLayersWCOW(ctx, layers, containerScratchPathInUVM) rootfs = containerScratchPathInUVM } else { - rootfs = ospath.Join(vm.OS(), guestRoot, uvm.RootfsPath) + rootfs = ospath.Join(vm.OS(), guestRoot, guestpath.RootfsPath) err = vm.CombineLayersLCOW(ctx, containerID, lcowUvmLayerPaths, containerScratchPathInUVM, rootfs) } if err != nil { @@ -289,7 +291,7 @@ func addLCOWLayer(ctx context.Context, vm *uvm.UtilityVM, layerPath string) (uvm } options := []string{"ro"} - uvmPath = fmt.Sprintf(uvm.LCOWGlobalMountPrefix, vm.UVMMountCounter()) + uvmPath = fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, vm.UVMMountCounter()) sm, err := vm.AddSCSI(ctx, layerPath, uvmPath, true, false, options, uvm.VMAccessTypeNoop) if err != nil { return "", fmt.Errorf("failed to add SCSI layer: %s", err) @@ -460,7 +462,7 @@ func containerRootfsPath(vm *uvm.UtilityVM, rootPath string) string { if vm.OS() == "windows" { return ospath.Join(vm.OS(), rootPath) } - return ospath.Join(vm.OS(), rootPath, uvm.RootfsPath) + return ospath.Join(vm.OS(), rootPath, guestpath.RootfsPath) } func getScratchVHDPath(layerFolders []string) (string, error) { diff --git a/internal/tools/securitypolicy/README.md b/internal/tools/securitypolicy/README.md index f15ab873e0..4efb7c1632 100644 --- a/internal/tools/securitypolicy/README.md +++ b/internal/tools/securitypolicy/README.md @@ -21,6 +21,7 @@ be downloaded, turned into an ext4, and finally a dm-verity root hash calculated image_name = "rust:1.52.1" command = ["rustc", "--help"] working_dir = "/home/user" +expected_mounts = ["/path/to/container/mount-1", "/path/to/container/mount-2"] [[container.env_rule]] strategy = "re2" @@ -86,7 +87,14 @@ represented in JSON. "5": "1b80f120dbd88e4355d6241b519c3e25290215c469516b49dece9cf07175a766" } }, - "working_dir": "/home/user" + "working_dir": "/home/user", + "expected_mounts": { + "length": 2, + "elements": { + "0": "/path/to/container/mount-1", + "1": "/path/to/container/mount-2" + } + } }, "1": { "command": { @@ -114,7 +122,11 @@ represented in JSON. "0": "16b514057a06ad665f92c02863aca074fd5976c755d26bff16365299169e8415" } }, - "working_dir": "/" + "working_dir": "/", + "expected_mounts": { + "length": 0, + "elements": {} + } } } } @@ -135,11 +147,11 @@ output raw JSON in addition to the Base64 encoded version Some images will be pulled from registries that require authorization. To add authorization information for a given image, you would add an `[auth]` object -to the TOML definiton for that image. For example: +to the TOML definition for that image. For example: ```toml -[[image]] -image_name = "rust:1.52.1" +[[container]] +name = "rust:1.52.1" command = ["rustc", "--help"] [auth] @@ -147,8 +159,8 @@ username = "my username" password = "my password" ``` -Authorization information needs added on a per-image basis as it can vary from -image to image and their respective registries. +Authorization information needs to be added on a per-image basis as it can vary +from image to image and their respective registries. To pull an image using anonymous access, no `[auth]` object is required. diff --git a/internal/tools/securitypolicy/helpers/helpers.go b/internal/tools/securitypolicy/helpers/helpers.go index 614e5c10e8..42f6fd8a70 100644 --- a/internal/tools/securitypolicy/helpers/helpers.go +++ b/internal/tools/securitypolicy/helpers/helpers.go @@ -74,6 +74,7 @@ func DefaultContainerConfigs() []securitypolicy.ContainerConfig { []securitypolicy.EnvRuleConfig{}, securitypolicy.AuthConfig{}, "", + []string{}, ) return []securitypolicy.ContainerConfig{pause} } diff --git a/internal/tools/securitypolicy/main.go b/internal/tools/securitypolicy/main.go index 4285541ea6..4073a5d847 100644 --- a/internal/tools/securitypolicy/main.go +++ b/internal/tools/securitypolicy/main.go @@ -75,7 +75,6 @@ func createPolicyFromConfig(config *securitypolicy.PolicyConfig) (*securitypolic // and any environment variable rules we might need defaultContainers := helpers.DefaultContainerConfigs() config.Containers = append(config.Containers, defaultContainers...) - policyContainers, err := helpers.PolicyContainersFromConfigs(config.Containers) if err != nil { return nil, err diff --git a/internal/uvm/constants.go b/internal/uvm/constants.go index 842a8a5e8c..1ddcf903ab 100644 --- a/internal/uvm/constants.go +++ b/internal/uvm/constants.go @@ -16,19 +16,6 @@ const ( // DefaultVPMemSizeBytes is the default size of a VPMem device if the create request // doesn't specify. DefaultVPMemSizeBytes = 4 * 1024 * 1024 * 1024 // 4GB - - // LCOWMountPathPrefix is the path format in the LCOW UVM where non global mounts, such - // as Plan9 mounts are added - LCOWMountPathPrefix = "/mounts/m%d" - // LCOWGlobalMountPrefix is the path format in the LCOW UVM where global mounts are added - LCOWGlobalMountPrefix = "/run/mounts/m%d" - // LCOWNvidiaMountPath is the path format in LCOW UVM where nvidia tools are mounted - // keep this value in sync with opengcs - LCOWNvidiaMountPath = "/run/nvidia" - // WCOWGlobalMountPrefix is the path prefix format in the WCOW UVM where mounts are added - WCOWGlobalMountPrefix = "C:\\mounts\\m%d" - // RootfsPath is part of the container's rootfs path - RootfsPath = "rootfs" ) var ( diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 7a51b2a0e4..2f06dfb8bd 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -261,7 +261,7 @@ const ( // GuestStateFile specifies the path of the vmgs file to use if required. Only applies in SNP mode. GuestStateFile = "io.microsoft.virtualmachine.lcow.gueststatefile" - // AnnotationDisableLCOWTimeSyncService is used to disable the chronyd time + // DisableLCOWTimeSyncService is used to disable the chronyd time // synchronization service inside the LCOW UVM. DisableLCOWTimeSyncService = "io.microsoft.virtualmachine.lcow.timesync.disable" diff --git a/pkg/securitypolicy/securitypolicy.go b/pkg/securitypolicy/securitypolicy.go index 42ea861024..d7ec30bdd6 100644 --- a/pkg/securitypolicy/securitypolicy.go +++ b/pkg/securitypolicy/securitypolicy.go @@ -38,11 +38,12 @@ type EnvRuleConfig struct { // ContainerConfig contains toml or JSON config for container described // in security policy. type ContainerConfig struct { - ImageName string `json:"image_name" toml:"image_name"` - Command []string `json:"command" toml:"command"` - Auth AuthConfig `json:"auth" toml:"auth"` - EnvRules []EnvRuleConfig `json:"env_rules" toml:"env_rule"` - WorkingDir string `json:"working_dir" toml:"working_dir"` + ImageName string `json:"image_name" toml:"image_name"` + Command []string `json:"command" toml:"command"` + Auth AuthConfig `json:"auth" toml:"auth"` + EnvRules []EnvRuleConfig `json:"env_rules" toml:"env_rule"` + WorkingDir string `json:"working_dir" toml:"working_dir"` + ExpectedMounts []string `json:"expected_mounts" toml:"expected_mounts"` } // NewContainerConfig creates a new ContainerConfig from the given values. @@ -52,13 +53,15 @@ func NewContainerConfig( envRules []EnvRuleConfig, auth AuthConfig, workingDir string, + expectedMounts []string, ) ContainerConfig { return ContainerConfig{ - ImageName: imageName, - Command: command, - EnvRules: envRules, - Auth: auth, - WorkingDir: workingDir, + ImageName: imageName, + Command: command, + EnvRules: envRules, + Auth: auth, + WorkingDir: workingDir, + ExpectedMounts: expectedMounts, } } @@ -97,6 +100,9 @@ type securityPolicyContainer struct { // WorkingDir is a path to container's working directory, which all the processes // will default to. WorkingDir string + // Unordered list of mounts which are expected to be present when the container + // starts + ExpectedMounts []string `json:"expected_mounts"` } // SecurityPolicyState is a structure that holds user supplied policy to enforce @@ -119,7 +125,7 @@ type EncodedSecurityPolicy struct { // security policy for given policy. The security policy is transmitted as json // in an annotation, so we first have to remove the base64 encoding that allows // the JSON based policy to be passed as a string. From there, we decode the -// JSONand setup our security policy struct +// JSON and setup our security policy struct func NewSecurityPolicyState(base64Policy string) (*SecurityPolicyState, error) { // construct an encoded security policy that holds the base64 representation encodedSecurityPolicy := EncodedSecurityPolicy{ @@ -174,10 +180,11 @@ type Containers struct { } type Container struct { - Command CommandArgs `json:"command"` - EnvRules EnvRules `json:"env_rules"` - Layers Layers `json:"layers"` - WorkingDir string `json:"working_dir"` + Command CommandArgs `json:"command"` + EnvRules EnvRules `json:"env_rules"` + Layers Layers `json:"layers"` + WorkingDir string `json:"working_dir"` + ExpectedMounts ExpectedMounts `json:"expected_mounts"` } type Layers struct { @@ -197,6 +204,11 @@ type EnvRules struct { Elements map[string]EnvRuleConfig `json:"elements"` } +type ExpectedMounts struct { + Length int `json:"length"` + Elements map[string]string `json:"elements"` +} + // NewContainer creates a new Container instance from the provided values // or an error if envRules validation fails. func NewContainer(command, layers []string, envRules []EnvRuleConfig, workingDir string) (*Container, error) { @@ -312,3 +324,14 @@ func (e EnvRules) MarshalJSON() ([]byte, error) { Alias: (*Alias)(&e), }) } + +func (em ExpectedMounts) MarshalJSON() ([]byte, error) { + type Alias ExpectedMounts + return json.Marshal(&struct { + Length int `json:"length"` + *Alias + }{ + Length: len(em.Elements), + Alias: (*Alias)(&em), + }) +} diff --git a/pkg/securitypolicy/securitypolicyenforcer.go b/pkg/securitypolicy/securitypolicyenforcer.go index d5daccbb98..3024f8f5cc 100644 --- a/pkg/securitypolicy/securitypolicyenforcer.go +++ b/pkg/securitypolicy/securitypolicyenforcer.go @@ -3,11 +3,19 @@ package securitypolicy import ( "errors" "fmt" + "os" + "path/filepath" "regexp" "strconv" + "strings" "sync" "github.com/google/go-cmp/cmp" + oci "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/Microsoft/hcsshim/internal/guestpath" + "github.com/Microsoft/hcsshim/internal/hooks" + "github.com/Microsoft/hcsshim/pkg/annotations" ) type SecurityPolicyEnforcer interface { @@ -15,6 +23,7 @@ type SecurityPolicyEnforcer interface { EnforceDeviceUnmountPolicy(unmountTarget string) (err error) EnforceOverlayMountPolicy(containerID string, layerPaths []string) (err error) EnforceCreateContainerPolicy(containerID string, argList []string, envList []string, workingDir string) (err error) + EnforceExpectedMountsPolicy(containerID string, spec *oci.Spec) error } func NewSecurityPolicyEnforcer(state SecurityPolicyState) (SecurityPolicyEnforcer, error) { @@ -160,13 +169,19 @@ func (c Container) toInternal() (securityPolicyContainer, error) { return securityPolicyContainer{}, err } + expectedMounts, err := c.ExpectedMounts.toInternal() + if err != nil { + return securityPolicyContainer{}, err + } + return securityPolicyContainer{ Command: command, EnvRules: envRules, Layers: layers, // No need to have toInternal(), because WorkingDir is a string both // internally and in the policy. - WorkingDir: c.WorkingDir, + WorkingDir: c.WorkingDir, + ExpectedMounts: expectedMounts, }, nil } @@ -205,6 +220,14 @@ func (l Layers) toInternal() ([]string, error) { return stringMapToStringArray(l.Elements), nil } +func (em *ExpectedMounts) toInternal() ([]string, error) { + if em.Length != len(em.Elements) { + return nil, fmt.Errorf("expectedMounts numbers don't match in policy. expected: %d, actual: %d", em.Length, len(em.Elements)) + } + + return stringMapToStringArray(em.Elements), nil +} + func stringMapToStringArray(in map[string]string) []string { inLength := len(in) out := make([]string, inLength) @@ -463,7 +486,7 @@ func equalForOverlay(a1 []string, a2 []string) bool { } func possibleIndicesForID(containerID string, mapping map[int]map[string]struct{}) []int { - possibles := []int{} + var possibles []int for index, ids := range mapping { for id := range ids { if containerID == id { @@ -475,6 +498,103 @@ func possibleIndicesForID(containerID string, mapping map[int]map[string]struct{ return possibles } +// EnforceExpectedMountsPolicy for StandardSecurityPolicyEnforcer injects a +// hooks.CreateRuntime hook into container spec and the hook ensures that +// the expected mounts appear prior container start. At the moment enforcement +// is expected to take place inside LCOW UVM. +// +// Supported scenarios: +// 1. expected mount is provided as a path inside the sandbox, and it should resolve inside UVM +// e.g, "sandbox://path/on/the/host", which will correspond to "/run/gcs/c//sandboxMounts/path/on/the/host" +// 2. expected mount is provided as a path under a sandbox mount path inside container, e.g., +// sandbox mount is at path "/sandbox/mount" and wait path is "/sandbox/mount/wait/path", which +// corresponds to "/run/gcs/c//sandboxMounts/path/on/the/host/wait/path" +// 3. expected mount is provided as an arbitrary path inside container, and it should resolve +// inside UVM, e.g., "/arbitrary/container/path", which corresponds to +// "/run/gcs/c//rootfs/arbitrary/container/path" +func (pe *StandardSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(containerID string, spec *oci.Spec) error { + pe.mutex.Lock() + defer pe.mutex.Unlock() + + if len(pe.Containers) < 1 { + return errors.New("policy doesn't allow mounting containers") + } + + sandboxID := spec.Annotations[annotations.KubernetesSandboxID] + if sandboxID == "" { + return errors.New("no sandbox ID present in spec annotations") + } + + var wMounts []string + pIndices := possibleIndicesForID(containerID, pe.ContainerIndexToContainerIds) + if len(pIndices) == 0 { + return errors.New("no valid container indices found") + } + + // Unlike environment variable and command line enforcement, there isn't anything + // to validate here, since we're essentially just injecting hooks when necessary + // for all containers. + matchFound := false + for _, index := range pIndices { + if !matchFound { + matchFound = true + wMounts = pe.Containers[index].ExpectedMounts + } else { + pe.narrowMatchesForContainerIndex(index, containerID) + } + } + + if len(wMounts) == 0 { + return nil + } + + var wPaths []string + for _, mount := range wMounts { + // By default, handle scenario #3 and resolve container path to the actual path inside UVM. + wp := filepath.Join(guestpath.LCOWRootPrefixInUVM, containerID, guestpath.RootfsPath, mount) + if strings.HasPrefix(mount, guestpath.SandboxMountPrefix) { + // This covers case #1, and we replace sandbox mount prefix with the sandbox + // mounts path inside UVM + sandboxPath := strings.TrimPrefix(mount, guestpath.SandboxMountPrefix) + wp = filepath.Join(guestpath.LCOWRootPrefixInUVM, "sandboxMounts", sandboxPath) + } else { + // This covers case #2. Iterate through container mounts to identify the + // correct sandbox mount where the wait path is nested under. The mount + // spec will be something like: + // { + // "source": "/run/gcs/c//sandboxMounts/path/on/host", + // "destination": "/sandbox/mount" + // } + // The wait path will be "/sandbox/mount/wait/path". To find the corresponding + // sandbox mount do a prefix match on wait path against all container mounts Destination + // and resolve the full path inside UVM. For example above it becomes + // "/run/gcs/c//sandboxMounts/path/on/host/wait/path" + for _, m := range spec.Mounts { + if strings.HasPrefix(mount, m.Destination) { + wp = filepath.Join(m.Source, strings.TrimPrefix(mount, m.Destination)) + break + } + } + if wp == "" { + return fmt.Errorf("invalid mount path: %q", mount) + } + } + wPaths = append(wPaths, filepath.Clean(wp)) + } + + pathsArg := strings.Join(wPaths, ",") + waitPathsBinary := "/bin/wait-paths" + args := []string{ + waitPathsBinary, + "--paths", + pathsArg, + "--timeout", + "60", + } + hook := hooks.NewOCIHook(waitPathsBinary, args, os.Environ()) + return hooks.AddOCIHook(spec, hooks.CreateRuntime, hook) +} + type OpenDoorSecurityPolicyEnforcer struct{} var _ SecurityPolicyEnforcer = (*OpenDoorSecurityPolicyEnforcer)(nil) @@ -495,6 +615,10 @@ func (p *OpenDoorSecurityPolicyEnforcer) EnforceCreateContainerPolicy(_ string, return nil } +func (p *OpenDoorSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(_ string, _ *oci.Spec) error { + return nil +} + type ClosedDoorSecurityPolicyEnforcer struct{} var _ SecurityPolicyEnforcer = (*ClosedDoorSecurityPolicyEnforcer)(nil) @@ -514,3 +638,7 @@ func (p *ClosedDoorSecurityPolicyEnforcer) EnforceOverlayMountPolicy(containerID func (p *ClosedDoorSecurityPolicyEnforcer) EnforceCreateContainerPolicy(_ string, _ []string, _ []string, _ string) (err error) { return errors.New("running commands is denied by policy") } + +func (p *ClosedDoorSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(_ string, _ *oci.Spec) error { + return errors.New("enforcing expected mounts is denied by policy") +} diff --git a/test/cri-containerd/container_test.go b/test/cri-containerd/container_test.go index 52ae6b0efd..1afbc1b7ce 100644 --- a/test/cri-containerd/container_test.go +++ b/test/cri-containerd/container_test.go @@ -6,6 +6,7 @@ package cri_containerd import ( "bufio" "context" + "fmt" "io" "os" "path/filepath" @@ -14,9 +15,11 @@ import ( "testing" "time" - "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/sirupsen/logrus" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + + "github.com/Microsoft/hcsshim/internal/guestpath" + "github.com/Microsoft/hcsshim/pkg/annotations" ) func runLogRotationContainer(t *testing.T, sandboxRequest *runtime.RunPodSandboxRequest, request *runtime.CreateContainerRequest, log string, logArchive string) { @@ -725,7 +728,7 @@ func Test_CreateContainer_HugePageMount_LCOW(t *testing.T) { }, Mounts: []*runtime.Mount{ { - HostPath: "hugepages://2M/hugepage2M", + HostPath: fmt.Sprintf("%s2M/hugepage2M", guestpath.HugePagesMountPrefix), ContainerPath: "/mnt/hugepage2M", Readonly: false, Propagation: runtime.MountPropagation_PROPAGATION_BIDIRECTIONAL, diff --git a/test/cri-containerd/policy_test.go b/test/cri-containerd/policy_test.go index b4dfa87777..0a1abacbfb 100644 --- a/test/cri-containerd/policy_test.go +++ b/test/cri-containerd/policy_test.go @@ -47,6 +47,7 @@ func alpineSecurityPolicy(t *testing.T) string { []securitypolicy.EnvRuleConfig{}, securitypolicy.AuthConfig{}, "", + []string{}, ) containers := append(defaultContainers, alpineContainer) diff --git a/test/vendor/github.com/Microsoft/hcsshim/Makefile b/test/vendor/github.com/Microsoft/hcsshim/Makefile index ac00def635..c09b1e192a 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/Makefile +++ b/test/vendor/github.com/Microsoft/hcsshim/Makefile @@ -32,7 +32,7 @@ clean: test: cd $(SRCROOT) && go test -v ./internal/guest/... -out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools Makefile +out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools bin/cmd/hooks/wait-paths Makefile @mkdir -p out rm -rf rootfs mkdir -p rootfs/bin/ @@ -40,6 +40,7 @@ out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools Makefile cp bin/vsockexec rootfs/bin/ cp bin/cmd/gcs rootfs/bin/ cp bin/cmd/gcstools rootfs/bin/ + cp bin/cmd/hooks/wait-paths rootfs/bin/ for tool in $(GCS_TOOLS); do ln -s gcstools rootfs/bin/$$tool; done git -C $(SRCROOT) rev-parse HEAD > rootfs/gcs.commit && \ git -C $(SRCROOT) rev-parse --abbrev-ref HEAD > rootfs/gcs.branch @@ -60,6 +61,7 @@ out/initrd.img: $(BASE) out/delta.tar.gz $(SRCROOT)/hack/catcpio.sh -include deps/cmd/gcs.gomake -include deps/cmd/gcstools.gomake +-include deps/cmd/hooks/wait-paths.gomake # Implicit rule for includes that define Go targets. %.gomake: $(SRCROOT)/Makefile diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/devices/drivers.go b/test/vendor/github.com/Microsoft/hcsshim/internal/devices/drivers.go index 4efe677e22..0dae33c963 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/devices/drivers.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/devices/drivers.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/Microsoft/hcsshim/internal/cmd" + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" @@ -45,7 +46,7 @@ func InstallKernelDriver(ctx context.Context, vm *uvm.UtilityVM, driver string) } return closer, execPnPInstallDriver(ctx, vm, uvmPath) } - uvmPathForShare := fmt.Sprintf(uvm.LCOWGlobalMountPrefix, vm.UVMMountCounter()) + uvmPathForShare := fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, vm.UVMMountCounter()) scsiCloser, err := vm.AddSCSI(ctx, driver, uvmPathForShare, true, false, []string{}, uvm.VMAccessTypeIndividual) if err != nil { return closer, fmt.Errorf("failed to add SCSI disk to utility VM for path %+v: %s", driver, err) diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/guestpath/paths.go b/test/vendor/github.com/Microsoft/hcsshim/internal/guestpath/paths.go new file mode 100644 index 0000000000..62bbfeb636 --- /dev/null +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/guestpath/paths.go @@ -0,0 +1,24 @@ +package guestpath + +const ( + // LCOWNvidiaMountPath is the path format in LCOW UVM where nvidia tools are mounted + // keep this value in sync with opengcs + LCOWNvidiaMountPath = "/run/nvidia" + // LCOWRootPrefixInUVM is the path inside UVM where LCOW container's root file system will be mounted + LCOWRootPrefixInUVM = "/run/gcs/c" + // WCOWRootPrefixInUVM is the path inside UVM where WCOW container's root file system will be mounted + WCOWRootPrefixInUVM = `C:\c` + // SandboxMountPrefix is mount prefix used in container spec to mark a sandbox-mount + SandboxMountPrefix = "sandbox://" + // HugePagesMountPrefix is mount prefix used in container spec to mark a huge-pages mount + HugePagesMountPrefix = "hugepages://" + // LCOWMountPathPrefixFmt is the path format in the LCOW UVM where non global mounts, such + // as Plan9 mounts are added + LCOWMountPathPrefixFmt = "/mounts/m%d" + // LCOWGlobalMountPrefixFmt is the path format in the LCOW UVM where global mounts are added + LCOWGlobalMountPrefixFmt = "/run/mounts/m%d" + // WCOWGlobalMountPrefixFmt is the path prefix format in the WCOW UVM where mounts are added + WCOWGlobalMountPrefixFmt = "C:\\mounts\\m%d" + // RootfsPath is part of the container's rootfs path + RootfsPath = "rootfs" +) diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go index 16821a1861..058530aac1 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go @@ -14,6 +14,7 @@ import ( "github.com/Microsoft/go-winio/pkg/guid" "github.com/Microsoft/hcsshim/internal/clone" "github.com/Microsoft/hcsshim/internal/cow" + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/hcs" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" @@ -27,8 +28,8 @@ import ( ) var ( - lcowRootInUVM = "/run/gcs/c/%s" - wcowRootInUVM = `C:\c\%s` + lcowRootInUVM = guestpath.LCOWRootPrefixInUVM + "/%s" + wcowRootInUVM = guestpath.WCOWRootPrefixInUVM + "/%s" ) // CreateOptions are the set of fields used to call CreateContainer(). diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go index 42d11aac52..4f17a715d6 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go @@ -9,7 +9,11 @@ import ( "path/filepath" "strconv" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/Microsoft/hcsshim/internal/devices" + "github.com/Microsoft/hcsshim/internal/guestpath" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oci" @@ -17,8 +21,6 @@ import ( "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" ) const deviceUtilExeName = "device-util.exe" @@ -222,14 +224,14 @@ func handleAssignedDevicesLCOW( scsiMount, err := vm.AddSCSI( ctx, gpuSupportVhdPath, - uvm.LCOWNvidiaMountPath, + guestpath.LCOWNvidiaMountPath, true, false, options, uvm.VMAccessTypeNoop, ) if err != nil { - return resultDevs, closers, errors.Wrapf(err, "failed to add scsi device %s in the UVM %s at %s", gpuSupportVhdPath, vm.ID(), uvm.LCOWNvidiaMountPath) + return resultDevs, closers, errors.Wrapf(err, "failed to add scsi device %s in the UVM %s at %s", gpuSupportVhdPath, vm.ID(), guestpath.LCOWNvidiaMountPath) } closers = append(closers, scsiMount) } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go index 688d901609..b3080399a6 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go @@ -11,6 +11,10 @@ import ( "regexp" "strings" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" + + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/hcs/schema1" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/layers" @@ -22,8 +26,6 @@ import ( "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/sirupsen/logrus" ) // A simple wrapper struct around the container mount configs that should be added to the @@ -78,7 +80,7 @@ func createMountsConfig(ctx context.Context, coi *createOptionsInternal) (*mount return nil, err } mdv2.HostPath = uvmPath - } else if strings.HasPrefix(mount.Source, "sandbox://") { + } else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) { // Convert to the path in the guest that was asked for. mdv2.HostPath = convertToWCOWSandboxMountPath(mount.Source) } else { diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go index 085854a4c7..d438738c28 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go @@ -13,12 +13,14 @@ import ( "path/filepath" "strings" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" ) func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *resources.Resources, isSandbox bool) error { @@ -39,7 +41,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * // This is the "Plan 9" root filesystem. // TODO: We need a test for this. Ask @jstarks how you can even lay this out on Windows. hostPath := coi.Spec.Root.Path - uvmPathForContainersFileSystem := path.Join(r.ContainerRootInUVM(), uvm.RootfsPath) + uvmPathForContainersFileSystem := path.Join(r.ContainerRootInUVM(), guestpath.RootfsPath) share, err := coi.HostingSystem.AddPlan9(ctx, hostPath, uvmPathForContainersFileSystem, coi.Spec.Root.Readonly, false, nil) if err != nil { return errors.Wrap(err, "adding plan9 root") @@ -65,7 +67,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * if coi.HostingSystem != nil { hostPath := mount.Source - uvmPathForShare := path.Join(containerRootInUVM, fmt.Sprintf(uvm.LCOWMountPathPrefix, i)) + uvmPathForShare := path.Join(containerRootInUVM, fmt.Sprintf(guestpath.LCOWMountPathPrefixFmt, i)) uvmPathForFile := uvmPathForShare readOnly := false @@ -79,7 +81,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * l := log.G(ctx).WithField("mount", fmt.Sprintf("%+v", mount)) if mount.Type == "physical-disk" { l.Debug("hcsshim::allocateLinuxResources Hot-adding SCSI physical disk for OCI mount") - uvmPathForShare = fmt.Sprintf(uvm.LCOWGlobalMountPrefix, coi.HostingSystem.UVMMountCounter()) + uvmPathForShare = fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, coi.HostingSystem.UVMMountCounter()) scsiMount, err := coi.HostingSystem.AddSCSIPhysicalDisk(ctx, hostPath, uvmPathForShare, readOnly, mount.Options) if err != nil { return errors.Wrapf(err, "adding SCSI physical disk mount %+v", mount) @@ -90,7 +92,7 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * coi.Spec.Mounts[i].Type = "none" } else if mount.Type == "virtual-disk" { l.Debug("hcsshim::allocateLinuxResources Hot-adding SCSI virtual disk for OCI mount") - uvmPathForShare = fmt.Sprintf(uvm.LCOWGlobalMountPrefix, coi.HostingSystem.UVMMountCounter()) + uvmPathForShare = fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, coi.HostingSystem.UVMMountCounter()) // if the scsi device is already attached then we take the uvm path that the function below returns // that is where it was previously mounted in UVM @@ -110,15 +112,19 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * uvmPathForFile = scsiMount.UVMPath r.Add(scsiMount) coi.Spec.Mounts[i].Type = "none" - } else if strings.HasPrefix(mount.Source, "sandbox://") { + } else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) { // Mounts that map to a path in UVM are specified with 'sandbox://' prefix. // example: sandbox:///a/dirInUvm destination:/b/dirInContainer uvmPathForFile = mount.Source - } else if strings.HasPrefix(mount.Source, "hugepages://") { + } else if strings.HasPrefix(mount.Source, guestpath.HugePagesMountPrefix) { // currently we only support 2M hugepage size - hugePageSubDirs := strings.Split(strings.TrimPrefix(mount.Source, "hugepages://"), "/") + hugePageSubDirs := strings.Split(strings.TrimPrefix(mount.Source, guestpath.HugePagesMountPrefix), "/") if len(hugePageSubDirs) < 2 { - return errors.Errorf(`%s mount path is invalid, expected format: hugepages:///`, mount.Source) + return errors.Errorf( + `%s mount path is invalid, expected format: %s/`, + mount.Source, + guestpath.HugePagesMountPrefix, + ) } // hugepages:// should be followed by pagesize @@ -155,15 +161,17 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * } } - if coi.HostingSystem != nil { - if coi.hasWindowsAssignedDevices() { - windowsDevices, closers, err := handleAssignedDevicesLCOW(ctx, coi.HostingSystem, coi.Spec.Annotations, coi.Spec.Windows.Devices) - if err != nil { - return err - } - r.Add(closers...) - coi.Spec.Windows.Devices = windowsDevices + if coi.HostingSystem == nil { + return nil + } + + if coi.hasWindowsAssignedDevices() { + windowsDevices, closers, err := handleAssignedDevicesLCOW(ctx, coi.HostingSystem, coi.Spec.Annotations, coi.Spec.Windows.Devices) + if err != nil { + return err } + r.Add(closers...) + coi.Spec.Windows.Devices = windowsDevices } return nil } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go index 1150ca5c3c..fa22c8047e 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go @@ -13,16 +13,18 @@ import ( "path/filepath" "strings" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/Microsoft/hcsshim/internal/cmd" "github.com/Microsoft/hcsshim/internal/credentials" + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/schemaversion" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/internal/wclayer" - specs "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" ) const wcowSandboxMountPath = "C:\\SandboxMounts" @@ -140,7 +142,7 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R } if coi.HostingSystem != nil && schemaversion.IsV21(coi.actualSchemaVersion) { - uvmPath := fmt.Sprintf(uvm.WCOWGlobalMountPrefix, coi.HostingSystem.UVMMountCounter()) + uvmPath := fmt.Sprintf(guestpath.WCOWGlobalMountPrefixFmt, coi.HostingSystem.UVMMountCounter()) readOnly := false for _, o := range mount.Options { if strings.ToLower(o) == "ro" { @@ -178,7 +180,7 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R return errors.Wrapf(err, "adding SCSI EVD mount failed %+v", mount) } r.Add(scsiMount) - } else if strings.HasPrefix(mount.Source, "sandbox://") { + } else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) { // Mounts that map to a path in the UVM are specified with a 'sandbox://' prefix. // // Example: sandbox:///a/dirInUvm destination:C:\\dirInContainer. @@ -228,6 +230,6 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R } func convertToWCOWSandboxMountPath(source string) string { - subPath := strings.TrimPrefix(source, "sandbox://") + subPath := strings.TrimPrefix(source, guestpath.SandboxMountPrefix) return filepath.Join(wcowSandboxMountPath, subPath) } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hooks/spec.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hooks/spec.go new file mode 100644 index 0000000000..51ba3aa592 --- /dev/null +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hooks/spec.go @@ -0,0 +1,53 @@ +package hooks + +import ( + "fmt" + + oci "github.com/opencontainers/runtime-spec/specs-go" +) + +// Note: The below type definition as well as constants have been copied from +// https://github.com/opencontainers/runc/blob/master/libcontainer/configs/config.go. +// This is done to not introduce a direct dependency on runc, which would complicate +// integration with windows. +type HookName string + +const ( + + // Prestart commands are executed after the container namespaces are created, + // but before the user supplied command is executed from init. + // Note: This hook is now deprecated + // Prestart commands are called in the Runtime namespace. + Prestart HookName = "prestart" + + // CreateRuntime commands MUST be called as part of the create operation after + // the runtime environment has been created but before the pivot_root has been executed. + // CreateRuntime is called immediately after the deprecated Prestart hook. + // CreateRuntime commands are called in the Runtime Namespace. + CreateRuntime HookName = "createRuntime" +) + +// NewOCIHook creates a new oci.Hook with given parameters +func NewOCIHook(path string, args, env []string) oci.Hook { + return oci.Hook{ + Path: path, + Args: args, + Env: env, + } +} + +// AddOCIHook adds oci.Hook of the given hook name to spec +func AddOCIHook(spec *oci.Spec, hn HookName, hk oci.Hook) error { + if spec.Hooks == nil { + spec.Hooks = &oci.Hooks{} + } + switch hn { + case Prestart: + spec.Hooks.Prestart = append(spec.Hooks.Prestart, hk) + case CreateRuntime: + spec.Hooks.CreateRuntime = append(spec.Hooks.CreateRuntime, hk) + default: + return fmt.Errorf("hook %q is not supported", hn) + } + return nil +} diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/layers/layers.go b/test/vendor/github.com/Microsoft/hcsshim/internal/layers/layers.go index 573ad72aa7..0334076365 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/layers/layers.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/layers/layers.go @@ -11,15 +11,17 @@ import ( "path/filepath" "time" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" + + "github.com/Microsoft/hcsshim/internal/guestpath" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/hcserror" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/ospath" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/internal/wclayer" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "golang.org/x/sys/windows" ) // ImageLayers contains all the layers for an image. @@ -261,7 +263,7 @@ func MountContainerLayers(ctx context.Context, containerID string, layerFolders err = vm.CombineLayersWCOW(ctx, layers, containerScratchPathInUVM) rootfs = containerScratchPathInUVM } else { - rootfs = ospath.Join(vm.OS(), guestRoot, uvm.RootfsPath) + rootfs = ospath.Join(vm.OS(), guestRoot, guestpath.RootfsPath) err = vm.CombineLayersLCOW(ctx, containerID, lcowUvmLayerPaths, containerScratchPathInUVM, rootfs) } if err != nil { @@ -289,7 +291,7 @@ func addLCOWLayer(ctx context.Context, vm *uvm.UtilityVM, layerPath string) (uvm } options := []string{"ro"} - uvmPath = fmt.Sprintf(uvm.LCOWGlobalMountPrefix, vm.UVMMountCounter()) + uvmPath = fmt.Sprintf(guestpath.LCOWGlobalMountPrefixFmt, vm.UVMMountCounter()) sm, err := vm.AddSCSI(ctx, layerPath, uvmPath, true, false, options, uvm.VMAccessTypeNoop) if err != nil { return "", fmt.Errorf("failed to add SCSI layer: %s", err) @@ -460,7 +462,7 @@ func containerRootfsPath(vm *uvm.UtilityVM, rootPath string) string { if vm.OS() == "windows" { return ospath.Join(vm.OS(), rootPath) } - return ospath.Join(vm.OS(), rootPath, uvm.RootfsPath) + return ospath.Join(vm.OS(), rootPath, guestpath.RootfsPath) } func getScratchVHDPath(layerFolders []string) (string, error) { diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/tools/securitypolicy/helpers/helpers.go b/test/vendor/github.com/Microsoft/hcsshim/internal/tools/securitypolicy/helpers/helpers.go index 614e5c10e8..42f6fd8a70 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/tools/securitypolicy/helpers/helpers.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/tools/securitypolicy/helpers/helpers.go @@ -74,6 +74,7 @@ func DefaultContainerConfigs() []securitypolicy.ContainerConfig { []securitypolicy.EnvRuleConfig{}, securitypolicy.AuthConfig{}, "", + []string{}, ) return []securitypolicy.ContainerConfig{pause} } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/constants.go b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/constants.go index 842a8a5e8c..1ddcf903ab 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/constants.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/constants.go @@ -16,19 +16,6 @@ const ( // DefaultVPMemSizeBytes is the default size of a VPMem device if the create request // doesn't specify. DefaultVPMemSizeBytes = 4 * 1024 * 1024 * 1024 // 4GB - - // LCOWMountPathPrefix is the path format in the LCOW UVM where non global mounts, such - // as Plan9 mounts are added - LCOWMountPathPrefix = "/mounts/m%d" - // LCOWGlobalMountPrefix is the path format in the LCOW UVM where global mounts are added - LCOWGlobalMountPrefix = "/run/mounts/m%d" - // LCOWNvidiaMountPath is the path format in LCOW UVM where nvidia tools are mounted - // keep this value in sync with opengcs - LCOWNvidiaMountPath = "/run/nvidia" - // WCOWGlobalMountPrefix is the path prefix format in the WCOW UVM where mounts are added - WCOWGlobalMountPrefix = "C:\\mounts\\m%d" - // RootfsPath is part of the container's rootfs path - RootfsPath = "rootfs" ) var ( diff --git a/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go b/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go index 7a51b2a0e4..2f06dfb8bd 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go +++ b/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go @@ -261,7 +261,7 @@ const ( // GuestStateFile specifies the path of the vmgs file to use if required. Only applies in SNP mode. GuestStateFile = "io.microsoft.virtualmachine.lcow.gueststatefile" - // AnnotationDisableLCOWTimeSyncService is used to disable the chronyd time + // DisableLCOWTimeSyncService is used to disable the chronyd time // synchronization service inside the LCOW UVM. DisableLCOWTimeSyncService = "io.microsoft.virtualmachine.lcow.timesync.disable" diff --git a/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicy.go b/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicy.go index 42ea861024..d7ec30bdd6 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicy.go +++ b/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicy.go @@ -38,11 +38,12 @@ type EnvRuleConfig struct { // ContainerConfig contains toml or JSON config for container described // in security policy. type ContainerConfig struct { - ImageName string `json:"image_name" toml:"image_name"` - Command []string `json:"command" toml:"command"` - Auth AuthConfig `json:"auth" toml:"auth"` - EnvRules []EnvRuleConfig `json:"env_rules" toml:"env_rule"` - WorkingDir string `json:"working_dir" toml:"working_dir"` + ImageName string `json:"image_name" toml:"image_name"` + Command []string `json:"command" toml:"command"` + Auth AuthConfig `json:"auth" toml:"auth"` + EnvRules []EnvRuleConfig `json:"env_rules" toml:"env_rule"` + WorkingDir string `json:"working_dir" toml:"working_dir"` + ExpectedMounts []string `json:"expected_mounts" toml:"expected_mounts"` } // NewContainerConfig creates a new ContainerConfig from the given values. @@ -52,13 +53,15 @@ func NewContainerConfig( envRules []EnvRuleConfig, auth AuthConfig, workingDir string, + expectedMounts []string, ) ContainerConfig { return ContainerConfig{ - ImageName: imageName, - Command: command, - EnvRules: envRules, - Auth: auth, - WorkingDir: workingDir, + ImageName: imageName, + Command: command, + EnvRules: envRules, + Auth: auth, + WorkingDir: workingDir, + ExpectedMounts: expectedMounts, } } @@ -97,6 +100,9 @@ type securityPolicyContainer struct { // WorkingDir is a path to container's working directory, which all the processes // will default to. WorkingDir string + // Unordered list of mounts which are expected to be present when the container + // starts + ExpectedMounts []string `json:"expected_mounts"` } // SecurityPolicyState is a structure that holds user supplied policy to enforce @@ -119,7 +125,7 @@ type EncodedSecurityPolicy struct { // security policy for given policy. The security policy is transmitted as json // in an annotation, so we first have to remove the base64 encoding that allows // the JSON based policy to be passed as a string. From there, we decode the -// JSONand setup our security policy struct +// JSON and setup our security policy struct func NewSecurityPolicyState(base64Policy string) (*SecurityPolicyState, error) { // construct an encoded security policy that holds the base64 representation encodedSecurityPolicy := EncodedSecurityPolicy{ @@ -174,10 +180,11 @@ type Containers struct { } type Container struct { - Command CommandArgs `json:"command"` - EnvRules EnvRules `json:"env_rules"` - Layers Layers `json:"layers"` - WorkingDir string `json:"working_dir"` + Command CommandArgs `json:"command"` + EnvRules EnvRules `json:"env_rules"` + Layers Layers `json:"layers"` + WorkingDir string `json:"working_dir"` + ExpectedMounts ExpectedMounts `json:"expected_mounts"` } type Layers struct { @@ -197,6 +204,11 @@ type EnvRules struct { Elements map[string]EnvRuleConfig `json:"elements"` } +type ExpectedMounts struct { + Length int `json:"length"` + Elements map[string]string `json:"elements"` +} + // NewContainer creates a new Container instance from the provided values // or an error if envRules validation fails. func NewContainer(command, layers []string, envRules []EnvRuleConfig, workingDir string) (*Container, error) { @@ -312,3 +324,14 @@ func (e EnvRules) MarshalJSON() ([]byte, error) { Alias: (*Alias)(&e), }) } + +func (em ExpectedMounts) MarshalJSON() ([]byte, error) { + type Alias ExpectedMounts + return json.Marshal(&struct { + Length int `json:"length"` + *Alias + }{ + Length: len(em.Elements), + Alias: (*Alias)(&em), + }) +} diff --git a/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicyenforcer.go b/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicyenforcer.go index d5daccbb98..3024f8f5cc 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicyenforcer.go +++ b/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicyenforcer.go @@ -3,11 +3,19 @@ package securitypolicy import ( "errors" "fmt" + "os" + "path/filepath" "regexp" "strconv" + "strings" "sync" "github.com/google/go-cmp/cmp" + oci "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/Microsoft/hcsshim/internal/guestpath" + "github.com/Microsoft/hcsshim/internal/hooks" + "github.com/Microsoft/hcsshim/pkg/annotations" ) type SecurityPolicyEnforcer interface { @@ -15,6 +23,7 @@ type SecurityPolicyEnforcer interface { EnforceDeviceUnmountPolicy(unmountTarget string) (err error) EnforceOverlayMountPolicy(containerID string, layerPaths []string) (err error) EnforceCreateContainerPolicy(containerID string, argList []string, envList []string, workingDir string) (err error) + EnforceExpectedMountsPolicy(containerID string, spec *oci.Spec) error } func NewSecurityPolicyEnforcer(state SecurityPolicyState) (SecurityPolicyEnforcer, error) { @@ -160,13 +169,19 @@ func (c Container) toInternal() (securityPolicyContainer, error) { return securityPolicyContainer{}, err } + expectedMounts, err := c.ExpectedMounts.toInternal() + if err != nil { + return securityPolicyContainer{}, err + } + return securityPolicyContainer{ Command: command, EnvRules: envRules, Layers: layers, // No need to have toInternal(), because WorkingDir is a string both // internally and in the policy. - WorkingDir: c.WorkingDir, + WorkingDir: c.WorkingDir, + ExpectedMounts: expectedMounts, }, nil } @@ -205,6 +220,14 @@ func (l Layers) toInternal() ([]string, error) { return stringMapToStringArray(l.Elements), nil } +func (em *ExpectedMounts) toInternal() ([]string, error) { + if em.Length != len(em.Elements) { + return nil, fmt.Errorf("expectedMounts numbers don't match in policy. expected: %d, actual: %d", em.Length, len(em.Elements)) + } + + return stringMapToStringArray(em.Elements), nil +} + func stringMapToStringArray(in map[string]string) []string { inLength := len(in) out := make([]string, inLength) @@ -463,7 +486,7 @@ func equalForOverlay(a1 []string, a2 []string) bool { } func possibleIndicesForID(containerID string, mapping map[int]map[string]struct{}) []int { - possibles := []int{} + var possibles []int for index, ids := range mapping { for id := range ids { if containerID == id { @@ -475,6 +498,103 @@ func possibleIndicesForID(containerID string, mapping map[int]map[string]struct{ return possibles } +// EnforceExpectedMountsPolicy for StandardSecurityPolicyEnforcer injects a +// hooks.CreateRuntime hook into container spec and the hook ensures that +// the expected mounts appear prior container start. At the moment enforcement +// is expected to take place inside LCOW UVM. +// +// Supported scenarios: +// 1. expected mount is provided as a path inside the sandbox, and it should resolve inside UVM +// e.g, "sandbox://path/on/the/host", which will correspond to "/run/gcs/c//sandboxMounts/path/on/the/host" +// 2. expected mount is provided as a path under a sandbox mount path inside container, e.g., +// sandbox mount is at path "/sandbox/mount" and wait path is "/sandbox/mount/wait/path", which +// corresponds to "/run/gcs/c//sandboxMounts/path/on/the/host/wait/path" +// 3. expected mount is provided as an arbitrary path inside container, and it should resolve +// inside UVM, e.g., "/arbitrary/container/path", which corresponds to +// "/run/gcs/c//rootfs/arbitrary/container/path" +func (pe *StandardSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(containerID string, spec *oci.Spec) error { + pe.mutex.Lock() + defer pe.mutex.Unlock() + + if len(pe.Containers) < 1 { + return errors.New("policy doesn't allow mounting containers") + } + + sandboxID := spec.Annotations[annotations.KubernetesSandboxID] + if sandboxID == "" { + return errors.New("no sandbox ID present in spec annotations") + } + + var wMounts []string + pIndices := possibleIndicesForID(containerID, pe.ContainerIndexToContainerIds) + if len(pIndices) == 0 { + return errors.New("no valid container indices found") + } + + // Unlike environment variable and command line enforcement, there isn't anything + // to validate here, since we're essentially just injecting hooks when necessary + // for all containers. + matchFound := false + for _, index := range pIndices { + if !matchFound { + matchFound = true + wMounts = pe.Containers[index].ExpectedMounts + } else { + pe.narrowMatchesForContainerIndex(index, containerID) + } + } + + if len(wMounts) == 0 { + return nil + } + + var wPaths []string + for _, mount := range wMounts { + // By default, handle scenario #3 and resolve container path to the actual path inside UVM. + wp := filepath.Join(guestpath.LCOWRootPrefixInUVM, containerID, guestpath.RootfsPath, mount) + if strings.HasPrefix(mount, guestpath.SandboxMountPrefix) { + // This covers case #1, and we replace sandbox mount prefix with the sandbox + // mounts path inside UVM + sandboxPath := strings.TrimPrefix(mount, guestpath.SandboxMountPrefix) + wp = filepath.Join(guestpath.LCOWRootPrefixInUVM, "sandboxMounts", sandboxPath) + } else { + // This covers case #2. Iterate through container mounts to identify the + // correct sandbox mount where the wait path is nested under. The mount + // spec will be something like: + // { + // "source": "/run/gcs/c//sandboxMounts/path/on/host", + // "destination": "/sandbox/mount" + // } + // The wait path will be "/sandbox/mount/wait/path". To find the corresponding + // sandbox mount do a prefix match on wait path against all container mounts Destination + // and resolve the full path inside UVM. For example above it becomes + // "/run/gcs/c//sandboxMounts/path/on/host/wait/path" + for _, m := range spec.Mounts { + if strings.HasPrefix(mount, m.Destination) { + wp = filepath.Join(m.Source, strings.TrimPrefix(mount, m.Destination)) + break + } + } + if wp == "" { + return fmt.Errorf("invalid mount path: %q", mount) + } + } + wPaths = append(wPaths, filepath.Clean(wp)) + } + + pathsArg := strings.Join(wPaths, ",") + waitPathsBinary := "/bin/wait-paths" + args := []string{ + waitPathsBinary, + "--paths", + pathsArg, + "--timeout", + "60", + } + hook := hooks.NewOCIHook(waitPathsBinary, args, os.Environ()) + return hooks.AddOCIHook(spec, hooks.CreateRuntime, hook) +} + type OpenDoorSecurityPolicyEnforcer struct{} var _ SecurityPolicyEnforcer = (*OpenDoorSecurityPolicyEnforcer)(nil) @@ -495,6 +615,10 @@ func (p *OpenDoorSecurityPolicyEnforcer) EnforceCreateContainerPolicy(_ string, return nil } +func (p *OpenDoorSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(_ string, _ *oci.Spec) error { + return nil +} + type ClosedDoorSecurityPolicyEnforcer struct{} var _ SecurityPolicyEnforcer = (*ClosedDoorSecurityPolicyEnforcer)(nil) @@ -514,3 +638,7 @@ func (p *ClosedDoorSecurityPolicyEnforcer) EnforceOverlayMountPolicy(containerID func (p *ClosedDoorSecurityPolicyEnforcer) EnforceCreateContainerPolicy(_ string, _ []string, _ []string, _ string) (err error) { return errors.New("running commands is denied by policy") } + +func (p *ClosedDoorSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(_ string, _ *oci.Spec) error { + return errors.New("enforcing expected mounts is denied by policy") +} diff --git a/test/vendor/modules.txt b/test/vendor/modules.txt index 496e3d05fe..201ced4412 100644 --- a/test/vendor/modules.txt +++ b/test/vendor/modules.txt @@ -28,6 +28,7 @@ github.com/Microsoft/hcsshim/internal/credentials github.com/Microsoft/hcsshim/internal/devices github.com/Microsoft/hcsshim/internal/extendedtask github.com/Microsoft/hcsshim/internal/gcs +github.com/Microsoft/hcsshim/internal/guestpath github.com/Microsoft/hcsshim/internal/hcs github.com/Microsoft/hcsshim/internal/hcs/resourcepaths github.com/Microsoft/hcsshim/internal/hcs/schema1 @@ -35,6 +36,7 @@ github.com/Microsoft/hcsshim/internal/hcs/schema2 github.com/Microsoft/hcsshim/internal/hcserror github.com/Microsoft/hcsshim/internal/hcsoci github.com/Microsoft/hcsshim/internal/hns +github.com/Microsoft/hcsshim/internal/hooks github.com/Microsoft/hcsshim/internal/interop github.com/Microsoft/hcsshim/internal/layers github.com/Microsoft/hcsshim/internal/lcow From f5c3596ef502c0b0ddee1611ab1b544b50c2196f Mon Sep 17 00:00:00 2001 From: Maksim An Date: Tue, 15 Mar 2022 15:10:46 -0700 Subject: [PATCH 2/3] pr feedback: wait-path only supports paths under sandbox mounts Per pr feedback, only container paths under sandbox mounts are supported as wait-paths. The support for other 2 scenarios can be added as needed. Add positive and negative CRI tests. Signed-off-by: Maksim An --- .../tools/securitypolicy/helpers/helpers.go | 8 +- pkg/securitypolicy/securitypolicy.go | 21 ++- pkg/securitypolicy/securitypolicyenforcer.go | 67 ++++----- test/cri-containerd/policy_test.go | 134 ++++++++++++++++++ .../tools/securitypolicy/helpers/helpers.go | 8 +- .../pkg/securitypolicy/securitypolicy.go | 21 ++- .../securitypolicy/securitypolicyenforcer.go | 67 ++++----- 7 files changed, 234 insertions(+), 92 deletions(-) diff --git a/internal/tools/securitypolicy/helpers/helpers.go b/internal/tools/securitypolicy/helpers/helpers.go index 42f6fd8a70..103b2bae3a 100644 --- a/internal/tools/securitypolicy/helpers/helpers.go +++ b/internal/tools/securitypolicy/helpers/helpers.go @@ -137,7 +137,13 @@ func PolicyContainersFromConfigs(containerConfigs []securitypolicy.ContainerConf workingDir = containerConfig.WorkingDir } - container, err := securitypolicy.NewContainer(containerConfig.Command, layerHashes, envRules, workingDir) + container, err := securitypolicy.NewContainer( + containerConfig.Command, + layerHashes, + envRules, + workingDir, + containerConfig.ExpectedMounts, + ) if err != nil { return nil, err } diff --git a/pkg/securitypolicy/securitypolicy.go b/pkg/securitypolicy/securitypolicy.go index d7ec30bdd6..207a2779b3 100644 --- a/pkg/securitypolicy/securitypolicy.go +++ b/pkg/securitypolicy/securitypolicy.go @@ -211,15 +211,16 @@ type ExpectedMounts struct { // NewContainer creates a new Container instance from the provided values // or an error if envRules validation fails. -func NewContainer(command, layers []string, envRules []EnvRuleConfig, workingDir string) (*Container, error) { +func NewContainer(command, layers []string, envRules []EnvRuleConfig, workingDir string, eMounts []string) (*Container, error) { if err := validateEnvRules(envRules); err != nil { return nil, err } return &Container{ - Command: newCommandArgs(command), - Layers: newLayers(layers), - EnvRules: newEnvRules(envRules), - WorkingDir: workingDir, + Command: newCommandArgs(command), + Layers: newLayers(layers), + EnvRules: newEnvRules(envRules), + WorkingDir: workingDir, + ExpectedMounts: newExpectedMounts(eMounts), }, nil } @@ -279,6 +280,16 @@ func newLayers(ls []string) Layers { } } +func newExpectedMounts(em []string) ExpectedMounts { + mounts := map[string]string{} + for i, m := range em { + mounts[strconv.Itoa(i)] = m + } + return ExpectedMounts{ + Elements: mounts, + } +} + // Custom JSON marshalling to add `lenth` field that matches the number of // elements present in the `elements` field. func (c Containers) MarshalJSON() ([]byte, error) { diff --git a/pkg/securitypolicy/securitypolicyenforcer.go b/pkg/securitypolicy/securitypolicyenforcer.go index 3024f8f5cc..b435080e9e 100644 --- a/pkg/securitypolicy/securitypolicyenforcer.go +++ b/pkg/securitypolicy/securitypolicyenforcer.go @@ -10,12 +10,10 @@ import ( "strings" "sync" - "github.com/google/go-cmp/cmp" - oci "github.com/opencontainers/runtime-spec/specs-go" - - "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/hooks" "github.com/Microsoft/hcsshim/pkg/annotations" + "github.com/google/go-cmp/cmp" + oci "github.com/opencontainers/runtime-spec/specs-go" ) type SecurityPolicyEnforcer interface { @@ -503,15 +501,22 @@ func possibleIndicesForID(containerID string, mapping map[int]map[string]struct{ // the expected mounts appear prior container start. At the moment enforcement // is expected to take place inside LCOW UVM. // -// Supported scenarios: -// 1. expected mount is provided as a path inside the sandbox, and it should resolve inside UVM -// e.g, "sandbox://path/on/the/host", which will correspond to "/run/gcs/c//sandboxMounts/path/on/the/host" -// 2. expected mount is provided as a path under a sandbox mount path inside container, e.g., -// sandbox mount is at path "/sandbox/mount" and wait path is "/sandbox/mount/wait/path", which -// corresponds to "/run/gcs/c//sandboxMounts/path/on/the/host/wait/path" -// 3. expected mount is provided as an arbitrary path inside container, and it should resolve -// inside UVM, e.g., "/arbitrary/container/path", which corresponds to -// "/run/gcs/c//rootfs/arbitrary/container/path" +// Expected mount is provided as a path under a sandbox mount path inside +// container, e.g., sandbox mount is at path "/path/in/container" and wait path +// is "/path/in/container/wait/path", which corresponds to +// "/run/gcs/c//sandboxMounts/path/on/the/host/wait/path" +// +// Iterates through container mounts to identify the correct sandbox +// mount where the wait path is nested under. The mount spec will +// be something like: +// { +// "source": "/run/gcs/c//sandboxMounts/path/on/host", +// "destination": "/path/in/container" +// } +// The wait path will be "/path/in/container/wait/path". To find the corresponding +// sandbox mount do a prefix match on wait path against all container mounts +// Destination and resolve the full path inside UVM. For example above it becomes +// "/run/gcs/c//sandboxMounts/path/on/host/wait/path" func (pe *StandardSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(containerID string, spec *oci.Spec) error { pe.mutex.Lock() defer pe.mutex.Unlock() @@ -550,35 +555,17 @@ func (pe *StandardSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(containerI var wPaths []string for _, mount := range wMounts { - // By default, handle scenario #3 and resolve container path to the actual path inside UVM. - wp := filepath.Join(guestpath.LCOWRootPrefixInUVM, containerID, guestpath.RootfsPath, mount) - if strings.HasPrefix(mount, guestpath.SandboxMountPrefix) { - // This covers case #1, and we replace sandbox mount prefix with the sandbox - // mounts path inside UVM - sandboxPath := strings.TrimPrefix(mount, guestpath.SandboxMountPrefix) - wp = filepath.Join(guestpath.LCOWRootPrefixInUVM, "sandboxMounts", sandboxPath) - } else { - // This covers case #2. Iterate through container mounts to identify the - // correct sandbox mount where the wait path is nested under. The mount - // spec will be something like: - // { - // "source": "/run/gcs/c//sandboxMounts/path/on/host", - // "destination": "/sandbox/mount" - // } - // The wait path will be "/sandbox/mount/wait/path". To find the corresponding - // sandbox mount do a prefix match on wait path against all container mounts Destination - // and resolve the full path inside UVM. For example above it becomes - // "/run/gcs/c//sandboxMounts/path/on/host/wait/path" - for _, m := range spec.Mounts { - if strings.HasPrefix(mount, m.Destination) { - wp = filepath.Join(m.Source, strings.TrimPrefix(mount, m.Destination)) - break - } - } - if wp == "" { - return fmt.Errorf("invalid mount path: %q", mount) + var wp string + for _, m := range spec.Mounts { + // prefix matching to find correct sandbox mount + if strings.HasPrefix(mount, m.Destination) { + wp = filepath.Join(m.Source, strings.TrimPrefix(mount, m.Destination)) + break } } + if wp == "" { + return fmt.Errorf("invalid mount path: %q", mount) + } wPaths = append(wPaths, filepath.Clean(wp)) } diff --git a/test/cri-containerd/policy_test.go b/test/cri-containerd/policy_test.go index 0a1abacbfb..d5e99e64a5 100644 --- a/test/cri-containerd/policy_test.go +++ b/test/cri-containerd/policy_test.go @@ -5,6 +5,7 @@ package cri_containerd import ( "context" + "strings" "testing" "github.com/Microsoft/hcsshim/internal/tools/securitypolicy/helpers" @@ -17,6 +18,15 @@ var ( validPolicyAlpineCommand = []string{"ash", "-c", "echo 'Hello'"} ) +type configOpt func(*securitypolicy.ContainerConfig) error + +func withExpectedMounts(em []string) configOpt { + return func(conf *securitypolicy.ContainerConfig) error { + conf.ExpectedMounts = append(conf.ExpectedMounts, em...) + return nil + } +} + func securityPolicyFromContainers(containers []securitypolicy.ContainerConfig) (string, error) { pc, err := helpers.PolicyContainersFromConfigs(containers) if err != nil { @@ -117,3 +127,127 @@ func Test_RunSimpleAlpineContainer_WithPolicy_Allowed(t *testing.T) { startContainer(t, client, ctx, containerID) stopContainer(t, client, ctx, containerID) } + +func Test_RunContainers_WithSyncHooks_Positive(t *testing.T) { + requireFeatures(t, featureLCOW, featureLCOWIntegrity) + + type config struct { + name string + waiterSideEffect func(containerConfig *securitypolicy.ContainerConfig) + shouldError bool + expectedErrString string + } + + for _, testConfig := range []config{ + { + name: "ValidWaitPath", + waiterSideEffect: nil, + shouldError: false, + }, + { + // This is a long test that will wait for a timeout + name: "InvalidWaitPath", + waiterSideEffect: func(cfg *securitypolicy.ContainerConfig) { + cfg.ExpectedMounts = []string{"/mnt/shared/container-B/wrong-sync-file"} + }, + shouldError: true, + expectedErrString: "timeout while waiting for path", + }, + } { + t.Run(testConfig.name, func(t *testing.T) { + // create container #1 that writes a file + touchCmdArgs := []string{"ash", "-c", "touch /mnt/shared/container-A/sync-file && while true; do echo hello; sleep 1; done"} + configWriter := securitypolicy.ContainerConfig{ + ImageName: "alpine:latest", + Command: touchCmdArgs, + } + // create container #2 that waits for a path to appear + echoCmdArgs := []string{"ash", "-c", "while true; do echo hello2; sleep 1; done"} + configWaiter := securitypolicy.ContainerConfig{ + ImageName: "alpine:latest", + Command: echoCmdArgs, + ExpectedMounts: []string{"/mnt/shared/container-B/sync-file"}, + } + if testConfig.waiterSideEffect != nil { + testConfig.waiterSideEffect(&configWaiter) + } + + // create appropriate policies for the two containers + containerConfigs := append(helpers.DefaultContainerConfigs(), configWriter, configWaiter) + policyContainers, err := helpers.PolicyContainersFromConfigs(containerConfigs) + if err != nil { + t.Fatalf("failed to create security policy containers: %s", err) + } + policy := securitypolicy.NewSecurityPolicy(false, policyContainers) + policyString, err := policy.EncodeToString() + if err != nil { + t.Fatalf("failed to generate security policy string: %s", err) + } + + client := newTestRuntimeClient(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // create pod with security policy + podRequest := sandboxRequestWithPolicy(t, policyString) + podID := runPodSandbox(t, client, ctx, podRequest) + defer removePodSandbox(t, client, ctx, podID) + defer stopPodSandbox(t, client, ctx, podID) + + sbMountWriter := v1alpha2.Mount{ + HostPath: "sandbox://host/path", + ContainerPath: "/mnt/shared/container-A", + Readonly: false, + } + // start containers async and make sure that both of the start + requestWriter := getCreateContainerRequest( + podID, + "alpine-writer", + "alpine:latest", + touchCmdArgs, + podRequest.Config, + ) + requestWriter.Config.Mounts = append(requestWriter.Config.Mounts, &sbMountWriter) + + sbMountWaiter := v1alpha2.Mount{ + HostPath: "sandbox://host/path", + ContainerPath: "/mnt/shared/container-B", + Readonly: false, + } + requestWaiter := getCreateContainerRequest( + podID, + "alpine-waiter", + "alpine:latest", + echoCmdArgs, + podRequest.Config, + ) + requestWaiter.Config.Mounts = append(requestWaiter.Config.Mounts, &sbMountWaiter) + + cidWriter := createContainer(t, client, ctx, requestWriter) + cidWaiter := createContainer(t, client, ctx, requestWaiter) + + startContainer(t, client, ctx, cidWriter) + defer removeContainer(t, client, ctx, cidWriter) + defer stopContainer(t, client, ctx, cidWriter) + + if !testConfig.shouldError { + startContainer(t, client, ctx, cidWaiter) + defer removeContainer(t, client, ctx, cidWaiter) + defer stopContainer(t, client, ctx, cidWaiter) + } else { + _, err := client.StartContainer(ctx, &v1alpha2.StartContainerRequest{ + ContainerId: cidWaiter, + }) + if err == nil { + defer removeContainer(t, client, ctx, cidWaiter) + defer stopContainer(t, client, ctx, cidWaiter) + t.Fatalf("should fail, succeeded instead") + } else { + if !strings.Contains(err.Error(), testConfig.expectedErrString) { + t.Fatalf("expected error: %q, got: %q", testConfig.expectedErrString, err) + } + } + } + }) + } +} diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/tools/securitypolicy/helpers/helpers.go b/test/vendor/github.com/Microsoft/hcsshim/internal/tools/securitypolicy/helpers/helpers.go index 42f6fd8a70..103b2bae3a 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/tools/securitypolicy/helpers/helpers.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/tools/securitypolicy/helpers/helpers.go @@ -137,7 +137,13 @@ func PolicyContainersFromConfigs(containerConfigs []securitypolicy.ContainerConf workingDir = containerConfig.WorkingDir } - container, err := securitypolicy.NewContainer(containerConfig.Command, layerHashes, envRules, workingDir) + container, err := securitypolicy.NewContainer( + containerConfig.Command, + layerHashes, + envRules, + workingDir, + containerConfig.ExpectedMounts, + ) if err != nil { return nil, err } diff --git a/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicy.go b/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicy.go index d7ec30bdd6..207a2779b3 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicy.go +++ b/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicy.go @@ -211,15 +211,16 @@ type ExpectedMounts struct { // NewContainer creates a new Container instance from the provided values // or an error if envRules validation fails. -func NewContainer(command, layers []string, envRules []EnvRuleConfig, workingDir string) (*Container, error) { +func NewContainer(command, layers []string, envRules []EnvRuleConfig, workingDir string, eMounts []string) (*Container, error) { if err := validateEnvRules(envRules); err != nil { return nil, err } return &Container{ - Command: newCommandArgs(command), - Layers: newLayers(layers), - EnvRules: newEnvRules(envRules), - WorkingDir: workingDir, + Command: newCommandArgs(command), + Layers: newLayers(layers), + EnvRules: newEnvRules(envRules), + WorkingDir: workingDir, + ExpectedMounts: newExpectedMounts(eMounts), }, nil } @@ -279,6 +280,16 @@ func newLayers(ls []string) Layers { } } +func newExpectedMounts(em []string) ExpectedMounts { + mounts := map[string]string{} + for i, m := range em { + mounts[strconv.Itoa(i)] = m + } + return ExpectedMounts{ + Elements: mounts, + } +} + // Custom JSON marshalling to add `lenth` field that matches the number of // elements present in the `elements` field. func (c Containers) MarshalJSON() ([]byte, error) { diff --git a/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicyenforcer.go b/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicyenforcer.go index 3024f8f5cc..b435080e9e 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicyenforcer.go +++ b/test/vendor/github.com/Microsoft/hcsshim/pkg/securitypolicy/securitypolicyenforcer.go @@ -10,12 +10,10 @@ import ( "strings" "sync" - "github.com/google/go-cmp/cmp" - oci "github.com/opencontainers/runtime-spec/specs-go" - - "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/hooks" "github.com/Microsoft/hcsshim/pkg/annotations" + "github.com/google/go-cmp/cmp" + oci "github.com/opencontainers/runtime-spec/specs-go" ) type SecurityPolicyEnforcer interface { @@ -503,15 +501,22 @@ func possibleIndicesForID(containerID string, mapping map[int]map[string]struct{ // the expected mounts appear prior container start. At the moment enforcement // is expected to take place inside LCOW UVM. // -// Supported scenarios: -// 1. expected mount is provided as a path inside the sandbox, and it should resolve inside UVM -// e.g, "sandbox://path/on/the/host", which will correspond to "/run/gcs/c//sandboxMounts/path/on/the/host" -// 2. expected mount is provided as a path under a sandbox mount path inside container, e.g., -// sandbox mount is at path "/sandbox/mount" and wait path is "/sandbox/mount/wait/path", which -// corresponds to "/run/gcs/c//sandboxMounts/path/on/the/host/wait/path" -// 3. expected mount is provided as an arbitrary path inside container, and it should resolve -// inside UVM, e.g., "/arbitrary/container/path", which corresponds to -// "/run/gcs/c//rootfs/arbitrary/container/path" +// Expected mount is provided as a path under a sandbox mount path inside +// container, e.g., sandbox mount is at path "/path/in/container" and wait path +// is "/path/in/container/wait/path", which corresponds to +// "/run/gcs/c//sandboxMounts/path/on/the/host/wait/path" +// +// Iterates through container mounts to identify the correct sandbox +// mount where the wait path is nested under. The mount spec will +// be something like: +// { +// "source": "/run/gcs/c//sandboxMounts/path/on/host", +// "destination": "/path/in/container" +// } +// The wait path will be "/path/in/container/wait/path". To find the corresponding +// sandbox mount do a prefix match on wait path against all container mounts +// Destination and resolve the full path inside UVM. For example above it becomes +// "/run/gcs/c//sandboxMounts/path/on/host/wait/path" func (pe *StandardSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(containerID string, spec *oci.Spec) error { pe.mutex.Lock() defer pe.mutex.Unlock() @@ -550,35 +555,17 @@ func (pe *StandardSecurityPolicyEnforcer) EnforceExpectedMountsPolicy(containerI var wPaths []string for _, mount := range wMounts { - // By default, handle scenario #3 and resolve container path to the actual path inside UVM. - wp := filepath.Join(guestpath.LCOWRootPrefixInUVM, containerID, guestpath.RootfsPath, mount) - if strings.HasPrefix(mount, guestpath.SandboxMountPrefix) { - // This covers case #1, and we replace sandbox mount prefix with the sandbox - // mounts path inside UVM - sandboxPath := strings.TrimPrefix(mount, guestpath.SandboxMountPrefix) - wp = filepath.Join(guestpath.LCOWRootPrefixInUVM, "sandboxMounts", sandboxPath) - } else { - // This covers case #2. Iterate through container mounts to identify the - // correct sandbox mount where the wait path is nested under. The mount - // spec will be something like: - // { - // "source": "/run/gcs/c//sandboxMounts/path/on/host", - // "destination": "/sandbox/mount" - // } - // The wait path will be "/sandbox/mount/wait/path". To find the corresponding - // sandbox mount do a prefix match on wait path against all container mounts Destination - // and resolve the full path inside UVM. For example above it becomes - // "/run/gcs/c//sandboxMounts/path/on/host/wait/path" - for _, m := range spec.Mounts { - if strings.HasPrefix(mount, m.Destination) { - wp = filepath.Join(m.Source, strings.TrimPrefix(mount, m.Destination)) - break - } - } - if wp == "" { - return fmt.Errorf("invalid mount path: %q", mount) + var wp string + for _, m := range spec.Mounts { + // prefix matching to find correct sandbox mount + if strings.HasPrefix(mount, m.Destination) { + wp = filepath.Join(m.Source, strings.TrimPrefix(mount, m.Destination)) + break } } + if wp == "" { + return fmt.Errorf("invalid mount path: %q", mount) + } wPaths = append(wPaths, filepath.Clean(wp)) } From f2d32fa74104ef8ddf58d4d61d3af64617996955 Mon Sep 17 00:00:00 2001 From: Maksim An Date: Tue, 15 Mar 2022 16:45:56 -0700 Subject: [PATCH 3/3] pr feedback: explict split between positive and negative tests Signed-off-by: Maksim An --- test/cri-containerd/policy_test.go | 243 +++++++++++++++-------------- 1 file changed, 126 insertions(+), 117 deletions(-) diff --git a/test/cri-containerd/policy_test.go b/test/cri-containerd/policy_test.go index d5e99e64a5..f46f34b555 100644 --- a/test/cri-containerd/policy_test.go +++ b/test/cri-containerd/policy_test.go @@ -5,6 +5,7 @@ package cri_containerd import ( "context" + "fmt" "strings" "testing" @@ -128,126 +129,134 @@ func Test_RunSimpleAlpineContainer_WithPolicy_Allowed(t *testing.T) { stopContainer(t, client, ctx, containerID) } -func Test_RunContainers_WithSyncHooks_Positive(t *testing.T) { +func syncContainerConfigs(writePath, waitPath string) (writer, waiter *securitypolicy.ContainerConfig) { + writerCmdArgs := []string{"ash", "-c", fmt.Sprintf("touch %s && while true; do echo hello1; sleep 1; done", writePath)} + writer = &securitypolicy.ContainerConfig{ + ImageName: "alpine:latest", + Command: writerCmdArgs, + } + // create container #2 that waits for a path to appear + echoCmdArgs := []string{"ash", "-c", "while true; do echo hello2; sleep 1; done"} + waiter = &securitypolicy.ContainerConfig{ + ImageName: "alpine:latest", + Command: echoCmdArgs, + ExpectedMounts: []string{waitPath}, + } + return writer, waiter +} + +func syncContainerRequests( + writer, waiter *securitypolicy.ContainerConfig, + podID string, + podConfig *v1alpha2.PodSandboxConfig, +) (writerReq, waiterReq *v1alpha2.CreateContainerRequest) { + writerReq = getCreateContainerRequest( + podID, + "alpine-writer", + "alpine:latest", + writer.Command, + podConfig, + ) + writerReq.Config.Mounts = append(writerReq.Config.Mounts, &v1alpha2.Mount{ + HostPath: "sandbox://host/path", + ContainerPath: "/mnt/shared/container-A", + }) + + waiterReq = getCreateContainerRequest( + podID, + "alpine-waiter", + "alpine:latest", + waiter.Command, + podConfig, + ) + waiterReq.Config.Mounts = append(waiterReq.Config.Mounts, &v1alpha2.Mount{ + // The HostPath must be the same as for the "writer" container + HostPath: "sandbox://host/path", + ContainerPath: "/mnt/shared/container-B", + }) + + return writerReq, waiterReq +} + +func Test_RunContainers_WithSyncHooks_ValidWaitPath(t *testing.T) { + requireFeatures(t, featureLCOW, featureLCOWIntegrity) + + writerCfg, waiterCfg := syncContainerConfigs( + "/mnt/shared/container-A/sync-file", "/mnt/shared/container-B/sync-file") + + containerConfigs := append(helpers.DefaultContainerConfigs(), *writerCfg, *waiterCfg) + policyString, err := securityPolicyFromContainers(containerConfigs) + if err != nil { + t.Fatalf("failed to generate security policy string: %s", err) + } + + client := newTestRuntimeClient(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // create pod with security policy + podRequest := sandboxRequestWithPolicy(t, policyString) + podID := runPodSandbox(t, client, ctx, podRequest) + defer removePodSandbox(t, client, ctx, podID) + defer stopPodSandbox(t, client, ctx, podID) + + writerReq, waiterReq := syncContainerRequests(writerCfg, waiterCfg, podID, podRequest.Config) + + cidWriter := createContainer(t, client, ctx, writerReq) + cidWaiter := createContainer(t, client, ctx, waiterReq) + + startContainer(t, client, ctx, cidWriter) + defer removeContainer(t, client, ctx, cidWriter) + defer stopContainer(t, client, ctx, cidWriter) + + startContainer(t, client, ctx, cidWaiter) + defer removeContainer(t, client, ctx, cidWaiter) + defer stopContainer(t, client, ctx, cidWaiter) +} + +func Test_RunContainers_WithSyncHooks_InvalidWaitPath(t *testing.T) { requireFeatures(t, featureLCOW, featureLCOWIntegrity) - type config struct { - name string - waiterSideEffect func(containerConfig *securitypolicy.ContainerConfig) - shouldError bool - expectedErrString string + writerCfg, waiterCfg := syncContainerConfigs( + "/mnt/shared/container-A/sync-file", + "/mnt/shared/container-B/sync-file-invalid", // NOTE: this is an invalid wait path + ) + + containerConfigs := append(helpers.DefaultContainerConfigs(), *writerCfg, *waiterCfg) + policyString, err := securityPolicyFromContainers(containerConfigs) + if err != nil { + t.Fatalf("failed to generate security policy string: %s", policyString) } - for _, testConfig := range []config{ - { - name: "ValidWaitPath", - waiterSideEffect: nil, - shouldError: false, - }, - { - // This is a long test that will wait for a timeout - name: "InvalidWaitPath", - waiterSideEffect: func(cfg *securitypolicy.ContainerConfig) { - cfg.ExpectedMounts = []string{"/mnt/shared/container-B/wrong-sync-file"} - }, - shouldError: true, - expectedErrString: "timeout while waiting for path", - }, - } { - t.Run(testConfig.name, func(t *testing.T) { - // create container #1 that writes a file - touchCmdArgs := []string{"ash", "-c", "touch /mnt/shared/container-A/sync-file && while true; do echo hello; sleep 1; done"} - configWriter := securitypolicy.ContainerConfig{ - ImageName: "alpine:latest", - Command: touchCmdArgs, - } - // create container #2 that waits for a path to appear - echoCmdArgs := []string{"ash", "-c", "while true; do echo hello2; sleep 1; done"} - configWaiter := securitypolicy.ContainerConfig{ - ImageName: "alpine:latest", - Command: echoCmdArgs, - ExpectedMounts: []string{"/mnt/shared/container-B/sync-file"}, - } - if testConfig.waiterSideEffect != nil { - testConfig.waiterSideEffect(&configWaiter) - } - - // create appropriate policies for the two containers - containerConfigs := append(helpers.DefaultContainerConfigs(), configWriter, configWaiter) - policyContainers, err := helpers.PolicyContainersFromConfigs(containerConfigs) - if err != nil { - t.Fatalf("failed to create security policy containers: %s", err) - } - policy := securitypolicy.NewSecurityPolicy(false, policyContainers) - policyString, err := policy.EncodeToString() - if err != nil { - t.Fatalf("failed to generate security policy string: %s", err) - } - - client := newTestRuntimeClient(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // create pod with security policy - podRequest := sandboxRequestWithPolicy(t, policyString) - podID := runPodSandbox(t, client, ctx, podRequest) - defer removePodSandbox(t, client, ctx, podID) - defer stopPodSandbox(t, client, ctx, podID) - - sbMountWriter := v1alpha2.Mount{ - HostPath: "sandbox://host/path", - ContainerPath: "/mnt/shared/container-A", - Readonly: false, - } - // start containers async and make sure that both of the start - requestWriter := getCreateContainerRequest( - podID, - "alpine-writer", - "alpine:latest", - touchCmdArgs, - podRequest.Config, - ) - requestWriter.Config.Mounts = append(requestWriter.Config.Mounts, &sbMountWriter) - - sbMountWaiter := v1alpha2.Mount{ - HostPath: "sandbox://host/path", - ContainerPath: "/mnt/shared/container-B", - Readonly: false, - } - requestWaiter := getCreateContainerRequest( - podID, - "alpine-waiter", - "alpine:latest", - echoCmdArgs, - podRequest.Config, - ) - requestWaiter.Config.Mounts = append(requestWaiter.Config.Mounts, &sbMountWaiter) - - cidWriter := createContainer(t, client, ctx, requestWriter) - cidWaiter := createContainer(t, client, ctx, requestWaiter) - - startContainer(t, client, ctx, cidWriter) - defer removeContainer(t, client, ctx, cidWriter) - defer stopContainer(t, client, ctx, cidWriter) - - if !testConfig.shouldError { - startContainer(t, client, ctx, cidWaiter) - defer removeContainer(t, client, ctx, cidWaiter) - defer stopContainer(t, client, ctx, cidWaiter) - } else { - _, err := client.StartContainer(ctx, &v1alpha2.StartContainerRequest{ - ContainerId: cidWaiter, - }) - if err == nil { - defer removeContainer(t, client, ctx, cidWaiter) - defer stopContainer(t, client, ctx, cidWaiter) - t.Fatalf("should fail, succeeded instead") - } else { - if !strings.Contains(err.Error(), testConfig.expectedErrString) { - t.Fatalf("expected error: %q, got: %q", testConfig.expectedErrString, err) - } - } - } - }) + client := newTestRuntimeClient(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // create pod with security policy + podRequest := sandboxRequestWithPolicy(t, policyString) + podID := runPodSandbox(t, client, ctx, podRequest) + defer removePodSandbox(t, client, ctx, podID) + defer stopPodSandbox(t, client, ctx, podID) + + writerReq, waiterReq := syncContainerRequests(writerCfg, waiterCfg, podID, podRequest.Config) + cidWriter := createContainer(t, client, ctx, writerReq) + cidWaiter := createContainer(t, client, ctx, waiterReq) + + startContainer(t, client, ctx, cidWriter) + defer removeContainer(t, client, ctx, cidWriter) + defer stopContainer(t, client, ctx, cidWriter) + + _, err = client.StartContainer(ctx, &v1alpha2.StartContainerRequest{ + ContainerId: cidWaiter, + }) + expectedErrString := "timeout while waiting for path" + if err == nil { + defer removeContainer(t, client, ctx, cidWaiter) + defer stopContainer(t, client, ctx, cidWaiter) + t.Fatalf("should fail, succeeded instead") + } else { + if !strings.Contains(err.Error(), expectedErrString) { + t.Fatalf("expected error: %q, got: %q", expectedErrString, err) + } } }