From 209864385ea3fb137cccee219c8ceda54591bc7b Mon Sep 17 00:00:00 2001 From: Hamza El-Saawy Date: Wed, 14 May 2025 16:38:42 -0400 Subject: [PATCH 1/2] Add `uvm://` mount support for LCOW Allow privileged LCOW containers to mount the uVM's filesystem. This supports cadvisor and related tools to monitor the pod and associated containers without needing to run directly within the uVM. See: https://github.com/google/cadvisor Signed-off-by: Hamza El-Saawy --- .../guest/runtime/hcsv2/workload_container.go | 88 ++++++++++++------- internal/hcsoci/resources_lcow.go | 8 +- 2 files changed, 60 insertions(+), 36 deletions(-) diff --git a/internal/guest/runtime/hcsv2/workload_container.go b/internal/guest/runtime/hcsv2/workload_container.go index df9c87f40d..898ba43372 100644 --- a/internal/guest/runtime/hcsv2/workload_container.go +++ b/internal/guest/runtime/hcsv2/workload_container.go @@ -34,22 +34,23 @@ func mkdirAllModePerm(target string) error { func updateSandboxMounts(sbid string, spec *oci.Spec) error { for i, m := range spec.Mounts { - if strings.HasPrefix(m.Source, guestpath.SandboxMountPrefix) { - sandboxSource := specGuest.SandboxMountSource(sbid, m.Source) + if !strings.HasPrefix(m.Source, guestpath.SandboxMountPrefix) { + continue + } + sandboxSource := specGuest.SandboxMountSource(sbid, m.Source) - // filepath.Join cleans the resulting path before returning, so it would resolve the relative path if one was given. - // Hence, we need to ensure that the resolved path is still under the correct directory - if !strings.HasPrefix(sandboxSource, specGuest.SandboxMountsDir(sbid)) { - return errors.Errorf("mount path %v for mount %v is not within sandbox's mounts dir", sandboxSource, m.Source) - } + // filepath.Join cleans the resulting path before returning, so it would resolve the relative path if one was given. + // Hence, we need to ensure that the resolved path is still under the correct directory + if !strings.HasPrefix(sandboxSource, specGuest.SandboxMountsDir(sbid)) { + return errors.Errorf("mount path %v for mount %v is not within sandbox's mounts dir", sandboxSource, m.Source) + } - spec.Mounts[i].Source = sandboxSource + spec.Mounts[i].Source = sandboxSource - _, err := os.Stat(sandboxSource) - if os.IsNotExist(err) { - if err := mkdirAllModePerm(sandboxSource); err != nil { - return err - } + _, err := os.Stat(sandboxSource) + if os.IsNotExist(err) { + if err := mkdirAllModePerm(sandboxSource); err != nil { + return err } } } @@ -58,28 +59,29 @@ func updateSandboxMounts(sbid string, spec *oci.Spec) error { func updateHugePageMounts(sbid string, spec *oci.Spec) error { for i, m := range spec.Mounts { - if strings.HasPrefix(m.Source, guestpath.HugePagesMountPrefix) { - mountsDir := specGuest.HugePagesMountsDir(sbid) - subPath := strings.TrimPrefix(m.Source, guestpath.HugePagesMountPrefix) - pageSize := strings.Split(subPath, string(os.PathSeparator))[0] - hugePageMountSource := filepath.Join(mountsDir, subPath) - - // filepath.Join cleans the resulting path before returning so it would resolve the relative path if one was given. - // Hence, we need to ensure that the resolved path is still under the correct directory - if !strings.HasPrefix(hugePageMountSource, mountsDir) { - return errors.Errorf("mount path %v for mount %v is not within hugepages's mounts dir", hugePageMountSource, m.Source) - } + if !strings.HasPrefix(m.Source, guestpath.HugePagesMountPrefix) { + continue + } + mountsDir := specGuest.HugePagesMountsDir(sbid) + subPath := strings.TrimPrefix(m.Source, guestpath.HugePagesMountPrefix) + pageSize := strings.Split(subPath, string(os.PathSeparator))[0] + hugePageMountSource := filepath.Join(mountsDir, subPath) + + // filepath.Join cleans the resulting path before returning so it would resolve the relative path if one was given. + // Hence, we need to ensure that the resolved path is still under the correct directory + if !strings.HasPrefix(hugePageMountSource, mountsDir) { + return errors.Errorf("mount path %v for mount %v is not within hugepages's mounts dir", hugePageMountSource, m.Source) + } - spec.Mounts[i].Source = hugePageMountSource + spec.Mounts[i].Source = hugePageMountSource - _, err := os.Stat(hugePageMountSource) - if os.IsNotExist(err) { - if err := mkdirAllModePerm(hugePageMountSource); err != nil { - return err - } - if err := unix.Mount("none", hugePageMountSource, "hugetlbfs", 0, "pagesize="+pageSize); err != nil { - return errors.Errorf("mount operation failed for %v failed with error %v", hugePageMountSource, err) - } + _, err := os.Stat(hugePageMountSource) + if os.IsNotExist(err) { + if err := mkdirAllModePerm(hugePageMountSource); err != nil { + return err + } + if err := unix.Mount("none", hugePageMountSource, "hugetlbfs", 0, "pagesize="+pageSize); err != nil { + return errors.Errorf("mount operation failed for %v failed with error %v", hugePageMountSource, err) } } } @@ -125,6 +127,22 @@ func updateBlockDeviceMounts(spec *oci.Spec) error { return nil } +func updateUVMMounts(spec *oci.Spec) error { + for i, m := range spec.Mounts { + if !strings.HasPrefix(m.Source, guestpath.UVMMountPrefix) { + continue + } + hostPath := strings.TrimPrefix(m.Source, guestpath.UVMMountPrefix) + + spec.Mounts[i].Source = hostPath + + if _, err := os.Stat(hostPath); err != nil { + return errors.Wrap(err, "could not open uVM mount target") + } + } + return nil +} + func specHasGPUDevice(spec *oci.Spec) bool { for _, d := range spec.Windows.Devices { if d.IDType == "gpu" { @@ -160,6 +178,10 @@ func setupWorkloadContainerSpec(ctx context.Context, sbid, id string, spec *oci. return fmt.Errorf("failed to update block device mounts for container %v in sandbox %v: %w", id, sbid, err) } + if err = updateUVMMounts(spec); err != nil { + return errors.Wrapf(err, "failed to update uVM mounts for container %v in sandbox %v", id, sbid) + } + // Add default mounts for container networking (e.g. /etc/hostname, /etc/hosts), // if spec didn't override them explicitly. networkingMounts := specGuest.GenerateWorkloadContainerNetworkMounts(sbid, spec) diff --git a/internal/hcsoci/resources_lcow.go b/internal/hcsoci/resources_lcow.go index 633bde02eb..b961de72c0 100644 --- a/internal/hcsoci/resources_lcow.go +++ b/internal/hcsoci/resources_lcow.go @@ -129,9 +129,11 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * mt = "bind" } coi.Spec.Mounts[i].Type = mt - } else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) { - // Mounts that map to a path in UVM are specified with 'sandbox://' prefix. - // example: sandbox:///a/dirInUvm destination:/b/dirInContainer + } else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) || strings.HasPrefix(mount.Source, guestpath.UVMMountPrefix) { + // Mounts that map to a path in UVM are specified with a 'sandbox://' or 'uvm://' prefix. + // examples: + // - sandbox:///a/dirInUvm destination:/b/dirInContainer + // - uvm:///a/dirInUvm destination:/b/dirInContainer uvmPathForFile = mount.Source } else if strings.HasPrefix(mount.Source, guestpath.HugePagesMountPrefix) { // currently we only support 2M hugepage size From 02ebb7c05ec03f9dc7f031d6b2f2aad663633dcc Mon Sep 17 00:00:00 2001 From: Hamza El-Saawy Date: Fri, 23 May 2025 11:18:52 -0400 Subject: [PATCH 2/2] PR: path name Signed-off-by: Hamza El-Saawy --- internal/guest/runtime/hcsv2/workload_container.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/guest/runtime/hcsv2/workload_container.go b/internal/guest/runtime/hcsv2/workload_container.go index 898ba43372..e2b52137b4 100644 --- a/internal/guest/runtime/hcsv2/workload_container.go +++ b/internal/guest/runtime/hcsv2/workload_container.go @@ -132,11 +132,11 @@ func updateUVMMounts(spec *oci.Spec) error { if !strings.HasPrefix(m.Source, guestpath.UVMMountPrefix) { continue } - hostPath := strings.TrimPrefix(m.Source, guestpath.UVMMountPrefix) + uvmPath := strings.TrimPrefix(m.Source, guestpath.UVMMountPrefix) - spec.Mounts[i].Source = hostPath + spec.Mounts[i].Source = uvmPath - if _, err := os.Stat(hostPath); err != nil { + if _, err := os.Stat(uvmPath); err != nil { return errors.Wrap(err, "could not open uVM mount target") } }