diff --git a/test/functional/wcow_test.go b/test/functional/wcow_test.go index b806835bad..174405c7bb 100644 --- a/test/functional/wcow_test.go +++ b/test/functional/wcow_test.go @@ -380,12 +380,12 @@ func TestWCOWArgonShim(t *testing.T) { // For cleanup on failure defer func() { if argonShimMounted { - layerspkg.UnmountContainerLayers(context.Background(), append(imageLayers, argonShimScratchDir), "", nil, layerspkg.UnmountOperationAll) + layerspkg.UnmountContainerLayers(context.Background(), append(imageLayers, argonShimScratchDir), "", "", nil, layerspkg.UnmountOperationAll) } }() // This is a cheat but stops us re-writing exactly the same code just for test - argonShimLocalMountPath, err := layerspkg.MountContainerLayers(context.Background(), append(imageLayers, argonShimScratchDir), "", nil) + argonShimLocalMountPath, err := layerspkg.MountContainerLayers(context.Background(), append(imageLayers, argonShimScratchDir), "", "", nil) if err != nil { t.Fatal(err) } @@ -418,7 +418,7 @@ func TestWCOWArgonShim(t *testing.T) { } runShimCommands(t, argonShim) stopContainer(t, argonShim) - if err := layerspkg.UnmountContainerLayers(context.Background(), append(imageLayers, argonShimScratchDir), "", nil, layerspkg.UnmountOperationAll); err != nil { + if err := layerspkg.UnmountContainerLayers(context.Background(), append(imageLayers, argonShimScratchDir), "", "", nil, layerspkg.UnmountOperationAll); err != nil { t.Fatal(err) } argonShimMounted = false diff --git a/test/vendor/github.com/Microsoft/hcsshim/README.md b/test/vendor/github.com/Microsoft/hcsshim/README.md index 95c3003656..86959fbf27 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/README.md +++ b/test/vendor/github.com/Microsoft/hcsshim/README.md @@ -2,9 +2,63 @@ [![Build status](https://github.com/microsoft/hcsshim/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/microsoft/hcsshim/actions?query=branch%3Amaster) -This package contains the Golang interface for using the Windows [Host Compute Service](https://techcommunity.microsoft.com/t5/containers/introducing-the-host-compute-service-hcs/ba-p/382332) (HCS) to launch and manage [Windows Containers](https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/). It also contains other helpers and functions for managing Windows Containers such as the Golang interface for the Host Network Service (HNS). +This package contains the Golang interface for using the Windows [Host Compute Service](https://techcommunity.microsoft.com/t5/containers/introducing-the-host-compute-service-hcs/ba-p/382332) (HCS) to launch and manage [Windows Containers](https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/). It also contains other helpers and functions for managing Windows Containers such as the Golang interface for the Host Network Service (HNS), as well as code for the [guest agent](./internal/guest/README.md) (commonly referred to as the GCS or Guest Compute Service in the codebase) used to support running Linux Hyper-V containers. -It is primarily used in the [Moby Project](https://github.com/moby/moby), but it can be freely used by other projects as well. +It is primarily used in the [Moby](https://github.com/moby/moby) and [Containerd](https://github.com/containerd/containerd) projects, but it can be freely used by other projects as well. + +## Building + +While this repository can be used as a library of sorts to call the HCS apis, there are a couple binaries built out of the repository as well. The main ones being the Linux guest agent, and an implementation of the [runtime v2 containerd shim api](https://github.com/containerd/containerd/blob/master/runtime/v2/README.md). +### Linux Hyper-V Container Guest Agent + +To build the Linux guest agent itself all that's needed is to set your GOOS to "Linux" and build out of ./cmd/gcs. +```powershell +C:\> $env:GOOS="linux" +C:\> go build .\cmd\gcs\ +``` + +or on a Linux machine +```sh +> go build ./cmd/gcs +``` + +If you want it to be packaged inside of a rootfs to boot with alongside all of the other tools then you'll need to provide a rootfs that it can be packaged inside of. An easy way is to export the rootfs of a container. + +```sh +docker pull busybox +docker run --name base_image_container busybox +docker export base_image_container | gzip > base.tar.gz +BASE=./base.tar.gz +make all +``` + +If the build is successful, in the `./out` folder you should see: +```sh +> ls ./out/ +delta.tar.gz initrd.img rootfs.tar.gz +``` + +### Containerd Shim +For info on the Runtime V2 API: https://github.com/containerd/containerd/blob/master/runtime/v2/README.md. + +Contrary to the typical Linux architecture of shim -> runc, the runhcs shim is used both to launch and manage the lifetime of containers. + +```powershell +C:\> $env:GOOS="windows" +C:\> go build .\cmd\containerd-shim-runhcs-v1 +``` + +Then place the binary in the same directory that Containerd is located at in your environment. A default Containerd configuration file can be generated by running: +```powershell +.\containerd.exe config default | Out-File "C:\Program Files\containerd\config.toml" -Encoding ascii +``` + +This config file will already have the shim set as the default runtime for cri interactions. + +To trial using the shim out with ctr.exe: +```powershell +C:\> ctr.exe run --runtime io.containerd.runhcs.v1 --rm mcr.microsoft.com/windows/nanoserver:2004 windows-test cmd /c "echo Hello World!" +``` ## Contributing @@ -16,7 +70,7 @@ When you submit a pull request, a CLA-bot will automatically determine whether y a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA. -We also ask that contributors [sign their commits](https://git-scm.com/docs/git-commit) using `git commit -s` or `git commit --signoff` to certify they either authored the work themselves or otherwise have permission to use it in this project. +We also ask that contributors [sign their commits](https://git-scm.com/docs/git-commit) using `git commit -s` or `git commit --signoff` to certify they either authored the work themselves or otherwise have permission to use it in this project. ## Code of Conduct diff --git a/test/vendor/github.com/Microsoft/hcsshim/ext4/dmverity/dmverity.go b/test/vendor/github.com/Microsoft/hcsshim/ext4/dmverity/dmverity.go index 1b914b6e44..2fe201bebc 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/ext4/dmverity/dmverity.go +++ b/test/vendor/github.com/Microsoft/hcsshim/ext4/dmverity/dmverity.go @@ -65,6 +65,7 @@ type VerityInfo struct { DataBlockSize uint32 HashBlockSize uint32 DataBlocks uint64 + Version uint32 } // MerkleTree constructs dm-verity hash-tree for a given byte array with a fixed salt (0-byte) and algorithm (sha256). @@ -191,5 +192,6 @@ func ReadDMVerityInfo(vhdPath string, offsetInBytes int64) (*VerityInfo, error) DataBlocks: dmvSB.DataBlocks, DataBlockSize: dmvSB.DataBlockSize, HashBlockSize: blockSize, + Version: dmvSB.Version, }, nil } diff --git a/test/vendor/github.com/Microsoft/hcsshim/hcn/hcn.go b/test/vendor/github.com/Microsoft/hcsshim/hcn/hcn.go index 82fddee9f7..9f6317cc05 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/hcn/hcn.go +++ b/test/vendor/github.com/Microsoft/hcsshim/hcn/hcn.go @@ -249,6 +249,25 @@ func TierAclPolicySupported() error { return platformDoesNotSupportError("TierAcl") } +// NetworkACLPolicySupported returns an error if the HCN version does not support NetworkACLPolicy +func NetworkACLPolicySupported() error { + supported := GetSupportedFeatures() + if supported.NetworkACL { + return nil + } + return platformDoesNotSupportError("NetworkACL") +} + +// NestedIpSetSupported returns an error if the HCN version does not support NestedIpSet +func NestedIpSetSupported() error { + supported := GetSupportedFeatures() + if supported.NestedIpSet { + return nil + } + return platformDoesNotSupportError("NestedIpSet") +} + + // RequestType are the different operations performed to settings. // Used to update the settings of Endpoint/Namespace objects. type RequestType string diff --git a/test/vendor/github.com/Microsoft/hcsshim/hcn/hcnglobals.go b/test/vendor/github.com/Microsoft/hcsshim/hcn/hcnglobals.go index d03c48736d..15a7e12943 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/hcn/hcnglobals.go +++ b/test/vendor/github.com/Microsoft/hcsshim/hcn/hcnglobals.go @@ -76,6 +76,12 @@ var ( //HNS 14.0 allows for TierAcl Policy support TierAclPolicyVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 14, Minor: 0}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}} + + //HNS 15.0 allows for NetworkACL Policy support + NetworkACLPolicyVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 15, Minor: 0}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}} + + //HNS 15.0 allows for NestedIpSet support + NestedIpSetVersion = VersionRanges{VersionRange{MinVersion: Version{Major: 15, Minor: 0}, MaxVersion: Version{Major: math.MaxInt32, Minor: math.MaxInt32}}} ) // GetGlobals returns the global properties of the HCN Service. diff --git a/test/vendor/github.com/Microsoft/hcsshim/hcn/hcnpolicy.go b/test/vendor/github.com/Microsoft/hcsshim/hcn/hcnpolicy.go index 29651bb5f1..4aec8534f6 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/hcn/hcnpolicy.go +++ b/test/vendor/github.com/Microsoft/hcsshim/hcn/hcnpolicy.go @@ -50,6 +50,7 @@ const ( SetPolicy NetworkPolicyType = "SetPolicy" NetworkL4Proxy NetworkPolicyType = "L4Proxy" LayerConstraint NetworkPolicyType = "LayerConstraint" + NetworkACL NetworkPolicyType = "NetworkACL" ) // NetworkPolicy is a collection of Policy settings for a Network. @@ -132,7 +133,7 @@ type AclPolicySetting struct { RemotePorts string `json:",omitempty"` RuleType RuleType `json:",omitempty"` Priority uint16 `json:",omitempty"` -} +} // QosPolicySetting sets Quality of Service bandwidth caps on an Endpoint. type QosPolicySetting struct { @@ -154,6 +155,19 @@ type SDNRoutePolicySetting struct { NeedEncap bool `json:",omitempty"` } +// NetworkACLPolicySetting creates ACL rules on a network +type NetworkACLPolicySetting struct { + Protocols string `json:",omitempty"` // EX: 6 (TCP), 17 (UDP), 1 (ICMPv4), 58 (ICMPv6), 2 (IGMP) + Action ActionType `json:","` + Direction DirectionType `json:","` + LocalAddresses string `json:",omitempty"` + RemoteAddresses string `json:",omitempty"` + LocalPorts string `json:",omitempty"` + RemotePorts string `json:",omitempty"` + RuleType RuleType `json:",omitempty"` + Priority uint16 `json:",omitempty"` +} + // FiveTuple is nested in L4ProxyPolicySetting for WFP support. type FiveTuple struct { Protocols string `json:",omitempty"` @@ -271,6 +285,7 @@ type SetPolicyType string const ( SetPolicyTypeIpSet SetPolicyType = "IPSET" + SetPolicyTypeNestedIpSet SetPolicyType = "NESTEDIPSET" ) // SetPolicySetting creates IPSets on network diff --git a/test/vendor/github.com/Microsoft/hcsshim/hcn/hcnsupport.go b/test/vendor/github.com/Microsoft/hcsshim/hcn/hcnsupport.go index 1d9fe762e9..4be8df17db 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/hcn/hcnsupport.go +++ b/test/vendor/github.com/Microsoft/hcsshim/hcn/hcnsupport.go @@ -1,9 +1,14 @@ package hcn import ( + "fmt" + "sync" + "github.com/sirupsen/logrus" ) +var versionOnce sync.Once + // SupportedFeatures are the features provided by the Service. type SupportedFeatures struct { Acl AclFeatures `json:"ACL"` @@ -20,6 +25,8 @@ type SupportedFeatures struct { L4Proxy bool `json:"L4Proxy"` // network policy that applies VFP rules to all endpoints on the network to redirect traffic L4WfpProxy bool `json:"L4WfpProxy"` // endpoint policy that applies WFP filters to redirect traffic to/from that endpoint TierAcl bool `json:"TierAcl"` + NetworkACL bool `json:"NetworkACL"` + NestedIpSet bool `json:"NestedIpSet"` } // AclFeatures are the supported ACL possibilities. @@ -71,6 +78,19 @@ func GetSupportedFeatures() SupportedFeatures { features.L4Proxy = isFeatureSupported(globals.Version, L4ProxyPolicyVersion) features.L4WfpProxy = isFeatureSupported(globals.Version, L4WfpProxyPolicyVersion) features.TierAcl = isFeatureSupported(globals.Version, TierAclPolicyVersion) + features.NetworkACL = isFeatureSupported(globals.Version, NetworkACLPolicyVersion) + features.NestedIpSet = isFeatureSupported(globals.Version, NestedIpSetVersion) + + // Only print the HCN version and features supported once, instead of everytime this is invoked. These logs are useful to + // debug incidents where there's confusion on if a feature is supported on the host machine. The sync.Once helps to avoid redundant + // spam of these anytime a check needs to be made for if an HCN feature is supported. This is a common occurrence in kubeproxy + // for example. + versionOnce.Do(func() { + logrus.WithFields(logrus.Fields{ + "version": fmt.Sprintf("%+v", globals.Version), + "supportedFeatures": fmt.Sprintf("%+v", features), + }).Info("HCN feature check") + }) return features } @@ -87,19 +107,15 @@ func isFeatureSupported(currentVersion Version, versionsSupported VersionRanges) func isFeatureInRange(currentVersion Version, versionRange VersionRange) bool { if currentVersion.Major < versionRange.MinVersion.Major { - logrus.Infof("currentVersion.Major < versionRange.MinVersion.Major: %v, %v", currentVersion.Major, versionRange.MinVersion.Major) return false } if currentVersion.Major > versionRange.MaxVersion.Major { - logrus.Infof("currentVersion.Major > versionRange.MaxVersion.Major: %v, %v", currentVersion.Major, versionRange.MaxVersion.Major) return false } if currentVersion.Major == versionRange.MinVersion.Major && currentVersion.Minor < versionRange.MinVersion.Minor { - logrus.Infof("currentVersion.Minor < versionRange.MinVersion.Major: %v, %v", currentVersion.Minor, versionRange.MinVersion.Minor) return false } if currentVersion.Major == versionRange.MaxVersion.Major && currentVersion.Minor > versionRange.MaxVersion.Minor { - logrus.Infof("currentVersion.Minor > versionRange.MaxVersion.Major: %v, %v", currentVersion.Minor, versionRange.MaxVersion.Minor) return false } return true diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/diag.go b/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/diag.go index cbe9595842..4d338707c0 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/diag.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/cmd/diag.go @@ -9,7 +9,6 @@ import ( "github.com/Microsoft/hcsshim/internal/logfields" "github.com/Microsoft/hcsshim/internal/shimdiag" "github.com/Microsoft/hcsshim/internal/uvm" - errorspkg "github.com/pkg/errors" ) // ExecInUvm is a helper function used to execute commands specified in `req` inside the given UVM. @@ -48,13 +47,21 @@ func ExecInShimHost(ctx context.Context, req *shimdiag.ExecProcessRequest) (int, if len(req.Args) > 1 { cmdArgsWithoutName = req.Args[1:] } + np, err := NewNpipeIO(ctx, req.Stdin, req.Stdout, req.Stderr, req.Terminal) + if err != nil { + return 0, err + } + defer np.Close(ctx) cmd := exec.Command(req.Args[0], cmdArgsWithoutName...) - output, err := cmd.CombinedOutput() + cmd.Stdin = np.Stdin() + cmd.Stdout = np.Stdout() + cmd.Stderr = np.Stderr() + err = cmd.Run() if err != nil { if exiterr, ok := err.(*exec.ExitError); ok { - return exiterr.ExitCode(), errorspkg.Wrapf(exiterr, "command output: %v", string(output)) + return exiterr.ExitCode(), exiterr } - return -1, errorspkg.Wrapf(err, "command output: %v", string(output)) + return -1, err } return 0, nil } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/guestrequest/types.go b/test/vendor/github.com/Microsoft/hcsshim/internal/guestrequest/types.go index 5ac526102d..548a8b43fd 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/guestrequest/types.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/guestrequest/types.go @@ -51,12 +51,31 @@ type LCOWMappedLayer struct { DeviceSizeInBytes uint64 `json:"DeviceSizeInBytes,omitempty"` } +// DeviceVerityInfo represents dm-verity metadata of a block device. +// Most of the fields can be directly mapped to table entries https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html +type DeviceVerityInfo struct { + // Ext4SizeInBytes is the size of ext4 file system + Ext4SizeInBytes int64 `json:",omitempty"` + // Version is the on-disk hash format + Version int `json:",omitempty"` + // Algorithm is the algo used to produce the hashes for dm-verity hash tree + Algorithm string `json:",omitempty"` + // SuperBlock is set to true if dm-verity super block is present on the device + SuperBlock bool `json:",omitempty"` + // RootDigest is the root hash of the dm-verity hash tree + RootDigest string `json:",omitempty"` + // Salt is the salt used to compute the root hash + Salt string `json:",omitempty"` + // BlockSize is the data device block size + BlockSize int `json:",omitempty"` +} + // Read-only layers over VPMem type LCOWMappedVPMemDevice struct { - DeviceNumber uint32 `json:"DeviceNumber,omitempty"` - MountPath string `json:"MountPath,omitempty"` - // Mapping is ignored when MountPath is not empty - MappingInfo *LCOWMappedLayer `json:"MappingInfo,omitempty"` + DeviceNumber uint32 `json:"DeviceNumber,omitempty"` + MountPath string `json:"MountPath,omitempty"` + MappingInfo *LCOWMappedLayer `json:"MappingInfo,omitempty"` + VerityInfo *DeviceVerityInfo `json:"VerityInfo,omitempty"` } type LCOWMappedVPCIDevice struct { diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go index 2e634c6e2a..c024605f73 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go @@ -42,12 +42,12 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r * containerRootInUVM := r.ContainerRootInUVM() if coi.Spec.Windows != nil && len(coi.Spec.Windows.LayerFolders) > 0 { log.G(ctx).Debug("hcsshim::allocateLinuxResources mounting storage") - rootPath, err := layers.MountContainerLayers(ctx, coi.Spec.Windows.LayerFolders, containerRootInUVM, coi.HostingSystem) + rootPath, err := layers.MountContainerLayers(ctx, coi.Spec.Windows.LayerFolders, containerRootInUVM, "", coi.HostingSystem) if err != nil { return errors.Wrap(err, "failed to mount container storage") } coi.Spec.Root.Path = rootPath - layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders, isSandbox) + layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders, "", isSandbox) r.SetLayers(layers) } else if coi.Spec.Root.Path != "" { // This is the "Plan 9" root filesystem. diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go index 46599be037..7f3c67140b 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_wcow.go @@ -52,12 +52,12 @@ func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r if coi.Spec.Root.Path == "" && (coi.HostingSystem != nil || coi.Spec.Windows.HyperV == nil) { log.G(ctx).Debug("hcsshim::allocateWindowsResources mounting storage") containerRootInUVM := r.ContainerRootInUVM() - containerRootPath, err := layers.MountContainerLayers(ctx, coi.Spec.Windows.LayerFolders, containerRootInUVM, coi.HostingSystem) + containerRootPath, err := layers.MountContainerLayers(ctx, coi.Spec.Windows.LayerFolders, containerRootInUVM, "", coi.HostingSystem) if err != nil { return errors.Wrap(err, "failed to mount container storage") } coi.Spec.Root.Path = containerRootPath - layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders, isSandbox) + layers := layers.NewImageLayers(coi.HostingSystem, containerRootInUVM, coi.Spec.Windows.LayerFolders, "", isSandbox) r.SetLayers(layers) } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsendpoint.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsendpoint.go index b36315a397..6f899c0d05 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsendpoint.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsendpoint.go @@ -20,6 +20,7 @@ type HNSEndpoint struct { IPv6Address net.IP `json:",omitempty"` DNSSuffix string `json:",omitempty"` DNSServerList string `json:",omitempty"` + DNSDomain string `json:",omitempty"` GatewayAddress string `json:",omitempty"` GatewayAddressV6 string `json:",omitempty"` EnableInternalDNS bool `json:",omitempty"` diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/layers/layers.go b/test/vendor/github.com/Microsoft/hcsshim/internal/layers/layers.go index 35ae9a8a39..86ff5e53a2 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/layers/layers.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/layers/layers.go @@ -6,6 +6,7 @@ package layers import ( "context" "fmt" + "os" "path/filepath" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" @@ -16,12 +17,14 @@ import ( "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" ) // ImageLayers contains all the layers for an image. type ImageLayers struct { vm *uvm.UtilityVM containerRootInUVM string + volumeMountPath string layers []string // In some instances we may want to avoid cleaning up the image layers, such as when tearing // down a sandbox container since the UVM will be torn down shortly after and the resources @@ -29,11 +32,12 @@ type ImageLayers struct { skipCleanup bool } -func NewImageLayers(vm *uvm.UtilityVM, containerRootInUVM string, layers []string, skipCleanup bool) *ImageLayers { +func NewImageLayers(vm *uvm.UtilityVM, containerRootInUVM string, layers []string, volumeMountPath string, skipCleanup bool) *ImageLayers { return &ImageLayers{ vm: vm, containerRootInUVM: containerRootInUVM, layers: layers, + volumeMountPath: volumeMountPath, skipCleanup: skipCleanup, } } @@ -51,7 +55,7 @@ func (layers *ImageLayers) Release(ctx context.Context, all bool) error { if layers.vm != nil { crp = containerRootfsPath(layers.vm, layers.containerRootInUVM) } - err := UnmountContainerLayers(ctx, layers.layers, crp, layers.vm, op) + err := UnmountContainerLayers(ctx, layers.layers, crp, layers.volumeMountPath, layers.vm, op) if err != nil { return err } @@ -67,9 +71,11 @@ func (layers *ImageLayers) Release(ctx context.Context, all bool) error { // v2: Xenon WCOW: Returns a CombinedLayersV2 structure where ContainerRootPath is a folder // inside the utility VM which is a GUID mapping of the scratch folder. Each // of the layers are the VSMB locations where the read-only layers are mounted. +// Job container: Returns the mount path on the host as a volume guid, with the volume mounted on +// the host at `volumeMountPath`. // // TODO dcantah: Keep better track of the layers that are added, don't simply discard the SCSI, VSMB, etc. resource types gotten inside. -func MountContainerLayers(ctx context.Context, layerFolders []string, guestRoot string, uvm *uvmpkg.UtilityVM) (_ string, err error) { +func MountContainerLayers(ctx context.Context, layerFolders []string, guestRoot, volumeMountPath string, uvm *uvmpkg.UtilityVM) (_ string, err error) { log.G(ctx).WithField("layerFolders", layerFolders).Debug("hcsshim::mountContainerLayers") if uvm == nil { @@ -100,6 +106,14 @@ func MountContainerLayers(ctx context.Context, layerFolders []string, guestRoot if err != nil { return "", err } + + // Mount the volume to a directory on the host if requested. This is the case for job containers. + if volumeMountPath != "" { + if err := mountSandboxVolume(ctx, volumeMountPath, mountPath); err != nil { + return "", err + } + } + return mountPath, nil } @@ -276,7 +290,7 @@ const ( ) // UnmountContainerLayers is a helper for clients to hide all the complexity of layer unmounting -func UnmountContainerLayers(ctx context.Context, layerFolders []string, containerRootPath string, uvm *uvmpkg.UtilityVM, op UnmountOperation) error { +func UnmountContainerLayers(ctx context.Context, layerFolders []string, containerRootPath, volumeMountPath string, uvm *uvmpkg.UtilityVM, op UnmountOperation) error { log.G(ctx).WithField("layerFolders", layerFolders).Debug("hcsshim::unmountContainerLayers") if uvm == nil { // Must be an argon - folders are mounted on the host @@ -286,6 +300,14 @@ func UnmountContainerLayers(ctx context.Context, layerFolders []string, containe if len(layerFolders) < 1 { return errors.New("need at least one layer for Unmount") } + + // Remove the mount point if there is one. This is the case for job containers. + if volumeMountPath != "" { + if err := removeSandboxMountPoint(ctx, volumeMountPath); err != nil { + return err + } + } + path := layerFolders[len(layerFolders)-1] if err := wclayer.UnprepareLayer(ctx, path); err != nil { return err @@ -398,3 +420,48 @@ func getScratchVHDPath(layerFolders []string) (string, error) { } return hostPath, nil } + +// Mount the sandbox vhd to a user friendly path. +func mountSandboxVolume(ctx context.Context, hostPath, volumeName string) (err error) { + log.G(ctx).WithFields(logrus.Fields{ + "hostpath": hostPath, + "volumeName": volumeName, + }).Debug("mounting volume for container") + + if _, err := os.Stat(hostPath); os.IsNotExist(err) { + if err := os.MkdirAll(hostPath, 0777); err != nil { + return err + } + } + + defer func() { + if err != nil { + os.RemoveAll(hostPath) + } + }() + + // Make sure volumeName ends with a trailing slash as required. + if volumeName[len(volumeName)-1] != '\\' { + volumeName += `\` // Be nice to clients and make sure well-formed for back-compat + } + + if err = windows.SetVolumeMountPoint(windows.StringToUTF16Ptr(hostPath), windows.StringToUTF16Ptr(volumeName)); err != nil { + return errors.Wrapf(err, "failed to mount sandbox volume to %s on host", hostPath) + } + return nil +} + +// Remove volume mount point. And remove folder afterwards. +func removeSandboxMountPoint(ctx context.Context, hostPath string) error { + log.G(ctx).WithFields(logrus.Fields{ + "hostpath": hostPath, + }).Debug("removing volume mount point for container") + + if err := windows.DeleteVolumeMountPoint(windows.StringToUTF16Ptr(hostPath)); err != nil { + return errors.Wrap(err, "failed to delete sandbox volume mount point") + } + if err := os.Remove(hostPath); err != nil { + return errors.Wrapf(err, "failed to remove sandbox mounted folder path %q", hostPath) + } + return nil +} diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/util.go b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/util.go index e57b52ae6c..85f52016ff 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/util.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/util.go @@ -16,3 +16,8 @@ func IsWCOW(s *specs.Spec) bool { func IsIsolated(s *specs.Spec) bool { return IsLCOW(s) || (s.Windows != nil && s.Windows.HyperV != nil) } + +// IsJobContainer checks if `s` is asking for a Windows job container. +func IsJobContainer(s *specs.Spec) bool { + return s.Annotations[AnnotationHostProcessContainer] == "true" +} diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/vpmem.go b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/vpmem.go index 061ea489b9..6c099bc840 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/vpmem.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/uvm/vpmem.go @@ -8,6 +8,8 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/Microsoft/hcsshim/ext4/dmverity" + "github.com/Microsoft/hcsshim/ext4/tar2ext4" "github.com/Microsoft/hcsshim/internal/guestrequest" "github.com/Microsoft/hcsshim/internal/hcs/resourcepaths" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" @@ -39,6 +41,42 @@ func newDefaultVPMemInfo(hostPath, uvmPath string) *vPMemInfoDefault { } } +// readVeritySuperBlock reads ext4 super block for a given VHD to then further read the dm-verity super block +// and root hash +func readVeritySuperBlock(ctx context.Context, layerPath string) (*guestrequest.DeviceVerityInfo, error) { + ext4sb, err := tar2ext4.ReadExt4SuperBlock(layerPath) + if err != nil { + return nil, errors.Wrap(err, "failed to read ext4 super block") + } + // Calculate the size of ext4 file system based on the information from ext4 super block, since + // the dm-verity information is expected to be appended, the size of ext4 data will be the offset + // of the dm-verity super block, followed by merkle hash tree + ext4BlockSize := 1024 * (1 << ext4sb.LogBlockSize) + ext4SizeInBytes := int64(ext4BlockSize) * int64(ext4sb.BlocksCountLow) + dmvsb, err := dmverity.ReadDMVerityInfo(layerPath, ext4SizeInBytes) + if err != nil { + return nil, errors.Wrap(err, "failed to read dm-verity super block") + } + log.G(ctx).WithFields(logrus.Fields{ + "layerPath": layerPath, + "rootHash": dmvsb.RootDigest, + "algorithm": dmvsb.Algorithm, + "salt": dmvsb.Salt, + "dataBlocks": dmvsb.DataBlocks, + "dataBlockSize": dmvsb.DataBlockSize, + }).Debug("dm-verity information") + + return &guestrequest.DeviceVerityInfo{ + Ext4SizeInBytes: ext4SizeInBytes, + BlockSize: ext4BlockSize, + RootDigest: dmvsb.RootDigest, + Algorithm: dmvsb.Algorithm, + Salt: dmvsb.Salt, + Version: int(dmvsb.Version), + SuperBlock: true, + }, nil +} + // findNextVPMemSlot finds next available VPMem slot. // // Lock MUST be held when calling this function. @@ -94,6 +132,7 @@ func (uvm *UtilityVM) addVPMemDefault(ctx context.Context, hostPath string) (_ s if err != nil { return "", err } + modification := &hcsschema.ModifySettingRequest{ RequestType: requesttype.Add, Settings: hcsschema.VirtualPMemDevice{ @@ -103,14 +142,28 @@ func (uvm *UtilityVM) addVPMemDefault(ctx context.Context, hostPath string) (_ s }, ResourcePath: fmt.Sprintf(resourcepaths.VPMemControllerResourceFormat, deviceNumber), } + uvmPath := fmt.Sprintf(lcowDefaultVPMemLayerFmt, deviceNumber) + guestSettings := guestrequest.LCOWMappedVPMemDevice{ + DeviceNumber: deviceNumber, + MountPath: uvmPath, + } + if v, iErr := readVeritySuperBlock(ctx, hostPath); iErr != nil { + log.G(ctx).WithError(iErr).WithField("hostPath", hostPath).Debug("unable to read dm-verity information from VHD") + } else { + if v != nil { + log.G(ctx).WithFields(logrus.Fields{ + "hostPath": hostPath, + "rootDigest": v.RootDigest, + }).Debug("adding VPMem with dm-verity") + } + guestSettings.VerityInfo = v + } + modification.GuestRequest = guestrequest.GuestRequest{ ResourceType: guestrequest.ResourceTypeVPMemDevice, RequestType: requesttype.Add, - Settings: guestrequest.LCOWMappedVPMemDevice{ - DeviceNumber: deviceNumber, - MountPath: uvmPath, - }, + Settings: guestSettings, } if err := uvm.modify(ctx, modification); err != nil { @@ -135,6 +188,14 @@ func (uvm *UtilityVM) removeVPMemDefault(ctx context.Context, hostPath string) e return nil } + var verity *guestrequest.DeviceVerityInfo + if v, _ := readVeritySuperBlock(ctx, hostPath); v != nil { + log.G(ctx).WithFields(logrus.Fields{ + "hostPath": hostPath, + "rootDigest": v.RootDigest, + }).Debug("removing VPMem with dm-verity") + verity = v + } modification := &hcsschema.ModifySettingRequest{ RequestType: requesttype.Remove, ResourcePath: fmt.Sprintf(resourcepaths.VPMemControllerResourceFormat, deviceNumber), @@ -144,6 +205,7 @@ func (uvm *UtilityVM) removeVPMemDefault(ctx context.Context, hostPath string) e Settings: guestrequest.LCOWMappedVPMemDevice{ DeviceNumber: deviceNumber, MountPath: device.uvmPath, + VerityInfo: verity, }, }, }