From 8e69c5855c574d830968f0f4837f0f5136b8f942 Mon Sep 17 00:00:00 2001 From: Kathryn Baldauf Date: Tue, 6 Jul 2021 18:08:38 -0700 Subject: [PATCH] export annotations for use in test suite Signed-off-by: Kathryn Baldauf --- internal/oci/annotations.go | 209 +++++++++++++++ internal/oci/uvm.go | 246 ++++-------------- internal/oci/uvm_test.go | 12 +- test/cri-containerd/clone_test.go | 11 +- .../container_layers_packing_test.go | 5 +- test/cri-containerd/container_test.go | 9 +- test/cri-containerd/container_update_test.go | 3 +- .../container_virtual_device_test.go | 25 +- test/cri-containerd/createcontainer_test.go | 17 +- test/cri-containerd/jobcontainer_test.go | 7 +- test/cri-containerd/pod_update_test.go | 7 +- test/cri-containerd/runpodsandbox_test.go | 97 +++---- .../scale_cpu_limits_to_sandbox_test.go | 3 +- test/cri-containerd/stats_test.go | 7 +- .../hcsshim/internal/oci/annotations.go | 147 +++++++++++ .../Microsoft/hcsshim/internal/oci/uvm.go | 246 ++++-------------- 16 files changed, 563 insertions(+), 488 deletions(-) create mode 100644 internal/oci/annotations.go create mode 100644 test/vendor/github.com/Microsoft/hcsshim/internal/oci/annotations.go diff --git a/internal/oci/annotations.go b/internal/oci/annotations.go new file mode 100644 index 0000000000..9ac11dcbcd --- /dev/null +++ b/internal/oci/annotations.go @@ -0,0 +1,209 @@ +package oci + +const ( + // AnnotationContainerMemorySizeInMB overrides the container memory size set + // via the OCI spec. + // + // Note: This annotation is in MB. OCI is in Bytes. When using this override + // the caller MUST use MB or sizing will be wrong. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Memory.Limit`. + AnnotationContainerMemorySizeInMB = "io.microsoft.container.memory.sizeinmb" + + // AnnotationContainerProcessorCount overrides the container processor count + // set via the OCI spec. + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Count`. + AnnotationContainerProcessorCount = "io.microsoft.container.processor.count" + + // AnnotationContainerProcessorLimit overrides the container processor limit + // set via the OCI spec. + // + // Limit allows values 1 - 10,000 where 10,000 means 100% CPU. (And is the + // default if omitted) + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.CPU.Maximum`. + AnnotationContainerProcessorLimit = "io.microsoft.container.processor.limit" + + // AnnotationContainerProcessorWeight overrides the container processor + // weight set via the OCI spec. + // + // Weight allows values 0 - 10,000. (100 is the default) + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Shares`. + AnnotationContainerProcessorWeight = "io.microsoft.container.processor.weight" + + // AnnotationContainerStorageQoSBandwidthMaximum overrides the container + // storage bandwidth per second set via the OCI spec. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Storage.Bps`. + AnnotationContainerStorageQoSBandwidthMaximum = "io.microsoft.container.storage.qos.bandwidthmaximum" + + // AnnotationContainerStorageQoSIopsMaximum overrides the container storage + // maximum iops set via the OCI spec. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Storage.Iops`. + AnnotationContainerStorageQoSIopsMaximum = "io.microsoft.container.storage.qos.iopsmaximum" + + // AnnotationGPUVHDPath overrides the default path to search for the gpu vhd + AnnotationGPUVHDPath = "io.microsoft.lcow.gpuvhdpath" + + // AnnotationAssignedDeviceKernelDrivers indicates what drivers to install in the pod during device + // assignment. This value should contain a list of comma separated directories containing all + // files and information needed to install given driver(s). This may include .sys, + // .inf, .cer, and/or other files used during standard installation with pnputil. + AnnotationAssignedDeviceKernelDrivers = "io.microsoft.assigneddevice.kerneldrivers" + + // AnnotationHostProcessInheritUser indicates whether to ignore the username passed in to run a host process + // container as and instead inherit the user token from the executable that is launching the container process. + AnnotationHostProcessInheritUser = "microsoft.com/hostprocess-inherit-user" + + // AnnotationHostProcessContainer indicates to launch a host process container (job container in this repository). + AnnotationHostProcessContainer = "microsoft.com/hostprocess-container" + + // AnnotationAllowOvercommit indicates if we should allow over commit memory for UVM. + // Defaults to true. For physical backed memory, set to false. + AnnotationAllowOvercommit = "io.microsoft.virtualmachine.computetopology.memory.allowovercommit" + + // AnnotationEnableDeferredCommit indicates if we should allow deferred memory commit for UVM. + // Defaults to false. For virtual memory with deferred commit, set to true. + AnnotationEnableDeferredCommit = "io.microsoft.virtualmachine.computetopology.memory.enabledeferredcommit" + + // AnnotationEnableColdDiscardHint indicates whether to enable cold discard hint, which allows the UVM + // to trim non-zeroed pages from the working set (if supported by the guest operating system). + AnnotationEnableColdDiscardHint = "io.microsoft.virtualmachine.computetopology.memory.enablecolddiscardhint" + + // AnnotationMemorySizeInMB overrides the container memory size set via the + // OCI spec. + // + // Note: This annotation is in MB. OCI is in Bytes. When using this override + // the caller MUST use MB or sizing will be wrong. + AnnotationMemorySizeInMB = "io.microsoft.virtualmachine.computetopology.memory.sizeinmb" + + // AnnotationMemoryLowMMIOGapInMB indicates the low MMIO gap in MB + AnnotationMemoryLowMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.lowmmiogapinmb" + + // AnnotationMemoryHighMMIOBaseInMB indicates the high MMIO base in MB + AnnotationMemoryHighMMIOBaseInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiobaseinmb" + + // AnnotationMemoryHighMMIOBaseInMB indicates the high MMIO gap in MB + AnnotationMemoryHighMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb" + + // annotationProcessorCount overrides the hypervisor isolated vCPU count set + // via the OCI spec. + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + AnnotationProcessorCount = "io.microsoft.virtualmachine.computetopology.processor.count" + + // annotationProcessorLimit overrides the hypervisor isolated vCPU limit set + // via the OCI spec. + // + // Limit allows values 1 - 100,000 where 100,000 means 100% CPU. (And is the + // default if omitted) + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + AnnotationProcessorLimit = "io.microsoft.virtualmachine.computetopology.processor.limit" + + // AnnotationProcessorWeight overrides the hypervisor isolated vCPU weight set + // via the OCI spec. + // + // Weight allows values 0 - 10,000. (100 is the default if omitted) + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + AnnotationProcessorWeight = "io.microsoft.virtualmachine.computetopology.processor.weight" + + // AnnotationVPMemCount indicates the max number of vpmem devices that can be used on the UVM + AnnotationVPMemCount = "io.microsoft.virtualmachine.devices.virtualpmem.maximumcount" + + // AnnotationVPMemSize indicates the size of the VPMem devices. + AnnotationVPMemSize = "io.microsoft.virtualmachine.devices.virtualpmem.maximumsizebytes" + + // AnnotationPreferredRootFSType indicates what the preferred rootfs type should be for an LCOW UVM. + // valid values are "initrd" or "vhd" + AnnotationPreferredRootFSType = "io.microsoft.virtualmachine.lcow.preferredrootfstype" + + // AnnotationBootFilesRootPath indicates the path to find the LCOW boot files to use when creating the UVM + AnnotationBootFilesRootPath = "io.microsoft.virtualmachine.lcow.bootfilesrootpath" + + // AnnotationKernelDirectBoot indicates that we should skip UEFI and boot directly to `kernel` + AnnotationKernelDirectBoot = "io.microsoft.virtualmachine.lcow.kerneldirectboot" + + // AnnotationVPCIEnabled indicates that pci support should be enabled for the LCOW UVM + AnnotationVPCIEnabled = "io.microsoft.virtualmachine.lcow.vpcienabled" + + // AnnotationVPMemNoMultiMapping indicates that we should disable LCOW vpmem layer multi mapping + AnnotationVPMemNoMultiMapping = "io.microsoft.virtualmachine.lcow.vpmem.nomultimapping" + + // AnnotationStorageQoSBandwidthMaximum indicates the maximum number of bytes per second. If `0` + // will default to the platform default. + AnnotationStorageQoSBandwidthMaximum = "io.microsoft.virtualmachine.storageqos.bandwidthmaximum" + + // AnnotationStorageQoSIopsMaximum indicates the maximum number of Iops. If `0` will + // default to the platform default. + AnnotationStorageQoSIopsMaximum = "io.microsoft.virtualmachine.storageqos.iopsmaximum" + + // AnnotationFullyPhysicallyBacked indicates that the UVM should use physically backed memory only, + // including for additional devices added later. + AnnotationFullyPhysicallyBacked = "io.microsoft.virtualmachine.fullyphysicallybacked" + + // AnnotationDisableCompartmentNamespace sets whether to disable namespacing the network compartment in the UVM + // for WCOW. + AnnotationDisableCompartmentNamespace = "io.microsoft.virtualmachine.disablecompartmentnamespace" + + // AnnotationVSMBNoDirectMap specifies that no direct mapping should be used for any VSMBs added to the UVM + AnnotationVSMBNoDirectMap = "io.microsoft.virtualmachine.wcow.virtualSMB.nodirectmap" + + // AnnotationCPUGroupID specifies the cpugroup ID that a UVM should be assigned to if any + AnnotationCPUGroupID = "io.microsoft.virtualmachine.cpugroup.id" + + // AnnotationSaveAsTemplate annotation must be used with a pod & container creation request. + // If this annotation is present in the request then it will save the UVM (pod) + // and the container(s) inside it as a template. However, this also means that this + // pod and the containers inside this pod will permananetly stay in the + // paused/templated state and can not be resumed again. + AnnotationSaveAsTemplate = "io.microsoft.virtualmachine.saveastemplate" + + // AnnotationTemplateID should be used when creating a pod or a container from a template. + // When creating a pod from a template use the ID of the templated pod as the + // TemplateID and when creating a container use the ID of the templated container as + // the TemplateID. It is the client's responsibility to make sure that the sandbox + // within which a cloned container needs to be created must also be created from the + // same template. + AnnotationTemplateID = "io.microsoft.virtualmachine.templateid" + + // AnnotationNetworkConfigProxy holds the address of the network config proxy service. + // If set, network setup will be attempted via ncproxy. + AnnotationNetworkConfigProxy = "io.microsoft.network.ncproxy" + + // AnnotationNcproxyContainerID indicates whether or not to use the hcsshim container ID + // when setting up ncproxy and computeagent + AnnotationNcproxyContainerID = "io.microsoft.network.ncproxy.containerid" +) diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index e821837ed0..5b8c5e75c1 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -16,152 +16,6 @@ import ( "github.com/sirupsen/logrus" ) -const ( - // AnnotationContainerMemorySizeInMB overrides the container memory size set - // via the OCI spec. - // - // Note: This annotation is in MB. OCI is in Bytes. When using this override - // the caller MUST use MB or sizing will be wrong. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Memory.Limit`. - AnnotationContainerMemorySizeInMB = "io.microsoft.container.memory.sizeinmb" - // AnnotationContainerProcessorCount overrides the container processor count - // set via the OCI spec. - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Count`. - AnnotationContainerProcessorCount = "io.microsoft.container.processor.count" - // AnnotationContainerProcessorLimit overrides the container processor limit - // set via the OCI spec. - // - // Limit allows values 1 - 10,000 where 10,000 means 100% CPU. (And is the - // default if omitted) - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.CPU.Maximum`. - AnnotationContainerProcessorLimit = "io.microsoft.container.processor.limit" - // AnnotationContainerProcessorWeight overrides the container processor - // weight set via the OCI spec. - // - // Weight allows values 0 - 10,000. (100 is the default) - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Shares`. - AnnotationContainerProcessorWeight = "io.microsoft.container.processor.weight" - // AnnotationContainerStorageQoSBandwidthMaximum overrides the container - // storage bandwidth per second set via the OCI spec. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Storage.Bps`. - AnnotationContainerStorageQoSBandwidthMaximum = "io.microsoft.container.storage.qos.bandwidthmaximum" - // AnnotationContainerStorageQoSIopsMaximum overrides the container storage - // maximum iops set via the OCI spec. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Storage.Iops`. - AnnotationContainerStorageQoSIopsMaximum = "io.microsoft.container.storage.qos.iopsmaximum" - // AnnotationGPUVHDPath overrides the default path to search for the gpu vhd - AnnotationGPUVHDPath = "io.microsoft.lcow.gpuvhdpath" - // AnnotationAssignedDeviceKernelDrivers indicates what drivers to install in the pod during device - // assignment. This value should contain a list of comma separated directories containing all - // files and information needed to install given driver(s). This may include .sys, - // .inf, .cer, and/or other files used during standard installation with pnputil. - AnnotationAssignedDeviceKernelDrivers = "io.microsoft.assigneddevice.kerneldrivers" - // AnnotationHostProcessInheritUser indicates whether to ignore the username passed in to run a host process - // container as and instead inherit the user token from the executable that is launching the container process. - AnnotationHostProcessInheritUser = "microsoft.com/hostprocess-inherit-user" - // AnnotationHostProcessContainer indicates to launch a host process container (job container in this repository). - AnnotationHostProcessContainer = "microsoft.com/hostprocess-container" - - annotationAllowOvercommit = "io.microsoft.virtualmachine.computetopology.memory.allowovercommit" - annotationEnableDeferredCommit = "io.microsoft.virtualmachine.computetopology.memory.enabledeferredcommit" - annotationEnableColdDiscardHint = "io.microsoft.virtualmachine.computetopology.memory.enablecolddiscardhint" - // annotationMemorySizeInMB overrides the container memory size set via the - // OCI spec. - // - // Note: This annotation is in MB. OCI is in Bytes. When using this override - // the caller MUST use MB or sizing will be wrong. - annotationMemorySizeInMB = "io.microsoft.virtualmachine.computetopology.memory.sizeinmb" - annotationMemoryLowMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.lowmmiogapinmb" - annotationMemoryHighMMIOBaseInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiobaseinmb" - annotationMemoryHighMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb" - // annotationProcessorCount overrides the hypervisor isolated vCPU count set - // via the OCI spec. - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - annotationProcessorCount = "io.microsoft.virtualmachine.computetopology.processor.count" - // annotationProcessorLimit overrides the hypervisor isolated vCPU limit set - // via the OCI spec. - // - // Limit allows values 1 - 100,000 where 100,000 means 100% CPU. (And is the - // default if omitted) - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - annotationProcessorLimit = "io.microsoft.virtualmachine.computetopology.processor.limit" - // annotationProcessorWeight overrides the hypervisor isolated vCPU weight set - // via the OCI spec. - // - // Weight allows values 0 - 10,000. (100 is the default if omitted) - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - annotationProcessorWeight = "io.microsoft.virtualmachine.computetopology.processor.weight" - annotationVPMemCount = "io.microsoft.virtualmachine.devices.virtualpmem.maximumcount" - annotationVPMemSize = "io.microsoft.virtualmachine.devices.virtualpmem.maximumsizebytes" - annotationPreferredRootFSType = "io.microsoft.virtualmachine.lcow.preferredrootfstype" - annotationBootFilesRootPath = "io.microsoft.virtualmachine.lcow.bootfilesrootpath" - annotationKernelDirectBoot = "io.microsoft.virtualmachine.lcow.kerneldirectboot" - annotationVPCIEnabled = "io.microsoft.virtualmachine.lcow.vpcienabled" - annotationVPMemNoMultiMapping = "io.microsoft.virtualmachine.lcow.vpmem.nomultimapping" - annotationStorageQoSBandwidthMaximum = "io.microsoft.virtualmachine.storageqos.bandwidthmaximum" - annotationStorageQoSIopsMaximum = "io.microsoft.virtualmachine.storageqos.iopsmaximum" - annotationFullyPhysicallyBacked = "io.microsoft.virtualmachine.fullyphysicallybacked" - annotationDisableCompartmentNamespace = "io.microsoft.virtualmachine.disablecompartmentnamespace" - annotationVSMBNoDirectMap = "io.microsoft.virtualmachine.wcow.virtualSMB.nodirectmap" - - // annotation used to specify the cpugroup ID that a UVM should be assigned to - annotationCPUGroupID = "io.microsoft.virtualmachine.cpugroup.id" - - // SaveAsTemplate annotation must be used with a pod & container creation request. - // If this annotation is present in the request then it will save the UVM (pod) - // and the container(s) inside it as a template. However, this also means that this - // pod and the containers inside this pod will permananetly stay in the - // paused/templated state and can not be resumed again. - annotationSaveAsTemplate = "io.microsoft.virtualmachine.saveastemplate" - - // This annotation should be used when creating a pod or a container from a template. - // When creating a pod from a template use the ID of the templated pod as the - // TemplateID and when creating a container use the ID of the templated container as - // the TemplateID. It is the client's responsibility to make sure that the sandbox - // within which a cloned container needs to be created must also be created from the - // same template. - annotationTemplateID = "io.microsoft.virtualmachine.templateid" - annotationNetworkConfigProxy = "io.microsoft.network.ncproxy" - AnnotationNcproxyContainerID = "io.microsoft.network.ncproxy.containerid" -) - // parseAnnotationsBool searches `a` for `key` and if found verifies that the // value is `true` or `false` in any case. If `key` is not found returns `def`. func parseAnnotationsBool(ctx context.Context, a map[string]string, key string, def bool) bool { @@ -354,13 +208,13 @@ func parseAnnotationsString(a map[string]string, key string, def string) string // if this create request should be considered as a template creation request. If value // is found the returns the actual value, returns false otherwise. func ParseAnnotationsSaveAsTemplate(ctx context.Context, s *specs.Spec) bool { - return parseAnnotationsBool(ctx, s.Annotations, annotationSaveAsTemplate, false) + return parseAnnotationsBool(ctx, s.Annotations, AnnotationSaveAsTemplate, false) } // ParseAnnotationsTemplateID searches for the templateID in the create request. If the // value is found then returns the value otherwise returns the empty string. func ParseAnnotationsTemplateID(ctx context.Context, s *specs.Spec) string { - return parseAnnotationsString(s.Annotations, annotationTemplateID, "") + return parseAnnotationsString(s.Annotations, AnnotationTemplateID, "") } func ParseCloneAnnotations(ctx context.Context, s *specs.Spec) (isTemplate bool, templateID string, err error) { @@ -379,7 +233,7 @@ func ParseCloneAnnotations(ctx context.Context, s *specs.Spec) (isTemplate bool, // handleAnnotationKernelDirectBoot handles parsing annotationKernelDirectBoot and setting // implied annotations from the result. func handleAnnotationKernelDirectBoot(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { - lopts.KernelDirect = parseAnnotationsBool(ctx, a, annotationKernelDirectBoot, lopts.KernelDirect) + lopts.KernelDirect = parseAnnotationsBool(ctx, a, AnnotationKernelDirectBoot, lopts.KernelDirect) if !lopts.KernelDirect { lopts.KernelFile = uvm.KernelFile } @@ -388,7 +242,7 @@ func handleAnnotationKernelDirectBoot(ctx context.Context, a map[string]string, // handleAnnotationPreferredRootFSType handles parsing annotationPreferredRootFSType and setting // implied annotations from the result func handleAnnotationPreferredRootFSType(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { - lopts.PreferredRootFSType = parseAnnotationsPreferredRootFSType(ctx, a, annotationPreferredRootFSType, lopts.PreferredRootFSType) + lopts.PreferredRootFSType = parseAnnotationsPreferredRootFSType(ctx, a, AnnotationPreferredRootFSType, lopts.PreferredRootFSType) switch lopts.PreferredRootFSType { case uvm.PreferredRootFSTypeInitRd: lopts.RootFSFile = uvm.InitrdFile @@ -402,7 +256,7 @@ func handleAnnotationPreferredRootFSType(ctx context.Context, a map[string]strin func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]string, opts interface{}) { switch options := opts.(type) { case *uvm.OptionsLCOW: - options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, annotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) + options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, AnnotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) if options.FullyPhysicallyBacked { options.AllowOvercommit = false options.PreferredRootFSType = uvm.PreferredRootFSTypeInitRd @@ -410,7 +264,7 @@ func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]str options.VPMemDeviceCount = 0 } case *uvm.OptionsWCOW: - options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, annotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) + options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, AnnotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) if options.FullyPhysicallyBacked { options.AllowOvercommit = false } @@ -420,8 +274,8 @@ func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]str // handleCloneAnnotations handles parsing annotations related to template creation and cloning // Since late cloning is only supported for WCOW this function only deals with WCOW options. func handleCloneAnnotations(ctx context.Context, a map[string]string, wopts *uvm.OptionsWCOW) (err error) { - wopts.IsTemplate = parseAnnotationsBool(ctx, a, annotationSaveAsTemplate, false) - templateID := parseAnnotationsString(a, annotationTemplateID, "") + wopts.IsTemplate = parseAnnotationsBool(ctx, a, AnnotationSaveAsTemplate, false) + templateID := parseAnnotationsString(a, AnnotationTemplateID, "") if templateID != "" { tc, err := clone.FetchTemplateConfig(ctx, templateID) if err != nil { @@ -445,25 +299,25 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( } if IsLCOW(s) { lopts := uvm.NewDefaultOptionsLCOW(id, owner) - lopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, annotationMemorySizeInMB, lopts.MemorySizeInMB) - lopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotationMemoryLowMMIOGapInMB, lopts.LowMMIOGapInMB) - lopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, annotationMemoryHighMMIOBaseInMB, lopts.HighMMIOBaseInMB) - lopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotationMemoryHighMMIOGapInMB, lopts.HighMMIOGapInMB) - lopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, annotationAllowOvercommit, lopts.AllowOvercommit) - lopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, annotationEnableDeferredCommit, lopts.EnableDeferredCommit) - lopts.EnableColdDiscardHint = parseAnnotationsBool(ctx, s.Annotations, annotationEnableColdDiscardHint, lopts.EnableColdDiscardHint) - lopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, annotationProcessorCount, lopts.ProcessorCount) - lopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, annotationProcessorLimit, lopts.ProcessorLimit) - lopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, annotationProcessorWeight, lopts.ProcessorWeight) - lopts.VPMemDeviceCount = parseAnnotationsUint32(ctx, s.Annotations, annotationVPMemCount, lopts.VPMemDeviceCount) - lopts.VPMemSizeBytes = parseAnnotationsUint64(ctx, s.Annotations, annotationVPMemSize, lopts.VPMemSizeBytes) - lopts.VPMemNoMultiMapping = parseAnnotationsBool(ctx, s.Annotations, annotationVPMemNoMultiMapping, lopts.VPMemNoMultiMapping) - lopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, annotationStorageQoSBandwidthMaximum, lopts.StorageQoSBandwidthMaximum) - lopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotationStorageQoSIopsMaximum, lopts.StorageQoSIopsMaximum) - lopts.VPCIEnabled = parseAnnotationsBool(ctx, s.Annotations, annotationVPCIEnabled, lopts.VPCIEnabled) - lopts.BootFilesPath = parseAnnotationsString(s.Annotations, annotationBootFilesRootPath, lopts.BootFilesPath) - lopts.CPUGroupID = parseAnnotationsString(s.Annotations, annotationCPUGroupID, lopts.CPUGroupID) - lopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, annotationNetworkConfigProxy, lopts.NetworkConfigProxy) + lopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, AnnotationMemorySizeInMB, lopts.MemorySizeInMB) + lopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryLowMMIOGapInMB, lopts.LowMMIOGapInMB) + lopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOBaseInMB, lopts.HighMMIOBaseInMB) + lopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOGapInMB, lopts.HighMMIOGapInMB) + lopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationAllowOvercommit, lopts.AllowOvercommit) + lopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableDeferredCommit, lopts.EnableDeferredCommit) + lopts.EnableColdDiscardHint = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableColdDiscardHint, lopts.EnableColdDiscardHint) + lopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, AnnotationProcessorCount, lopts.ProcessorCount) + lopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, AnnotationProcessorLimit, lopts.ProcessorLimit) + lopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, AnnotationProcessorWeight, lopts.ProcessorWeight) + lopts.VPMemDeviceCount = parseAnnotationsUint32(ctx, s.Annotations, AnnotationVPMemCount, lopts.VPMemDeviceCount) + lopts.VPMemSizeBytes = parseAnnotationsUint64(ctx, s.Annotations, AnnotationVPMemSize, lopts.VPMemSizeBytes) + lopts.VPMemNoMultiMapping = parseAnnotationsBool(ctx, s.Annotations, AnnotationVPMemNoMultiMapping, lopts.VPMemNoMultiMapping) + lopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, AnnotationStorageQoSBandwidthMaximum, lopts.StorageQoSBandwidthMaximum) + lopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, AnnotationStorageQoSIopsMaximum, lopts.StorageQoSIopsMaximum) + lopts.VPCIEnabled = parseAnnotationsBool(ctx, s.Annotations, AnnotationVPCIEnabled, lopts.VPCIEnabled) + lopts.BootFilesPath = parseAnnotationsString(s.Annotations, AnnotationBootFilesRootPath, lopts.BootFilesPath) + lopts.CPUGroupID = parseAnnotationsString(s.Annotations, AnnotationCPUGroupID, lopts.CPUGroupID) + lopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, AnnotationNetworkConfigProxy, lopts.NetworkConfigProxy) handleAnnotationPreferredRootFSType(ctx, s.Annotations, lopts) handleAnnotationKernelDirectBoot(ctx, s.Annotations, lopts) @@ -473,21 +327,21 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( return lopts, nil } else if IsWCOW(s) { wopts := uvm.NewDefaultOptionsWCOW(id, owner) - wopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, annotationMemorySizeInMB, wopts.MemorySizeInMB) - wopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotationMemoryLowMMIOGapInMB, wopts.LowMMIOGapInMB) - wopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, annotationMemoryHighMMIOBaseInMB, wopts.HighMMIOBaseInMB) - wopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotationMemoryHighMMIOGapInMB, wopts.HighMMIOGapInMB) - wopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, annotationAllowOvercommit, wopts.AllowOvercommit) - wopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, annotationEnableDeferredCommit, wopts.EnableDeferredCommit) - wopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, annotationProcessorCount, wopts.ProcessorCount) - wopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, annotationProcessorLimit, wopts.ProcessorLimit) - wopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, annotationProcessorWeight, wopts.ProcessorWeight) - wopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, annotationStorageQoSBandwidthMaximum, wopts.StorageQoSBandwidthMaximum) - wopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotationStorageQoSIopsMaximum, wopts.StorageQoSIopsMaximum) - wopts.DisableCompartmentNamespace = parseAnnotationsBool(ctx, s.Annotations, annotationDisableCompartmentNamespace, wopts.DisableCompartmentNamespace) - wopts.CPUGroupID = parseAnnotationsString(s.Annotations, annotationCPUGroupID, wopts.CPUGroupID) - wopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, annotationNetworkConfigProxy, wopts.NetworkConfigProxy) - wopts.NoDirectMap = parseAnnotationsBool(ctx, s.Annotations, annotationVSMBNoDirectMap, wopts.NoDirectMap) + wopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, AnnotationMemorySizeInMB, wopts.MemorySizeInMB) + wopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryLowMMIOGapInMB, wopts.LowMMIOGapInMB) + wopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOBaseInMB, wopts.HighMMIOBaseInMB) + wopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOGapInMB, wopts.HighMMIOGapInMB) + wopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationAllowOvercommit, wopts.AllowOvercommit) + wopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableDeferredCommit, wopts.EnableDeferredCommit) + wopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, AnnotationProcessorCount, wopts.ProcessorCount) + wopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, AnnotationProcessorLimit, wopts.ProcessorLimit) + wopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, AnnotationProcessorWeight, wopts.ProcessorWeight) + wopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, AnnotationStorageQoSBandwidthMaximum, wopts.StorageQoSBandwidthMaximum) + wopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, AnnotationStorageQoSIopsMaximum, wopts.StorageQoSIopsMaximum) + wopts.DisableCompartmentNamespace = parseAnnotationsBool(ctx, s.Annotations, AnnotationDisableCompartmentNamespace, wopts.DisableCompartmentNamespace) + wopts.CPUGroupID = parseAnnotationsString(s.Annotations, AnnotationCPUGroupID, wopts.CPUGroupID) + wopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, AnnotationNetworkConfigProxy, wopts.NetworkConfigProxy) + wopts.NoDirectMap = parseAnnotationsBool(ctx, s.Annotations, AnnotationVSMBNoDirectMap, wopts.NoDirectMap) handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, wopts) if err := handleCloneAnnotations(ctx, s.Annotations, wopts); err != nil { return nil, err @@ -504,24 +358,24 @@ func UpdateSpecFromOptions(s specs.Spec, opts *runhcsopts.Options) specs.Spec { return s } - if _, ok := s.Annotations[annotationBootFilesRootPath]; !ok && opts.BootFilesRootPath != "" { - s.Annotations[annotationBootFilesRootPath] = opts.BootFilesRootPath + if _, ok := s.Annotations[AnnotationBootFilesRootPath]; !ok && opts.BootFilesRootPath != "" { + s.Annotations[AnnotationBootFilesRootPath] = opts.BootFilesRootPath } - if _, ok := s.Annotations[annotationProcessorCount]; !ok && opts.VmProcessorCount != 0 { - s.Annotations[annotationProcessorCount] = strconv.FormatInt(int64(opts.VmProcessorCount), 10) + if _, ok := s.Annotations[AnnotationProcessorCount]; !ok && opts.VmProcessorCount != 0 { + s.Annotations[AnnotationProcessorCount] = strconv.FormatInt(int64(opts.VmProcessorCount), 10) } - if _, ok := s.Annotations[annotationMemorySizeInMB]; !ok && opts.VmMemorySizeInMb != 0 { - s.Annotations[annotationMemorySizeInMB] = strconv.FormatInt(int64(opts.VmMemorySizeInMb), 10) + if _, ok := s.Annotations[AnnotationMemorySizeInMB]; !ok && opts.VmMemorySizeInMb != 0 { + s.Annotations[AnnotationMemorySizeInMB] = strconv.FormatInt(int64(opts.VmMemorySizeInMb), 10) } if _, ok := s.Annotations[AnnotationGPUVHDPath]; !ok && opts.GPUVHDPath != "" { s.Annotations[AnnotationGPUVHDPath] = opts.GPUVHDPath } - if _, ok := s.Annotations[annotationNetworkConfigProxy]; !ok && opts.NCProxyAddr != "" { - s.Annotations[annotationNetworkConfigProxy] = opts.NCProxyAddr + if _, ok := s.Annotations[AnnotationNetworkConfigProxy]; !ok && opts.NCProxyAddr != "" { + s.Annotations[AnnotationNetworkConfigProxy] = opts.NCProxyAddr } return s diff --git a/internal/oci/uvm_test.go b/internal/oci/uvm_test.go index 14b2e75277..14b1691951 100644 --- a/internal/oci/uvm_test.go +++ b/internal/oci/uvm_test.go @@ -15,12 +15,12 @@ func Test_SpecUpdate_MemorySize_WithAnnotation_WithOpts(t *testing.T) { s := &specs.Spec{ Linux: &specs.Linux{}, Annotations: map[string]string{ - annotationMemorySizeInMB: "2048", + AnnotationMemorySizeInMB: "2048", }, } updatedSpec := UpdateSpecFromOptions(*s, opts) - if updatedSpec.Annotations[annotationMemorySizeInMB] != "2048" { + if updatedSpec.Annotations[AnnotationMemorySizeInMB] != "2048" { t.Fatal("should not have updated annotation to default when annotation is provided in the spec") } } @@ -36,7 +36,7 @@ func Test_SpecUpdate_MemorySize_NoAnnotation_WithOpts(t *testing.T) { } updatedSpec := UpdateSpecFromOptions(*s, opts) - if updatedSpec.Annotations[annotationMemorySizeInMB] != "3072" { + if updatedSpec.Annotations[AnnotationMemorySizeInMB] != "3072" { t.Fatal("should have updated annotation to default when annotation is not provided in the spec") } } @@ -49,12 +49,12 @@ func Test_SpecUpdate_ProcessorCount_WithAnnotation_WithOpts(t *testing.T) { s := &specs.Spec{ Linux: &specs.Linux{}, Annotations: map[string]string{ - annotationProcessorCount: "8", + AnnotationProcessorCount: "8", }, } updatedSpec := UpdateSpecFromOptions(*s, opts) - if updatedSpec.Annotations[annotationProcessorCount] != "8" { + if updatedSpec.Annotations[AnnotationProcessorCount] != "8" { t.Fatal("should not have updated annotation to default when annotation is provided in the spec") } } @@ -70,7 +70,7 @@ func Test_SpecUpdate_ProcessorCount_NoAnnotation_WithOpts(t *testing.T) { } updatedSpec := UpdateSpecFromOptions(*s, opts) - if updatedSpec.Annotations[annotationProcessorCount] != "4" { + if updatedSpec.Annotations[AnnotationProcessorCount] != "4" { t.Fatal("should have updated annotation to default when annotation is not provided in the spec") } } diff --git a/test/cri-containerd/clone_test.go b/test/cri-containerd/clone_test.go index fb9fef29ac..a816e29352 100644 --- a/test/cri-containerd/clone_test.go +++ b/test/cri-containerd/clone_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/osversion" testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" @@ -26,7 +27,7 @@ func getTemplatePodConfig(name string) *runtime.RunPodSandboxRequest { Namespace: testNamespace, }, Annotations: map[string]string{ - "io.microsoft.virtualmachine.saveastemplate": "true", + oci.AnnotationSaveAsTemplate: "true", }, }, RuntimeHandler: wcowHypervisorRuntimeHandler, @@ -49,7 +50,7 @@ func getTemplateContainerConfig(name string) *runtime.CreateContainerRequest { "127.0.0.1", }, Annotations: map[string]string{ - "io.microsoft.virtualmachine.saveastemplate": "true", + oci.AnnotationSaveAsTemplate: "true", }, }, } @@ -84,7 +85,7 @@ func getClonedPodConfig(uniqueID int, templateid string) *runtime.RunPodSandboxR Namespace: testNamespace, }, Annotations: map[string]string{ - "io.microsoft.virtualmachine.templateid": templateid + "@vm", + oci.AnnotationTemplateID: templateid + "@vm", }, }, RuntimeHandler: wcowHypervisorRuntimeHandler, @@ -108,7 +109,7 @@ func getClonedContainerConfig(uniqueID int, templateid string) *runtime.CreateCo "127.0.0.1", }, Annotations: map[string]string{ - "io.microsoft.virtualmachine.templateid": templateid, + oci.AnnotationTemplateID: templateid, }, }, } @@ -487,7 +488,7 @@ func Test_VerifyCloneAndTemplateConfig(t *testing.T) { // change pod config to make sure the request fails cloneSandboxRequest := getClonedPodConfig(0, templatePodID) - cloneSandboxRequest.Config.Annotations["io.microsoft.virtualmachine.computetopology.memory.allowovercommit"] = "false" + cloneSandboxRequest.Config.Annotations[oci.AnnotationAllowOvercommit] = "false" _, err := client.RunPodSandbox(ctx, cloneSandboxRequest) if err == nil { diff --git a/test/cri-containerd/container_layers_packing_test.go b/test/cri-containerd/container_layers_packing_test.go index 4131e454bb..bf1178f8a4 100644 --- a/test/cri-containerd/container_layers_packing_test.go +++ b/test/cri-containerd/container_layers_packing_test.go @@ -10,6 +10,7 @@ import ( "strings" "testing" + "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/shimdiag" "github.com/Microsoft/hcsshim/osversion" testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" @@ -101,7 +102,7 @@ func Test_Container_Layer_Packing_On_VPMem(t *testing.T) { } { t.Run(fmt.Sprintf("PreferredRootFSType-%s", scenario.rootfsType), func(t *testing.T) { annotations := map[string]string{ - "io.microsoft.virtualmachine.lcow.preferredrootfstype": scenario.rootfsType, + oci.AnnotationPreferredRootFSType: scenario.rootfsType, } podReq := getRunPodSandboxRequest(t, lcowRuntimeHandler, annotations) podID := runPodSandbox(t, client, ctx, podReq) @@ -169,7 +170,7 @@ func Test_Annotation_Disable_Multi_Mapping(t *testing.T) { pullRequiredLcowImages(t, []string{imageLcowK8sPause, alpine70ExtraLayers}) annotations := map[string]string{ - "io.microsoft.virtualmachine.lcow.vpmem.nomultimapping": "true", + oci.AnnotationVPMemNoMultiMapping: "true", } podReq := getRunPodSandboxRequest(t, lcowRuntimeHandler, annotations) podID := runPodSandbox(t, client, ctx, podReq) diff --git a/test/cri-containerd/container_test.go b/test/cri-containerd/container_test.go index f6b9014ea2..bf8fa6fe8f 100644 --- a/test/cri-containerd/container_test.go +++ b/test/cri-containerd/container_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "github.com/Microsoft/hcsshim/internal/oci" "github.com/sirupsen/logrus" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -267,8 +268,8 @@ func Test_RunContainer_ZeroVPMEM_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.lcow.preferredrootfstype": "initrd", - "io.microsoft.virtualmachine.devices.virtualpmem.maximumcount": "0", + oci.AnnotationPreferredRootFSType: "initrd", + oci.AnnotationVPMemCount: "0", }, ) @@ -309,8 +310,8 @@ func Test_RunContainer_ZeroVPMEM_Multiple_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.lcow.preferredrootfstype": "initrd", - "io.microsoft.virtualmachine.devices.virtualpmem.maximumcount": "0", + oci.AnnotationPreferredRootFSType: "initrd", + oci.AnnotationVPMemCount: "0", }, ) diff --git a/test/cri-containerd/container_update_test.go b/test/cri-containerd/container_update_test.go index 94ecbfd1c8..4e60ec4fa6 100644 --- a/test/cri-containerd/container_update_test.go +++ b/test/cri-containerd/container_update_test.go @@ -7,6 +7,7 @@ import ( "fmt" "testing" + "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/osversion" testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" @@ -335,7 +336,7 @@ func Test_Container_UpdateResources_Memory(t *testing.T) { }, Command: test.cmd, Annotations: map[string]string{ - "io.microsoft.container.memory.sizeinmb": fmt.Sprintf("%d", startingMemorySize), // 768MB + oci.AnnotationContainerMemorySizeInMB: fmt.Sprintf("%d", startingMemorySize), // 768MB }, }, PodSandboxId: podID, diff --git a/test/cri-containerd/container_virtual_device_test.go b/test/cri-containerd/container_virtual_device_test.go index c83eb448a4..cc7697ee86 100644 --- a/test/cri-containerd/container_virtual_device_test.go +++ b/test/cri-containerd/container_virtual_device_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/osversion" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -115,16 +116,16 @@ func findTestVirtualDeviceID() (string, error) { } var lcowPodGPUAnnotations = map[string]string{ - "io.microsoft.virtualmachine.lcow.kerneldirectboot": "false", - "io.microsoft.virtualmachine.computetopology.memory.allowovercommit": "false", - "io.microsoft.virtualmachine.lcow.preferredrootfstype": "initrd", - "io.microsoft.virtualmachine.devices.virtualpmem.maximumcount": "0", - "io.microsoft.virtualmachine.lcow.vpcienabled": "true", + oci.AnnotationKernelDirectBoot: "false", + oci.AnnotationAllowOvercommit: "false", + oci.AnnotationPreferredRootFSType: "initrd", + oci.AnnotationVPMemCount: "0", + oci.AnnotationVPCIEnabled: "true", // we believe this is a sufficiently large high MMIO space amount for this test. // if a given gpu device needs more, this test will fail to create the container // and may hang. - "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb": "64000", - "io.microsoft.virtualmachine.lcow.bootfilesrootpath": testGPUBootFiles, + oci.AnnotationMemoryHighMMIOGapInMB: "64000", + oci.AnnotationBootFilesRootPath: testGPUBootFiles, } func getGPUContainerRequestLCOW(t *testing.T, podID string, podConfig *runtime.PodSandboxConfig, device *runtime.Device) *runtime.CreateContainerRequest { @@ -178,7 +179,7 @@ func getGPUContainerRequestWCOW(t *testing.T, podID string, podConfig *runtime.P }, }, Annotations: map[string]string{ - "io.microsoft.assigneddevice.kerneldrivers": testDriversPath, + oci.AnnotationAssignedDeviceKernelDrivers: testDriversPath, }, }, PodSandboxId: podID, @@ -511,7 +512,7 @@ func Test_RunContainer_VirtualDevice_GPU_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.fullyphysicallybacked": "true", + oci.AnnotationFullyPhysicallyBacked: "true", }, ) @@ -559,7 +560,7 @@ func Test_RunContainer_VirtualDevice_GPU_and_NoGPU_WCOW_Hypervisor(t *testing.T) t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.fullyphysicallybacked": "true", + oci.AnnotationFullyPhysicallyBacked: "true", }, ) @@ -624,7 +625,7 @@ func Test_RunContainer_VirtualDevice_GPU_Multiple_WCOW_Hypervisor(t *testing.T) t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.fullyphysicallybacked": "true", + oci.AnnotationFullyPhysicallyBacked: "true", }, ) @@ -679,7 +680,7 @@ func Test_RunContainer_VirtualDevice_GPU_Multiple_Removal_WCOW_Hypervisor(t *tes t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.fullyphysicallybacked": "true", + oci.AnnotationFullyPhysicallyBacked: "true", }, ) diff --git a/test/cri-containerd/createcontainer_test.go b/test/cri-containerd/createcontainer_test.go index b420253d47..5467b08b95 100644 --- a/test/cri-containerd/createcontainer_test.go +++ b/test/cri-containerd/createcontainer_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/Microsoft/go-winio" + "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/osversion" testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" @@ -304,7 +305,7 @@ func Test_CreateContainer_MemorySize_Annotation_WCOW_Process(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - "io.microsoft.container.memory.sizeinmb": fmt.Sprintf("%d", 768*1024*1024), // 768MB + oci.AnnotationContainerMemorySizeInMB: fmt.Sprintf("%d", 768*1024*1024), // 768MB }, }, } @@ -364,7 +365,7 @@ func Test_CreateContainer_MemorySize_Annotation_WCOW_Hypervisor(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - "io.microsoft.container.memory.sizeinmb": fmt.Sprintf("%d", 768*1024*1024), // 768MB + oci.AnnotationContainerMemorySizeInMB: fmt.Sprintf("%d", 768*1024*1024), // 768MB }, }, } @@ -451,7 +452,7 @@ func Test_CreateContainer_CPUCount_Annotation_WCOW_Process(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - "io.microsoft.container.processor.count": "1", + oci.AnnotationContainerProcessorCount: "1", }, }, } @@ -511,7 +512,7 @@ func Test_CreateContainer_CPUCount_Annotation_WCOW_Hypervisor(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - "io.microsoft.container.processor.count": "1", + oci.AnnotationContainerProcessorCount: "1", }, }, } @@ -598,7 +599,7 @@ func Test_CreateContainer_CPULimit_Annotation_WCOW_Process(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - "io.microsoft.container.processor.limit": "9000", + oci.AnnotationContainerProcessorLimit: "9000", }, }, } @@ -658,7 +659,7 @@ func Test_CreateContainer_CPULimit_Annotation_WCOW_Hypervisor(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - "io.microsoft.container.processor.limit": "9000", + oci.AnnotationContainerProcessorLimit: "9000", }, }, } @@ -746,7 +747,7 @@ func Test_CreateContainer_CPUWeight_Annotation_WCOW_Process(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - "io.microsoft.container.processor.weight": "500", + oci.AnnotationContainerProcessorWeight: "500", }, }, } @@ -806,7 +807,7 @@ func Test_CreateContainer_CPUWeight_Annotation_WCOW_Hypervisor(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - "io.microsoft.container.processor.limit": "500", + oci.AnnotationContainerProcessorLimit: "500", }, }, } diff --git a/test/cri-containerd/jobcontainer_test.go b/test/cri-containerd/jobcontainer_test.go index d1d0678dd8..f0f55e11ce 100644 --- a/test/cri-containerd/jobcontainer_test.go +++ b/test/cri-containerd/jobcontainer_test.go @@ -15,6 +15,7 @@ import ( "github.com/Microsoft/go-winio/vhd" "github.com/Microsoft/hcsshim/hcn" + "github.com/Microsoft/hcsshim/internal/oci" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -26,7 +27,7 @@ func getJobContainerPodRequestWCOW(t *testing.T) *runtime.RunPodSandboxRequest { Namespace: testNamespace, }, Annotations: map[string]string{ - "microsoft.com/hostprocess-container": "true", + oci.AnnotationHostProcessContainer: "true", }, }, RuntimeHandler: wcowProcessRuntimeHandler, @@ -51,8 +52,8 @@ func getJobContainerRequestWCOW(t *testing.T, podID string, podConfig *runtime.P }, Annotations: map[string]string{ - "microsoft.com/hostprocess-container": "true", - "microsoft.com/hostprocess-inherit-user": "true", + oci.AnnotationHostProcessContainer: "true", + oci.AnnotationHostProcessInheritUser: "true", }, }, PodSandboxId: podID, diff --git a/test/cri-containerd/pod_update_test.go b/test/cri-containerd/pod_update_test.go index 9bedd9f20c..1c24031d7f 100644 --- a/test/cri-containerd/pod_update_test.go +++ b/test/cri-containerd/pod_update_test.go @@ -7,6 +7,7 @@ import ( "fmt" "testing" + "github.com/Microsoft/hcsshim/internal/oci" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -49,7 +50,7 @@ func Test_Pod_UpdateResources_Memory(t *testing.T) { Namespace: testNamespace, }, Annotations: map[string]string{ - "io.microsoft.container.memory.sizeinmb": fmt.Sprintf("%d", startingMemorySize), + oci.AnnotationContainerMemorySizeInMB: fmt.Sprintf("%d", startingMemorySize), }, }, RuntimeHandler: test.runtimeHandler, @@ -125,8 +126,8 @@ func Test_Pod_UpdateResources_Memory_PA(t *testing.T) { Namespace: testNamespace, }, Annotations: map[string]string{ - "io.microsoft.virtualmachine.fullyphysicallybacked": "true", - "io.microsoft.container.memory.sizeinmb": fmt.Sprintf("%d", startingMemorySize), + oci.AnnotationFullyPhysicallyBacked: "true", + oci.AnnotationContainerMemorySizeInMB: fmt.Sprintf("%d", startingMemorySize), }, }, RuntimeHandler: test.runtimeHandler, diff --git a/test/cri-containerd/runpodsandbox_test.go b/test/cri-containerd/runpodsandbox_test.go index 9ab32ce711..8f1538628b 100644 --- a/test/cri-containerd/runpodsandbox_test.go +++ b/test/cri-containerd/runpodsandbox_test.go @@ -17,6 +17,7 @@ import ( "github.com/Microsoft/hcsshim/internal/cpugroup" "github.com/Microsoft/hcsshim/internal/hcs" "github.com/Microsoft/hcsshim/internal/lcow" + "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/processorinfo" "github.com/Microsoft/hcsshim/osversion" testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" @@ -120,7 +121,7 @@ func Test_RunPodSandbox_VirtualMemory_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.allowovercommit": "true", + oci.AnnotationAllowOvercommit: "true", }, ) runPodSandboxTest(t, request) @@ -135,7 +136,7 @@ func Test_RunPodSandbox_VirtualMemory_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.allowovercommit": "true", + oci.AnnotationAllowOvercommit: "true", }, ) runPodSandboxTest(t, request) @@ -150,8 +151,8 @@ func Test_RunPodSandbox_VirtualMemory_DeferredCommit_WCOW_Hypervisor(t *testing. t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.allowovercommit": "true", - "io.microsoft.virtualmachine.computetopology.memory.enabledeferredcommit": "true", + oci.AnnotationAllowOvercommit: "true", + oci.AnnotationEnableDeferredCommit: "true", }, ) runPodSandboxTest(t, request) @@ -166,8 +167,8 @@ func Test_RunPodSandbox_VirtualMemory_DeferredCommit_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.allowovercommit": "true", - "io.microsoft.virtualmachine.computetopology.memory.enabledeferredcommit": "true", + oci.AnnotationAllowOvercommit: "true", + oci.AnnotationEnableDeferredCommit: "true", }, ) runPodSandboxTest(t, request) @@ -182,7 +183,7 @@ func Test_RunPodSandbox_PhysicalMemory_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.allowovercommit": "false", + oci.AnnotationAllowOvercommit: "false", }, ) runPodSandboxTest(t, request) @@ -197,7 +198,7 @@ func Test_RunPodSandbox_FullyPhysicallyBacked_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.fullyphysicallybacked": "true", + oci.AnnotationFullyPhysicallyBacked: "true", }, ) runPodSandboxTest(t, request) @@ -212,7 +213,7 @@ func Test_RunPodSandbox_VSMBNoDirectMap_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.wcow.virtualSMB.nodirectmap": "true", + oci.AnnotationVSMBNoDirectMap: "true", }, ) runPodSandboxTest(t, request) @@ -227,7 +228,7 @@ func Test_RunPodSandbox_PhysicalMemory_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.allowovercommit": "false", + oci.AnnotationAllowOvercommit: "false", }, ) runPodSandboxTest(t, request) @@ -242,7 +243,7 @@ func Test_RunPodSandbox_FullyPhysicallyBacked_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.fullyphysicallybacked": "true", + oci.AnnotationFullyPhysicallyBacked: "true", }, ) runPodSandboxTest(t, request) @@ -257,7 +258,7 @@ func Test_RunPodSandbox_MemorySize_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, map[string]string{ - "io.microsoft.container.memory.sizeinmb": "128", + oci.AnnotationContainerMemorySizeInMB: "128", }, ) runPodSandboxTest(t, request) @@ -272,7 +273,7 @@ func Test_RunPodSandbox_MemorySize_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.sizeinmb": "768", // 128 is too small for WCOW. It is really slow boot. + oci.AnnotationMemorySizeInMB: "768", // 128 is too small for WCOW. It is really slow boot. }, ) runPodSandboxTest(t, request) @@ -287,7 +288,7 @@ func Test_RunPodSandbox_MemorySize_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.sizeinmb": "200", + oci.AnnotationMemorySizeInMB: "200", }, ) runPodSandboxTest(t, request) @@ -305,9 +306,9 @@ func Test_RunPodSandbox_MMIO_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.lowmmiogapinmb": "100", - "io.microsoft.virtualmachine.computetopology.memory.highmmiobaseinmb": "100", - "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb": "100", + oci.AnnotationMemoryLowMMIOGapInMB: "100", + oci.AnnotationMemoryHighMMIOBaseInMB: "100", + oci.AnnotationMemoryHighMMIOGapInMB: "100", }, ) runPodSandboxTest(t, request) @@ -325,9 +326,9 @@ func Test_RunPodSandbox_MMIO_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.lowmmiogapinmb": "100", - "io.microsoft.virtualmachine.computetopology.memory.highmmiobaseinmb": "100", - "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb": "100", + oci.AnnotationMemoryLowMMIOGapInMB: "100", + oci.AnnotationMemoryHighMMIOBaseInMB: "100", + oci.AnnotationMemoryHighMMIOGapInMB: "100", }, ) runPodSandboxTest(t, request) @@ -345,9 +346,9 @@ func Test_RunPodSandbox_MMIO_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.lowmmiogapinmb": "100", - "io.microsoft.virtualmachine.computetopology.memory.highmmiobaseinmb": "100", - "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb": "100", + oci.AnnotationMemoryLowMMIOGapInMB: "100", + oci.AnnotationMemoryHighMMIOBaseInMB: "100", + oci.AnnotationMemoryHighMMIOGapInMB: "100", }, ) runPodSandboxTest(t, request) @@ -362,7 +363,7 @@ func Test_RunPodSandbox_CPUCount_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, map[string]string{ - "io.microsoft.container.processor.count": "1", + oci.AnnotationContainerProcessorCount: "1", }, ) runPodSandboxTest(t, request) @@ -377,7 +378,7 @@ func Test_RunPodSandbox_CPUCount_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.processor.count": "1", + oci.AnnotationProcessorCount: "1", }, ) runPodSandboxTest(t, request) @@ -392,7 +393,7 @@ func Test_RunPodSandbox_CPUCount_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.processor.count": "1", + oci.AnnotationProcessorCount: "1", }, ) runPodSandboxTest(t, request) @@ -407,7 +408,7 @@ func Test_RunPodSandbox_CPULimit_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, map[string]string{ - "io.microsoft.container.processor.limit": "9000", + oci.AnnotationContainerProcessorLimit: "9000", }, ) runPodSandboxTest(t, request) @@ -422,7 +423,7 @@ func Test_RunPodSandbox_CPULimit_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.processor.limit": "90000", + oci.AnnotationProcessorLimit: "90000", }, ) runPodSandboxTest(t, request) @@ -437,7 +438,7 @@ func Test_RunPodSandbox_CPULimit_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.processor.limit": "90000", + oci.AnnotationProcessorLimit: "90000", }, ) runPodSandboxTest(t, request) @@ -452,7 +453,7 @@ func Test_RunPodSandbox_CPUWeight_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, map[string]string{ - "io.microsoft.container.processor.weight": "500", + oci.AnnotationContainerProcessorWeight: "500", }, ) runPodSandboxTest(t, request) @@ -467,7 +468,7 @@ func Test_RunPodSandbox_CPUWeight_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.container.processor.weight": "500", + oci.AnnotationContainerProcessorWeight: "500", }, ) runPodSandboxTest(t, request) @@ -482,7 +483,7 @@ func Test_RunPodSandbox_CPUWeight_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.processor.weight": "500", + oci.AnnotationProcessorWeight: "500", }, ) runPodSandboxTest(t, request) @@ -497,7 +498,7 @@ func Test_RunPodSandbox_StorageQoSBandwithMax_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, map[string]string{ - "io.microsoft.container.storage.qos.bandwidthmaximum": fmt.Sprintf("%d", 1024*1024), // 1MB/s + oci.AnnotationContainerStorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s }, ) runPodSandboxTest(t, request) @@ -512,7 +513,7 @@ func Test_RunPodSandbox_StorageQoSBandwithMax_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.storageqos.bandwidthmaximum": fmt.Sprintf("%d", 1024*1024), // 1MB/s + oci.AnnotationStorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s }, ) runPodSandboxTest(t, request) @@ -527,7 +528,7 @@ func Test_RunPodSandbox_StorageQoSBandwithMax_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.storageqos.bandwidthmaximum": fmt.Sprintf("%d", 1024*1024), // 1MB/s + oci.AnnotationStorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s }, ) runPodSandboxTest(t, request) @@ -542,7 +543,7 @@ func Test_RunPodSandbox_StorageQoSIopsMax_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, map[string]string{ - "io.microsoft.container.storage.qos.iopsmaximum": "300", + oci.AnnotationContainerStorageQoSIopsMaximum: "300", }, ) runPodSandboxTest(t, request) @@ -557,7 +558,7 @@ func Test_RunPodSandbox_StorageQoSIopsMax_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.storageqos.iopsmaximum": "300", + oci.AnnotationStorageQoSIopsMaximum: "300", }, ) runPodSandboxTest(t, request) @@ -572,7 +573,7 @@ func Test_RunPodSandbox_StorageQoSIopsMax_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.storageqos.iopsmaximum": "300", + oci.AnnotationStorageQoSIopsMaximum: "300", }, ) runPodSandboxTest(t, request) @@ -587,7 +588,7 @@ func Test_RunPodSandbox_InitrdBoot_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.lcow.preferredrootfstype": "initrd", + oci.AnnotationPreferredRootFSType: "initrd", }, ) runPodSandboxTest(t, request) @@ -602,7 +603,7 @@ func Test_RunPodSandbox_RootfsVhdBoot_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.lcow.preferredrootfstype": "vhd", + oci.AnnotationPreferredRootFSType: "vhd", }, ) runPodSandboxTest(t, request) @@ -617,7 +618,7 @@ func Test_RunPodSandbox_VPCIEnabled_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.lcow.vpcienabled": "true", + oci.AnnotationVPCIEnabled: "true", }, ) runPodSandboxTest(t, request) @@ -632,7 +633,7 @@ func Test_RunPodSandbox_UEFIBoot_LCOW(t *testing.T) { t, lcowRuntimeHandler, map[string]string{ - "io.microsoft.virtualmachine.lcow.kerneldirectboot": "false", + oci.AnnotationKernelDirectBoot: "false", }, ) runPodSandboxTest(t, request) @@ -737,7 +738,7 @@ func Test_RunPodSandbox_CustomizableScratchDefaultSize_LCOW(t *testing.T) { pullRequiredLcowImages(t, []string{imageLcowK8sPause}) annotations := map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.allowovercommit": "true", + oci.AnnotationAllowOvercommit: "true", } output, errorMsg, exitCode := createSandboxContainerAndExecForCustomScratch(t, annotations) @@ -781,7 +782,7 @@ func Test_RunPodSandbox_CustomizableScratchCustomSize_LCOW(t *testing.T) { pullRequiredLcowImages(t, []string{imageLcowK8sPause}) annotations := map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.allowovercommit": "true", + oci.AnnotationAllowOvercommit: "true", "containerd.io/snapshot/io.microsoft.container.storage.rootfs.size-gb": "200", } @@ -828,7 +829,7 @@ func Test_RunPodSandbox_Mount_SandboxDir_LCOW(t *testing.T) { pullRequiredLcowImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) annotations := map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.allowovercommit": "true", + oci.AnnotationAllowOvercommit: "true", } mounts := []*runtime.Mount{ @@ -915,7 +916,7 @@ func Test_RunPodSandbox_CPUGroup(t *testing.T) { Namespace: testNamespace, }, Annotations: map[string]string{ - "io.microsoft.virtualmachine.cpugroup.id": presentID, + oci.AnnotationCPUGroupID: presentID, }, }, RuntimeHandler: test.runtimeHandler, @@ -943,7 +944,7 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_LCOW(t *testing.T) { defer cancel() annotations := map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.allowovercommit": "true", + oci.AnnotationAllowOvercommit: "true", } // Create a temporary ext4 VHD to mount into the container. @@ -1156,7 +1157,7 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_WCOW(t *testing.T) { defer cancel() annotations := map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.allowovercommit": "true", + oci.AnnotationAllowOvercommit: "true", } vhdHostDir, err := ioutil.TempDir("", "") diff --git a/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go b/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go index 702fc1a636..b7cf3b044f 100644 --- a/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go +++ b/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/Microsoft/hcsshim/internal/oci" criruntime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -31,7 +32,7 @@ func Test_Scale_CPU_Limits_To_Sandbox(t *testing.T) { // set the limit to (roughly) 1 processor processorLimit := 10000 / runtime.NumCPU() contReq.Config.Annotations = map[string]string{ - "io.microsoft.container.processor.limit": strconv.Itoa(processorLimit), + oci.AnnotationContainerProcessorLimit: strconv.Itoa(processorLimit), } contID := createContainer(t, client, ctx, contReq) diff --git a/test/cri-containerd/stats_test.go b/test/cri-containerd/stats_test.go index ff2abb83b8..dba9dbabfc 100644 --- a/test/cri-containerd/stats_test.go +++ b/test/cri-containerd/stats_test.go @@ -7,6 +7,7 @@ import ( "strconv" "testing" + "github.com/Microsoft/hcsshim/internal/oci" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -368,9 +369,9 @@ func Test_SandboxStats_WorkingSet_PhysicallyBacked(t *testing.T) { t, test.runtimeHandler, map[string]string{ - "io.microsoft.virtualmachine.computetopology.memory.allowovercommit": "false", - "io.microsoft.virtualmachine.computetopology.memory.enabledeferredcommit": "false", - "io.microsoft.virtualmachine.computetopology.memory.sizeinmb": sizeInMBStr, + oci.AnnotationAllowOvercommit: "false", + oci.AnnotationEnableDeferredCommit: "false", + oci.AnnotationMemorySizeInMB: sizeInMBStr, }, ) diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/annotations.go b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/annotations.go new file mode 100644 index 0000000000..ebb5a9f4a4 --- /dev/null +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/annotations.go @@ -0,0 +1,147 @@ +package oci + +const ( + // AnnotationContainerMemorySizeInMB overrides the container memory size set + // via the OCI spec. + // + // Note: This annotation is in MB. OCI is in Bytes. When using this override + // the caller MUST use MB or sizing will be wrong. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Memory.Limit`. + AnnotationContainerMemorySizeInMB = "io.microsoft.container.memory.sizeinmb" + // AnnotationContainerProcessorCount overrides the container processor count + // set via the OCI spec. + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Count`. + AnnotationContainerProcessorCount = "io.microsoft.container.processor.count" + // AnnotationContainerProcessorLimit overrides the container processor limit + // set via the OCI spec. + // + // Limit allows values 1 - 10,000 where 10,000 means 100% CPU. (And is the + // default if omitted) + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.CPU.Maximum`. + AnnotationContainerProcessorLimit = "io.microsoft.container.processor.limit" + // AnnotationContainerProcessorWeight overrides the container processor + // weight set via the OCI spec. + // + // Weight allows values 0 - 10,000. (100 is the default) + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Shares`. + AnnotationContainerProcessorWeight = "io.microsoft.container.processor.weight" + // AnnotationContainerStorageQoSBandwidthMaximum overrides the container + // storage bandwidth per second set via the OCI spec. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Storage.Bps`. + AnnotationContainerStorageQoSBandwidthMaximum = "io.microsoft.container.storage.qos.bandwidthmaximum" + // AnnotationContainerStorageQoSIopsMaximum overrides the container storage + // maximum iops set via the OCI spec. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Storage.Iops`. + AnnotationContainerStorageQoSIopsMaximum = "io.microsoft.container.storage.qos.iopsmaximum" + // AnnotationGPUVHDPath overrides the default path to search for the gpu vhd + AnnotationGPUVHDPath = "io.microsoft.lcow.gpuvhdpath" + // AnnotationAssignedDeviceKernelDrivers indicates what drivers to install in the pod during device + // assignment. This value should contain a list of comma separated directories containing all + // files and information needed to install given driver(s). This may include .sys, + // .inf, .cer, and/or other files used during standard installation with pnputil. + AnnotationAssignedDeviceKernelDrivers = "io.microsoft.assigneddevice.kerneldrivers" + // AnnotationHostProcessInheritUser indicates whether to ignore the username passed in to run a host process + // container as and instead inherit the user token from the executable that is launching the container process. + AnnotationHostProcessInheritUser = "microsoft.com/hostprocess-inherit-user" + // AnnotationHostProcessContainer indicates to launch a host process container (job container in this repository). + AnnotationHostProcessContainer = "microsoft.com/hostprocess-container" + + AnnotationAllowOvercommit = "io.microsoft.virtualmachine.computetopology.memory.allowovercommit" + AnnotationEnableDeferredCommit = "io.microsoft.virtualmachine.computetopology.memory.enabledeferredcommit" + AnnotationEnableColdDiscardHint = "io.microsoft.virtualmachine.computetopology.memory.enablecolddiscardhint" + // annotationMemorySizeInMB overrides the container memory size set via the + // OCI spec. + // + // Note: This annotation is in MB. OCI is in Bytes. When using this override + // the caller MUST use MB or sizing will be wrong. + AnnotationMemorySizeInMB = "io.microsoft.virtualmachine.computetopology.memory.sizeinmb" + AnnotationMemoryLowMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.lowmmiogapinmb" + AnnotationMemoryHighMMIOBaseInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiobaseinmb" + AnnotationMemoryHighMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb" + // annotationProcessorCount overrides the hypervisor isolated vCPU count set + // via the OCI spec. + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + AnnotationProcessorCount = "io.microsoft.virtualmachine.computetopology.processor.count" + // annotationProcessorLimit overrides the hypervisor isolated vCPU limit set + // via the OCI spec. + // + // Limit allows values 1 - 100,000 where 100,000 means 100% CPU. (And is the + // default if omitted) + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + AnnotationProcessorLimit = "io.microsoft.virtualmachine.computetopology.processor.limit" + // annotationProcessorWeight overrides the hypervisor isolated vCPU weight set + // via the OCI spec. + // + // Weight allows values 0 - 10,000. (100 is the default if omitted) + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + AnnotationProcessorWeight = "io.microsoft.virtualmachine.computetopology.processor.weight" + AnnotationVPMemCount = "io.microsoft.virtualmachine.devices.virtualpmem.maximumcount" + AnnotationVPMemSize = "io.microsoft.virtualmachine.devices.virtualpmem.maximumsizebytes" + AnnotationPreferredRootFSType = "io.microsoft.virtualmachine.lcow.preferredrootfstype" + AnnotationBootFilesRootPath = "io.microsoft.virtualmachine.lcow.bootfilesrootpath" + AnnotationKernelDirectBoot = "io.microsoft.virtualmachine.lcow.kerneldirectboot" + AnnotationVPCIEnabled = "io.microsoft.virtualmachine.lcow.vpcienabled" + AnnotationVPMemNoMultiMapping = "io.microsoft.virtualmachine.lcow.vpmem.nomultimapping" + AnnotationStorageQoSBandwidthMaximum = "io.microsoft.virtualmachine.storageqos.bandwidthmaximum" + AnnotationStorageQoSIopsMaximum = "io.microsoft.virtualmachine.storageqos.iopsmaximum" + AnnotationFullyPhysicallyBacked = "io.microsoft.virtualmachine.fullyphysicallybacked" + AnnotationDisableCompartmentNamespace = "io.microsoft.virtualmachine.disablecompartmentnamespace" + AnnotationVSMBNoDirectMap = "io.microsoft.virtualmachine.wcow.virtualSMB.nodirectmap" + + // annotation used to specify the cpugroup ID that a UVM should be assigned to + AnnotationCPUGroupID = "io.microsoft.virtualmachine.cpugroup.id" + + // SaveAsTemplate annotation must be used with a pod & container creation request. + // If this annotation is present in the request then it will save the UVM (pod) + // and the container(s) inside it as a template. However, this also means that this + // pod and the containers inside this pod will permananetly stay in the + // paused/templated state and can not be resumed again. + AnnotationSaveAsTemplate = "io.microsoft.virtualmachine.saveastemplate" + + // This annotation should be used when creating a pod or a container from a template. + // When creating a pod from a template use the ID of the templated pod as the + // TemplateID and when creating a container use the ID of the templated container as + // the TemplateID. It is the client's responsibility to make sure that the sandbox + // within which a cloned container needs to be created must also be created from the + // same template. + AnnotationTemplateID = "io.microsoft.virtualmachine.templateid" + AnnotationNetworkConfigProxy = "io.microsoft.network.ncproxy" + AnnotationNcproxyContainerID = "io.microsoft.network.ncproxy.containerid" +) diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go index e821837ed0..5b8c5e75c1 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go @@ -16,152 +16,6 @@ import ( "github.com/sirupsen/logrus" ) -const ( - // AnnotationContainerMemorySizeInMB overrides the container memory size set - // via the OCI spec. - // - // Note: This annotation is in MB. OCI is in Bytes. When using this override - // the caller MUST use MB or sizing will be wrong. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Memory.Limit`. - AnnotationContainerMemorySizeInMB = "io.microsoft.container.memory.sizeinmb" - // AnnotationContainerProcessorCount overrides the container processor count - // set via the OCI spec. - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Count`. - AnnotationContainerProcessorCount = "io.microsoft.container.processor.count" - // AnnotationContainerProcessorLimit overrides the container processor limit - // set via the OCI spec. - // - // Limit allows values 1 - 10,000 where 10,000 means 100% CPU. (And is the - // default if omitted) - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.CPU.Maximum`. - AnnotationContainerProcessorLimit = "io.microsoft.container.processor.limit" - // AnnotationContainerProcessorWeight overrides the container processor - // weight set via the OCI spec. - // - // Weight allows values 0 - 10,000. (100 is the default) - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Shares`. - AnnotationContainerProcessorWeight = "io.microsoft.container.processor.weight" - // AnnotationContainerStorageQoSBandwidthMaximum overrides the container - // storage bandwidth per second set via the OCI spec. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Storage.Bps`. - AnnotationContainerStorageQoSBandwidthMaximum = "io.microsoft.container.storage.qos.bandwidthmaximum" - // AnnotationContainerStorageQoSIopsMaximum overrides the container storage - // maximum iops set via the OCI spec. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Storage.Iops`. - AnnotationContainerStorageQoSIopsMaximum = "io.microsoft.container.storage.qos.iopsmaximum" - // AnnotationGPUVHDPath overrides the default path to search for the gpu vhd - AnnotationGPUVHDPath = "io.microsoft.lcow.gpuvhdpath" - // AnnotationAssignedDeviceKernelDrivers indicates what drivers to install in the pod during device - // assignment. This value should contain a list of comma separated directories containing all - // files and information needed to install given driver(s). This may include .sys, - // .inf, .cer, and/or other files used during standard installation with pnputil. - AnnotationAssignedDeviceKernelDrivers = "io.microsoft.assigneddevice.kerneldrivers" - // AnnotationHostProcessInheritUser indicates whether to ignore the username passed in to run a host process - // container as and instead inherit the user token from the executable that is launching the container process. - AnnotationHostProcessInheritUser = "microsoft.com/hostprocess-inherit-user" - // AnnotationHostProcessContainer indicates to launch a host process container (job container in this repository). - AnnotationHostProcessContainer = "microsoft.com/hostprocess-container" - - annotationAllowOvercommit = "io.microsoft.virtualmachine.computetopology.memory.allowovercommit" - annotationEnableDeferredCommit = "io.microsoft.virtualmachine.computetopology.memory.enabledeferredcommit" - annotationEnableColdDiscardHint = "io.microsoft.virtualmachine.computetopology.memory.enablecolddiscardhint" - // annotationMemorySizeInMB overrides the container memory size set via the - // OCI spec. - // - // Note: This annotation is in MB. OCI is in Bytes. When using this override - // the caller MUST use MB or sizing will be wrong. - annotationMemorySizeInMB = "io.microsoft.virtualmachine.computetopology.memory.sizeinmb" - annotationMemoryLowMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.lowmmiogapinmb" - annotationMemoryHighMMIOBaseInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiobaseinmb" - annotationMemoryHighMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb" - // annotationProcessorCount overrides the hypervisor isolated vCPU count set - // via the OCI spec. - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - annotationProcessorCount = "io.microsoft.virtualmachine.computetopology.processor.count" - // annotationProcessorLimit overrides the hypervisor isolated vCPU limit set - // via the OCI spec. - // - // Limit allows values 1 - 100,000 where 100,000 means 100% CPU. (And is the - // default if omitted) - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - annotationProcessorLimit = "io.microsoft.virtualmachine.computetopology.processor.limit" - // annotationProcessorWeight overrides the hypervisor isolated vCPU weight set - // via the OCI spec. - // - // Weight allows values 0 - 10,000. (100 is the default if omitted) - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - annotationProcessorWeight = "io.microsoft.virtualmachine.computetopology.processor.weight" - annotationVPMemCount = "io.microsoft.virtualmachine.devices.virtualpmem.maximumcount" - annotationVPMemSize = "io.microsoft.virtualmachine.devices.virtualpmem.maximumsizebytes" - annotationPreferredRootFSType = "io.microsoft.virtualmachine.lcow.preferredrootfstype" - annotationBootFilesRootPath = "io.microsoft.virtualmachine.lcow.bootfilesrootpath" - annotationKernelDirectBoot = "io.microsoft.virtualmachine.lcow.kerneldirectboot" - annotationVPCIEnabled = "io.microsoft.virtualmachine.lcow.vpcienabled" - annotationVPMemNoMultiMapping = "io.microsoft.virtualmachine.lcow.vpmem.nomultimapping" - annotationStorageQoSBandwidthMaximum = "io.microsoft.virtualmachine.storageqos.bandwidthmaximum" - annotationStorageQoSIopsMaximum = "io.microsoft.virtualmachine.storageqos.iopsmaximum" - annotationFullyPhysicallyBacked = "io.microsoft.virtualmachine.fullyphysicallybacked" - annotationDisableCompartmentNamespace = "io.microsoft.virtualmachine.disablecompartmentnamespace" - annotationVSMBNoDirectMap = "io.microsoft.virtualmachine.wcow.virtualSMB.nodirectmap" - - // annotation used to specify the cpugroup ID that a UVM should be assigned to - annotationCPUGroupID = "io.microsoft.virtualmachine.cpugroup.id" - - // SaveAsTemplate annotation must be used with a pod & container creation request. - // If this annotation is present in the request then it will save the UVM (pod) - // and the container(s) inside it as a template. However, this also means that this - // pod and the containers inside this pod will permananetly stay in the - // paused/templated state and can not be resumed again. - annotationSaveAsTemplate = "io.microsoft.virtualmachine.saveastemplate" - - // This annotation should be used when creating a pod or a container from a template. - // When creating a pod from a template use the ID of the templated pod as the - // TemplateID and when creating a container use the ID of the templated container as - // the TemplateID. It is the client's responsibility to make sure that the sandbox - // within which a cloned container needs to be created must also be created from the - // same template. - annotationTemplateID = "io.microsoft.virtualmachine.templateid" - annotationNetworkConfigProxy = "io.microsoft.network.ncproxy" - AnnotationNcproxyContainerID = "io.microsoft.network.ncproxy.containerid" -) - // parseAnnotationsBool searches `a` for `key` and if found verifies that the // value is `true` or `false` in any case. If `key` is not found returns `def`. func parseAnnotationsBool(ctx context.Context, a map[string]string, key string, def bool) bool { @@ -354,13 +208,13 @@ func parseAnnotationsString(a map[string]string, key string, def string) string // if this create request should be considered as a template creation request. If value // is found the returns the actual value, returns false otherwise. func ParseAnnotationsSaveAsTemplate(ctx context.Context, s *specs.Spec) bool { - return parseAnnotationsBool(ctx, s.Annotations, annotationSaveAsTemplate, false) + return parseAnnotationsBool(ctx, s.Annotations, AnnotationSaveAsTemplate, false) } // ParseAnnotationsTemplateID searches for the templateID in the create request. If the // value is found then returns the value otherwise returns the empty string. func ParseAnnotationsTemplateID(ctx context.Context, s *specs.Spec) string { - return parseAnnotationsString(s.Annotations, annotationTemplateID, "") + return parseAnnotationsString(s.Annotations, AnnotationTemplateID, "") } func ParseCloneAnnotations(ctx context.Context, s *specs.Spec) (isTemplate bool, templateID string, err error) { @@ -379,7 +233,7 @@ func ParseCloneAnnotations(ctx context.Context, s *specs.Spec) (isTemplate bool, // handleAnnotationKernelDirectBoot handles parsing annotationKernelDirectBoot and setting // implied annotations from the result. func handleAnnotationKernelDirectBoot(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { - lopts.KernelDirect = parseAnnotationsBool(ctx, a, annotationKernelDirectBoot, lopts.KernelDirect) + lopts.KernelDirect = parseAnnotationsBool(ctx, a, AnnotationKernelDirectBoot, lopts.KernelDirect) if !lopts.KernelDirect { lopts.KernelFile = uvm.KernelFile } @@ -388,7 +242,7 @@ func handleAnnotationKernelDirectBoot(ctx context.Context, a map[string]string, // handleAnnotationPreferredRootFSType handles parsing annotationPreferredRootFSType and setting // implied annotations from the result func handleAnnotationPreferredRootFSType(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { - lopts.PreferredRootFSType = parseAnnotationsPreferredRootFSType(ctx, a, annotationPreferredRootFSType, lopts.PreferredRootFSType) + lopts.PreferredRootFSType = parseAnnotationsPreferredRootFSType(ctx, a, AnnotationPreferredRootFSType, lopts.PreferredRootFSType) switch lopts.PreferredRootFSType { case uvm.PreferredRootFSTypeInitRd: lopts.RootFSFile = uvm.InitrdFile @@ -402,7 +256,7 @@ func handleAnnotationPreferredRootFSType(ctx context.Context, a map[string]strin func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]string, opts interface{}) { switch options := opts.(type) { case *uvm.OptionsLCOW: - options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, annotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) + options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, AnnotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) if options.FullyPhysicallyBacked { options.AllowOvercommit = false options.PreferredRootFSType = uvm.PreferredRootFSTypeInitRd @@ -410,7 +264,7 @@ func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]str options.VPMemDeviceCount = 0 } case *uvm.OptionsWCOW: - options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, annotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) + options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, AnnotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) if options.FullyPhysicallyBacked { options.AllowOvercommit = false } @@ -420,8 +274,8 @@ func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]str // handleCloneAnnotations handles parsing annotations related to template creation and cloning // Since late cloning is only supported for WCOW this function only deals with WCOW options. func handleCloneAnnotations(ctx context.Context, a map[string]string, wopts *uvm.OptionsWCOW) (err error) { - wopts.IsTemplate = parseAnnotationsBool(ctx, a, annotationSaveAsTemplate, false) - templateID := parseAnnotationsString(a, annotationTemplateID, "") + wopts.IsTemplate = parseAnnotationsBool(ctx, a, AnnotationSaveAsTemplate, false) + templateID := parseAnnotationsString(a, AnnotationTemplateID, "") if templateID != "" { tc, err := clone.FetchTemplateConfig(ctx, templateID) if err != nil { @@ -445,25 +299,25 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( } if IsLCOW(s) { lopts := uvm.NewDefaultOptionsLCOW(id, owner) - lopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, annotationMemorySizeInMB, lopts.MemorySizeInMB) - lopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotationMemoryLowMMIOGapInMB, lopts.LowMMIOGapInMB) - lopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, annotationMemoryHighMMIOBaseInMB, lopts.HighMMIOBaseInMB) - lopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotationMemoryHighMMIOGapInMB, lopts.HighMMIOGapInMB) - lopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, annotationAllowOvercommit, lopts.AllowOvercommit) - lopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, annotationEnableDeferredCommit, lopts.EnableDeferredCommit) - lopts.EnableColdDiscardHint = parseAnnotationsBool(ctx, s.Annotations, annotationEnableColdDiscardHint, lopts.EnableColdDiscardHint) - lopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, annotationProcessorCount, lopts.ProcessorCount) - lopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, annotationProcessorLimit, lopts.ProcessorLimit) - lopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, annotationProcessorWeight, lopts.ProcessorWeight) - lopts.VPMemDeviceCount = parseAnnotationsUint32(ctx, s.Annotations, annotationVPMemCount, lopts.VPMemDeviceCount) - lopts.VPMemSizeBytes = parseAnnotationsUint64(ctx, s.Annotations, annotationVPMemSize, lopts.VPMemSizeBytes) - lopts.VPMemNoMultiMapping = parseAnnotationsBool(ctx, s.Annotations, annotationVPMemNoMultiMapping, lopts.VPMemNoMultiMapping) - lopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, annotationStorageQoSBandwidthMaximum, lopts.StorageQoSBandwidthMaximum) - lopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotationStorageQoSIopsMaximum, lopts.StorageQoSIopsMaximum) - lopts.VPCIEnabled = parseAnnotationsBool(ctx, s.Annotations, annotationVPCIEnabled, lopts.VPCIEnabled) - lopts.BootFilesPath = parseAnnotationsString(s.Annotations, annotationBootFilesRootPath, lopts.BootFilesPath) - lopts.CPUGroupID = parseAnnotationsString(s.Annotations, annotationCPUGroupID, lopts.CPUGroupID) - lopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, annotationNetworkConfigProxy, lopts.NetworkConfigProxy) + lopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, AnnotationMemorySizeInMB, lopts.MemorySizeInMB) + lopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryLowMMIOGapInMB, lopts.LowMMIOGapInMB) + lopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOBaseInMB, lopts.HighMMIOBaseInMB) + lopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOGapInMB, lopts.HighMMIOGapInMB) + lopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationAllowOvercommit, lopts.AllowOvercommit) + lopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableDeferredCommit, lopts.EnableDeferredCommit) + lopts.EnableColdDiscardHint = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableColdDiscardHint, lopts.EnableColdDiscardHint) + lopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, AnnotationProcessorCount, lopts.ProcessorCount) + lopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, AnnotationProcessorLimit, lopts.ProcessorLimit) + lopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, AnnotationProcessorWeight, lopts.ProcessorWeight) + lopts.VPMemDeviceCount = parseAnnotationsUint32(ctx, s.Annotations, AnnotationVPMemCount, lopts.VPMemDeviceCount) + lopts.VPMemSizeBytes = parseAnnotationsUint64(ctx, s.Annotations, AnnotationVPMemSize, lopts.VPMemSizeBytes) + lopts.VPMemNoMultiMapping = parseAnnotationsBool(ctx, s.Annotations, AnnotationVPMemNoMultiMapping, lopts.VPMemNoMultiMapping) + lopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, AnnotationStorageQoSBandwidthMaximum, lopts.StorageQoSBandwidthMaximum) + lopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, AnnotationStorageQoSIopsMaximum, lopts.StorageQoSIopsMaximum) + lopts.VPCIEnabled = parseAnnotationsBool(ctx, s.Annotations, AnnotationVPCIEnabled, lopts.VPCIEnabled) + lopts.BootFilesPath = parseAnnotationsString(s.Annotations, AnnotationBootFilesRootPath, lopts.BootFilesPath) + lopts.CPUGroupID = parseAnnotationsString(s.Annotations, AnnotationCPUGroupID, lopts.CPUGroupID) + lopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, AnnotationNetworkConfigProxy, lopts.NetworkConfigProxy) handleAnnotationPreferredRootFSType(ctx, s.Annotations, lopts) handleAnnotationKernelDirectBoot(ctx, s.Annotations, lopts) @@ -473,21 +327,21 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( return lopts, nil } else if IsWCOW(s) { wopts := uvm.NewDefaultOptionsWCOW(id, owner) - wopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, annotationMemorySizeInMB, wopts.MemorySizeInMB) - wopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotationMemoryLowMMIOGapInMB, wopts.LowMMIOGapInMB) - wopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, annotationMemoryHighMMIOBaseInMB, wopts.HighMMIOBaseInMB) - wopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotationMemoryHighMMIOGapInMB, wopts.HighMMIOGapInMB) - wopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, annotationAllowOvercommit, wopts.AllowOvercommit) - wopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, annotationEnableDeferredCommit, wopts.EnableDeferredCommit) - wopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, annotationProcessorCount, wopts.ProcessorCount) - wopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, annotationProcessorLimit, wopts.ProcessorLimit) - wopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, annotationProcessorWeight, wopts.ProcessorWeight) - wopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, annotationStorageQoSBandwidthMaximum, wopts.StorageQoSBandwidthMaximum) - wopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotationStorageQoSIopsMaximum, wopts.StorageQoSIopsMaximum) - wopts.DisableCompartmentNamespace = parseAnnotationsBool(ctx, s.Annotations, annotationDisableCompartmentNamespace, wopts.DisableCompartmentNamespace) - wopts.CPUGroupID = parseAnnotationsString(s.Annotations, annotationCPUGroupID, wopts.CPUGroupID) - wopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, annotationNetworkConfigProxy, wopts.NetworkConfigProxy) - wopts.NoDirectMap = parseAnnotationsBool(ctx, s.Annotations, annotationVSMBNoDirectMap, wopts.NoDirectMap) + wopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, AnnotationMemorySizeInMB, wopts.MemorySizeInMB) + wopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryLowMMIOGapInMB, wopts.LowMMIOGapInMB) + wopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOBaseInMB, wopts.HighMMIOBaseInMB) + wopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOGapInMB, wopts.HighMMIOGapInMB) + wopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationAllowOvercommit, wopts.AllowOvercommit) + wopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableDeferredCommit, wopts.EnableDeferredCommit) + wopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, AnnotationProcessorCount, wopts.ProcessorCount) + wopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, AnnotationProcessorLimit, wopts.ProcessorLimit) + wopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, AnnotationProcessorWeight, wopts.ProcessorWeight) + wopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, AnnotationStorageQoSBandwidthMaximum, wopts.StorageQoSBandwidthMaximum) + wopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, AnnotationStorageQoSIopsMaximum, wopts.StorageQoSIopsMaximum) + wopts.DisableCompartmentNamespace = parseAnnotationsBool(ctx, s.Annotations, AnnotationDisableCompartmentNamespace, wopts.DisableCompartmentNamespace) + wopts.CPUGroupID = parseAnnotationsString(s.Annotations, AnnotationCPUGroupID, wopts.CPUGroupID) + wopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, AnnotationNetworkConfigProxy, wopts.NetworkConfigProxy) + wopts.NoDirectMap = parseAnnotationsBool(ctx, s.Annotations, AnnotationVSMBNoDirectMap, wopts.NoDirectMap) handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, wopts) if err := handleCloneAnnotations(ctx, s.Annotations, wopts); err != nil { return nil, err @@ -504,24 +358,24 @@ func UpdateSpecFromOptions(s specs.Spec, opts *runhcsopts.Options) specs.Spec { return s } - if _, ok := s.Annotations[annotationBootFilesRootPath]; !ok && opts.BootFilesRootPath != "" { - s.Annotations[annotationBootFilesRootPath] = opts.BootFilesRootPath + if _, ok := s.Annotations[AnnotationBootFilesRootPath]; !ok && opts.BootFilesRootPath != "" { + s.Annotations[AnnotationBootFilesRootPath] = opts.BootFilesRootPath } - if _, ok := s.Annotations[annotationProcessorCount]; !ok && opts.VmProcessorCount != 0 { - s.Annotations[annotationProcessorCount] = strconv.FormatInt(int64(opts.VmProcessorCount), 10) + if _, ok := s.Annotations[AnnotationProcessorCount]; !ok && opts.VmProcessorCount != 0 { + s.Annotations[AnnotationProcessorCount] = strconv.FormatInt(int64(opts.VmProcessorCount), 10) } - if _, ok := s.Annotations[annotationMemorySizeInMB]; !ok && opts.VmMemorySizeInMb != 0 { - s.Annotations[annotationMemorySizeInMB] = strconv.FormatInt(int64(opts.VmMemorySizeInMb), 10) + if _, ok := s.Annotations[AnnotationMemorySizeInMB]; !ok && opts.VmMemorySizeInMb != 0 { + s.Annotations[AnnotationMemorySizeInMB] = strconv.FormatInt(int64(opts.VmMemorySizeInMb), 10) } if _, ok := s.Annotations[AnnotationGPUVHDPath]; !ok && opts.GPUVHDPath != "" { s.Annotations[AnnotationGPUVHDPath] = opts.GPUVHDPath } - if _, ok := s.Annotations[annotationNetworkConfigProxy]; !ok && opts.NCProxyAddr != "" { - s.Annotations[annotationNetworkConfigProxy] = opts.NCProxyAddr + if _, ok := s.Annotations[AnnotationNetworkConfigProxy]; !ok && opts.NCProxyAddr != "" { + s.Annotations[AnnotationNetworkConfigProxy] = opts.NCProxyAddr } return s