diff --git a/cmd/containerd-shim-runhcs-v1/pod.go b/cmd/containerd-shim-runhcs-v1/pod.go index 2d89b0d8ea..d6c3cc7662 100644 --- a/cmd/containerd-shim-runhcs-v1/pod.go +++ b/cmd/containerd-shim-runhcs-v1/pod.go @@ -126,6 +126,10 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques parent.Close() return nil, err } + if err := oci.HandleCPUGroupSetup(ctx, parent, s.Annotations); err != nil { + parent.Close() + return nil, err + } } else if !isWCOW { return nil, errors.Wrap(errdefs.ErrFailedPrecondition, "oci spec does not contain WCOW or LCOW spec") } diff --git a/internal/oci/cpugroup.go b/internal/oci/cpugroup.go new file mode 100644 index 0000000000..b93eb93969 --- /dev/null +++ b/internal/oci/cpugroup.go @@ -0,0 +1,41 @@ +package oci + +import ( + "context" + "fmt" + + "github.com/Microsoft/hcsshim/internal/uvm" +) + +// HandleCPUGroupSetup will parse the cpugroup annotations and setup the cpugroup for `vm` +func HandleCPUGroupSetup(ctx context.Context, vm *uvm.UtilityVM, annotations map[string]string) error { + cpuGroupOpts, err := annotationsToCPUGroupOptions(ctx, annotations) + if err != nil { + return err + } + if err := vm.ConfigureVMCPUGroup(ctx, cpuGroupOpts); err != nil { + return err + } + return nil +} + +// annotationsToCPUGroupOptions parses the related cpugroup annotations and creates the CPUGroupOptions from the values +func annotationsToCPUGroupOptions(ctx context.Context, annotations map[string]string) (*uvm.CPUGroupOptions, error) { + processorTopology, err := uvm.HostProcessorInfo(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get host processor information: %s", err) + } + lpIndices := []uint32{} + for _, l := range processorTopology.LogicalProcessors { + lpIndices = append(lpIndices, l.LpIndex) + } + + opts := &uvm.CPUGroupOptions{ + CreateRandomID: parseAnnotationsBool(ctx, annotations, annotationCPUGroupCreateRandomID, false), + ID: parseAnnotationsString(annotations, annotationCPUGroupID, uvm.CPUGroupNullID), + LogicalProcessors: parseCommaSeperatedUint32(annotations, annotationCPUGroupLPs, lpIndices), + Cap: parseAnnotationsUint32(ctx, annotations, annotationCPUGroupCap, uvm.DefaultCPUGroupCap), + Priority: parseAnnotationsUint32(ctx, annotations, annotationCPUGroupPriority, uvm.DefaultCPUGroupPriority), + } + return opts, nil +} diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index c25d94fa43..cf492cc02e 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -128,6 +128,13 @@ const ( // HCS-GCS bridge. Default value is true which means external bridge will be used // by default. annotationUseExternalGCSBridge = "io.microsoft.virtualmachine.useexternalgcsbridge" + + // annotations relating to cpugroup creation + annotationCPUGroupCreateRandomID = "io.microsoft.virtualmachine.cpugroup.randomid" + annotationCPUGroupID = "io.microsoft.virtualmachine.cpugroup.id" + annotationCPUGroupLPs = "io.microsoft.virtualmachine.cpugroup.logicalprocessors" + annotationCPUGroupCap = "io.microsoft.virtualmachine.cpugroup.cap" + annotationCPUGroupPriority = "io.microsoft.virtualmachine.cpugroup.priority" ) // parseAnnotationsBool searches `a` for `key` and if found verifies that the @@ -318,6 +325,25 @@ func parseAnnotationsString(a map[string]string, key string, def string) string return def } +// parseCommaSeperatedUint32 searches `a` for `key`. If found verifies that the value +// is a comma separated slice of 32 bit unsigned integers. If `key` is not found, +// returns `def`. +func parseCommaSeperatedUint32(annotations map[string]string, key string, def []uint32) []uint32 { + if v, ok := annotations[key]; ok { + splitStrings := strings.Split(v, ",") + result := make([]uint32, len(splitStrings)) + for i, c := range splitStrings { + countu, err := strconv.ParseUint(c, 10, 32) + if err != nil { + return def + } + result[i] = uint32(countu) + } + return result + } + return def +} + // handleAnnotationKernelDirectBoot handles parsing annotationKernelDirectBoot and setting // implied annotations from the result. func handleAnnotationKernelDirectBoot(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { diff --git a/internal/schema2/cpu_group.go b/internal/schema2/cpu_group.go new file mode 100644 index 0000000000..90332a5190 --- /dev/null +++ b/internal/schema2/cpu_group.go @@ -0,0 +1,15 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// CPU groups allow Hyper-V administrators to better manage and allocate the host's CPU resources across guest virtual machines +type CpuGroup struct { + Id string `json:"Id,omitempty"` +} diff --git a/internal/schema2/cpu_group_affinity.go b/internal/schema2/cpu_group_affinity.go new file mode 100644 index 0000000000..8794961bf5 --- /dev/null +++ b/internal/schema2/cpu_group_affinity.go @@ -0,0 +1,15 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type CpuGroupAffinity struct { + LogicalProcessorCount int32 `json:"LogicalProcessorCount,omitempty"` + LogicalProcessors []int32 `json:"LogicalProcessors,omitempty"` +} diff --git a/internal/schema2/cpu_group_config.go b/internal/schema2/cpu_group_config.go new file mode 100644 index 0000000000..f1a28cd389 --- /dev/null +++ b/internal/schema2/cpu_group_config.go @@ -0,0 +1,18 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type CpuGroupConfig struct { + GroupId string `json:"GroupId,omitempty"` + Affinity *CpuGroupAffinity `json:"Affinity,omitempty"` + GroupProperties []CpuGroupProperty `json:"GroupProperties,omitempty"` + // Hypervisor CPU group IDs exposed to clients + HypervisorGroupId int32 `json:"HypervisorGroupId,omitempty"` +} diff --git a/internal/schema2/cpu_group_configurations.go b/internal/schema2/cpu_group_configurations.go new file mode 100644 index 0000000000..3ace0ccc3b --- /dev/null +++ b/internal/schema2/cpu_group_configurations.go @@ -0,0 +1,15 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// Structure used to return cpu groups for a Service property query +type CpuGroupConfigurations struct { + CpuGroups []CpuGroupConfig `json:"CpuGroups,omitempty"` +} diff --git a/internal/schema2/cpu_group_operations.go b/internal/schema2/cpu_group_operations.go new file mode 100644 index 0000000000..195557cfe0 --- /dev/null +++ b/internal/schema2/cpu_group_operations.go @@ -0,0 +1,18 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type CPUGroupOperation string + +const ( + CreateGroup CPUGroupOperation = "CreateGroup" + DeleteGroup CPUGroupOperation = "DeleteGroup" + SetProperty CPUGroupOperation = "SetProperty" +) diff --git a/internal/schema2/cpu_group_property.go b/internal/schema2/cpu_group_property.go new file mode 100644 index 0000000000..bbad6a2c45 --- /dev/null +++ b/internal/schema2/cpu_group_property.go @@ -0,0 +1,15 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type CpuGroupProperty struct { + PropertyCode uint32 `json:"PropertyCode,omitempty"` + PropertyValue uint32 `json:"PropertyValue,omitempty"` +} diff --git a/internal/schema2/create_group_operation.go b/internal/schema2/create_group_operation.go new file mode 100644 index 0000000000..91a8278fe3 --- /dev/null +++ b/internal/schema2/create_group_operation.go @@ -0,0 +1,17 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// Create group operation settings +type CreateGroupOperation struct { + GroupId string `json:"GroupId,omitempty"` + LogicalProcessorCount uint32 `json:"LogicalProcessorCount,omitempty"` + LogicalProcessors []uint32 `json:"LogicalProcessors,omitempty"` +} diff --git a/internal/schema2/delete_group_operation.go b/internal/schema2/delete_group_operation.go new file mode 100644 index 0000000000..134bd98817 --- /dev/null +++ b/internal/schema2/delete_group_operation.go @@ -0,0 +1,15 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// Delete group operation settings +type DeleteGroupOperation struct { + GroupId string `json:"GroupId,omitempty"` +} diff --git a/internal/schema2/host_processor_modify_request.go b/internal/schema2/host_processor_modify_request.go new file mode 100644 index 0000000000..2238ce5306 --- /dev/null +++ b/internal/schema2/host_processor_modify_request.go @@ -0,0 +1,16 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +// Structure used to request a service processor modification +type HostProcessorModificationRequest struct { + Operation CPUGroupOperation `json:"Operation,omitempty"` + OperationDetails interface{} `json:"OperationDetails,omitempty"` +} diff --git a/internal/schema2/property_type.go b/internal/schema2/property_type.go index 0f1ee621a6..98f2c96edb 100644 --- a/internal/schema2/property_type.go +++ b/internal/schema2/property_type.go @@ -22,4 +22,5 @@ const ( PTGuestConnection PropertyType = "GuestConnection" PTICHeartbeatStatus PropertyType = "ICHeartbeatStatus" PTProcessorTopology PropertyType = "ProcessorTopology" + PTCPUGroup PropertyType = "CpuGroup" ) diff --git a/internal/schema2/set_property_operation.go b/internal/schema2/set_property_operation.go new file mode 100644 index 0000000000..7359fc3e7a --- /dev/null +++ b/internal/schema2/set_property_operation.go @@ -0,0 +1,22 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +const ( + CpuCapPropertyCode = uint32(0x00010000) + SchedulingPriorityPropertyCode = uint32(0x00020000) +) + +// Set properties operation settings +type SetPropertyOperation struct { + GroupId string `json:"GroupId,omitempty"` + PropertyCode uint32 `json:"PropertyCode,omitempty"` + PropertyValue uint32 `json:"PropertyValue,omitempty"` +} diff --git a/internal/uvm/cpugroups.go b/internal/uvm/cpugroups.go new file mode 100644 index 0000000000..1278938221 --- /dev/null +++ b/internal/uvm/cpugroups.go @@ -0,0 +1,289 @@ +package uvm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/hcsshim/internal/hcs" + "github.com/Microsoft/hcsshim/internal/log" + hcsschema "github.com/Microsoft/hcsshim/internal/schema2" +) + +const ( + CPUGroupNullID = "00000000-0000-0000-0000-000000000000" + MaxCPUGroupCap = 65536 + + // Default values in host for cpugroups + DefaultCPUGroupCap = MaxCPUGroupCap + DefaultCPUGroupPriority = 1 +) + +var _HV_STATUS_INVALID_CPU_GROUP_STATE = errors.New("The hypervisor could not perform the operation because the CPU group is entering or in an invalid state.") + +// ReleaseCPUGroup unsets the cpugroup from the VM and attemps to delete it +func (uvm *UtilityVM) ReleaseCPUGroup(ctx context.Context) error { + groupID := uvm.cpuGroupID + if err := uvm.unsetCPUGroup(ctx); err != nil { + return fmt.Errorf("failed to remove VM %s from cpugroup %s", uvm.ID(), groupID) + } + + err := deleteCPUGroup(ctx, groupID) + if err != nil && err == _HV_STATUS_INVALID_CPU_GROUP_STATE { + log.G(ctx).WithField("error", err).Warn("cpugroup could not be deleted, other VMs may be in this group") + return nil + } + return err +} + +// CPUGroupOptions is used to construct the various options for setting up/creating +// a cpugroup for a UVM. +type CPUGroupOptions struct { + CreateRandomID bool + ID string + LogicalProcessors []uint32 + Cap uint32 + Priority uint32 +} + +// verifyCPUGroupOptions verifies that the CPUGroupOptions are a valid cpugroup configuration +func verifyCPUGroupOptions(opts *CPUGroupOptions) error { + if opts.CreateRandomID && opts.ID != CPUGroupNullID { + return fmt.Errorf("cannot use a specific cpugroup ID when the `CreateRandomID` option is set") + } + if len(opts.LogicalProcessors) == 0 { + return fmt.Errorf("must specify the logical processors to use when creating a cpugroup") + } + return nil +} + +// ConfigureVMCPUGroup parses the CPUGroupOptions `opts` and setups up the cpugroup for the VM +// with the requested settings. +func (uvm *UtilityVM) ConfigureVMCPUGroup(ctx context.Context, opts *CPUGroupOptions) error { + if err := verifyCPUGroupOptions(opts); err != nil { + return err + } + if opts.CreateRandomID { + createdID, err := createNewCPUGroup(ctx, opts.LogicalProcessors) + if err != nil { + return err + } + opts.ID = createdID + } else { + exists, err := cpuGroupExists(ctx, opts.ID) + if err != nil { + return err + } + + if !exists { + if err := createNewCPUGroupWithID(ctx, opts.ID, opts.LogicalProcessors); err != nil { + return err + } + } + } + + if err := uvm.setCPUGroup(ctx, opts.ID); err != nil { + return err + } + + if opts.Cap != DefaultCPUGroupCap { + if err := setCPUGroupCap(ctx, uvm.cpuGroupID, opts.Cap); err != nil { + return err + } + } + + if opts.Priority != DefaultCPUGroupPriority { + if err := setCPUGroupSchedulingPriority(ctx, uvm.cpuGroupID, opts.Priority); err != nil { + return err + } + } + + return nil +} + +// setCPUGroup sets the VM's cpugroup +func (uvm *UtilityVM) setCPUGroup(ctx context.Context, id string) error { + req := &hcsschema.ModifySettingRequest{ + ResourcePath: cpuGroupResourcePath, + Settings: &hcsschema.CpuGroup{ + Id: id, + }, + } + if err := uvm.modify(ctx, req); err != nil { + return err + } + uvm.cpuGroupID = id + return nil +} + +// unsetCPUGroup sets the VM's cpugroup to the null group ID +// set groupID to 00000000-0000-0000-0000-000000000000 to remove the VM from a cpugroup +func (uvm *UtilityVM) unsetCPUGroup(ctx context.Context) error { + log.G(ctx).WithField("previous group id", uvm.cpuGroupID).Debug("unsetting the VM's CPU Group") + return uvm.setCPUGroup(ctx, CPUGroupNullID) +} + +// deleteCPUGroup deletes the cpugroup from the host +func deleteCPUGroup(ctx context.Context, id string) error { + operation := hcsschema.DeleteGroup + details := hcsschema.DeleteGroupOperation{ + GroupId: id, + } + + return modifyCPUGroupRequest(ctx, operation, details) +} + +// modifyCPUGroupRequest is a helper function for making modify calls to a cpugroup +func modifyCPUGroupRequest(ctx context.Context, operation hcsschema.CPUGroupOperation, details interface{}) error { + req := hcsschema.ModificationRequest{ + PropertyType: hcsschema.PTCPUGroup, + Settings: &hcsschema.HostProcessorModificationRequest{ + Operation: operation, + OperationDetails: details, + }, + } + + return hcs.ModifyServiceSettings(ctx, req) +} + +// createNewCPUGroup creates a new cpugroup on the host with a random id +func createNewCPUGroup(ctx context.Context, logicalProcessors []uint32) (string, error) { + id, err := guid.NewV4() + if err != nil { + return "", err + } + err = createNewCPUGroupWithID(ctx, id.String(), logicalProcessors) + if err != nil { + return "", err + } + return id.String(), nil +} + +// createNewCPUGroupWithID creates a new cpugroup on the host with a prespecified id +func createNewCPUGroupWithID(ctx context.Context, id string, logicalProcessors []uint32) error { + operation := hcsschema.CreateGroup + details := &hcsschema.CreateGroupOperation{ + GroupId: strings.ToLower(id), + LogicalProcessors: logicalProcessors, + LogicalProcessorCount: uint32(len(logicalProcessors)), + } + if err := modifyCPUGroupRequest(ctx, operation, details); err != nil { + return fmt.Errorf("failed to make cpugroups CreateGroup request with details %v with: %s", details, err) + } + return nil +} + +// setCPUGroupCap sets the cpugroup cap. +// Param `cap` must be an integer in the range [0, 65536]. A `cap` value of 32768 = 50% group cap. +func setCPUGroupCap(ctx context.Context, id string, cap uint32) error { + if cap > MaxCPUGroupCap { + return fmt.Errorf("cpugroup cap must be between [0, %d] inclusive, tried to use a cap of %d for group %v", MaxCPUGroupCap, cap, id) + } + + operation := hcsschema.SetProperty + details := hcsschema.SetPropertyOperation{ + GroupId: id, + PropertyCode: hcsschema.CpuCapPropertyCode, + PropertyValue: cap, + } + if err := modifyCPUGroupRequest(ctx, operation, details); err != nil { + return fmt.Errorf("failed to make cpugroups SetProperty request with details %v with: %s", details, err) + } + + return nil +} + +// setCPUGroupSchedulingPriority sets the cpugroup's scheduling priority +func setCPUGroupSchedulingPriority(ctx context.Context, id string, priority uint32) error { + operation := hcsschema.SetProperty + details := &hcsschema.SetPropertyOperation{ + GroupId: id, + PropertyCode: hcsschema.SchedulingPriorityPropertyCode, + PropertyValue: priority, + } + + if err := modifyCPUGroupRequest(ctx, operation, details); err != nil { + return fmt.Errorf("failed to make cpugroups SetProperty request with details %v with: %s", details, err) + } + + return nil +} + +// getHostCPUGroups queries the host for cpugroups and their properties. +func getHostCPUGroups(ctx context.Context) (*hcsschema.CpuGroupConfigurations, error) { + query := hcsschema.PropertyQuery{ + PropertyTypes: []hcsschema.PropertyType{hcsschema.PTCPUGroup}, + } + + cpuGroupsPresent, err := hcs.GetServiceProperties(ctx, query) + if err != nil { + return nil, err + } + + groupConfigs := &hcsschema.CpuGroupConfigurations{} + if err := json.Unmarshal(cpuGroupsPresent.Properties[0], groupConfigs); err != nil { + return nil, fmt.Errorf("failed to unmarshal host cpugroups: %v", err) + } + + return groupConfigs, nil +} + +// getCPUGroupConfig finds the cpugroup config information for group with `id` +func getCPUGroupConfig(ctx context.Context, id string) (*hcsschema.CpuGroupConfig, error) { + groupConfigs, err := getHostCPUGroups(ctx) + if err != nil { + return nil, err + } + for _, c := range groupConfigs.CpuGroups { + if strings.ToLower(c.GroupId) == strings.ToLower(id) { + return &c, nil + } + } + return nil, nil +} + +// cpuGroupExists is a helper fucntion to determine if cpugroup with `id` exists +// already on the host. +func cpuGroupExists(ctx context.Context, id string) (bool, error) { + groupConfig, err := getCPUGroupConfig(ctx, id) + if err != nil { + return false, err + } + + return groupConfig != nil, nil +} + +// getCPUGroupCap is a helper function to return the group cpu capacity of +// cpugroup with `id` +func getCPUGroupCap(ctx context.Context, id string) (uint32, error) { + config, err := getCPUGroupConfig(ctx, id) + if err != nil { + return 0, err + } + props := config.GroupProperties + for _, p := range props { + if p.PropertyCode == hcsschema.CpuCapPropertyCode { + return p.PropertyValue, nil + } + } + return 0, fmt.Errorf("failed to get cpu cap property information for cpugroup %s", id) +} + +// getCPUGroupPriority is a helper function to return the group scheduling priority of +// cpugroup with `id` +func getCPUGroupPriority(ctx context.Context, id string) (uint32, error) { + config, err := getCPUGroupConfig(ctx, id) + if err != nil { + return 0, err + } + props := config.GroupProperties + for _, p := range props { + if p.PropertyCode == hcsschema.SchedulingPriorityPropertyCode { + return p.PropertyValue, nil + } + } + return 0, fmt.Errorf("failed to get cpu priority property information for cpugroup %s", id) +} diff --git a/internal/uvm/cpugroups_test.go b/internal/uvm/cpugroups_test.go new file mode 100644 index 0000000000..80d5786889 --- /dev/null +++ b/internal/uvm/cpugroups_test.go @@ -0,0 +1,116 @@ +package uvm + +import ( + "context" + "testing" + + "github.com/Microsoft/go-winio/pkg/guid" +) + +// Unit tests for cpugroup creation, modification, and deletion +func DisabledTestCPUGroupCreateAndDelete(t *testing.T) { + lps := []uint32{0, 1} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + id, err := createNewCPUGroup(ctx, lps) + if err != nil { + t.Fatalf("failed to create cpugroup %s with: %v", id, err) + } + + defer func() { + if err := deleteCPUGroup(ctx, id); err != nil { + t.Fatalf("failed to delete cpugroup %s with: %v", id, err) + } + }() + + exists, err := cpuGroupExists(ctx, id) + if err != nil { + t.Fatalf("failed to determine if cpugroup exists with: %v", err) + } + if !exists { + t.Fatalf("expected to find cpugroup %s on machine but didn't", id) + } +} + +func DisabledTestCPUGroupCreateWithIDAndDelete(t *testing.T) { + lps := []uint32{0, 1} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + id, err := guid.NewV4() + if err != nil { + t.Fatalf("failed to create cpugroup guid with: %v", err) + } + err = createNewCPUGroupWithID(ctx, id.String(), lps) + if err != nil { + t.Fatalf("failed to create cpugroup %s with: %v", id.String(), err) + } + defer func() { + if err := deleteCPUGroup(ctx, id.String()); err != nil { + t.Fatalf("failed to delete cpugroup %s with: %v", id.String(), err) + } + }() + + exists, err := cpuGroupExists(ctx, id.String()) + if err != nil { + t.Fatalf("failed to determine if cpugroup exists with: %v", err) + } + if !exists { + t.Fatalf("expected to find cpugroup %s on machine but didn't", id.String()) + } +} + +func DisabledTestCPUGroupSetCap(t *testing.T) { + lps := []uint32{0, 1} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + id, err := createNewCPUGroup(ctx, lps) + if err != nil { + t.Fatalf("failed to create cpugroup %s with: %v", id, err) + } + + defer func() { + if err := deleteCPUGroup(ctx, id); err != nil { + t.Fatalf("failed to delete cpugroup %s with: %v", id, err) + } + }() + cap := uint32(32768) + if err := setCPUGroupCap(ctx, id, cap); err != nil { + t.Fatalf("failed to set cpugroup %s cap to %d with: %v", id, cap, err) + } + + actualCap, err := getCPUGroupCap(ctx, id) + if err != nil { + t.Fatalf("failed to get group cap with: %v", err) + } + if actualCap != cap { + t.Fatalf("expected to get a cpugroup cap of %d, instead got %d for group %s", cap, actualCap, id) + } +} + +func DisabledTestCPUGroupSetPriority(t *testing.T) { + lps := []uint32{0, 1} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + id, err := createNewCPUGroup(ctx, lps) + if err != nil { + t.Fatalf("failed to create cpugroup %s with: %v", id, err) + } + + defer func() { + if err := deleteCPUGroup(ctx, id); err != nil { + t.Fatalf("failed to delete cpugroup %s with: %v", id, err) + } + }() + priority := uint32(1) + if err := setCPUGroupSchedulingPriority(ctx, id, priority); err != nil { + t.Fatalf("failed to set cpugroup %s priority to %d with: %v", id, priority, err) + } + + actualPriority, err := getCPUGroupPriority(ctx, id) + if err != nil { + t.Fatalf("failed to get group priority value with: %v", err) + } + if actualPriority != priority { + t.Fatalf("expected to get cpugroup priority of %d, instead got %d for cpugroup %s", priority, actualPriority, id) + } +} diff --git a/internal/uvm/create.go b/internal/uvm/create.go index 8993e5dd38..9e4f92ff54 100644 --- a/internal/uvm/create.go +++ b/internal/uvm/create.go @@ -185,6 +185,7 @@ func (uvm *UtilityVM) Close() (err error) { windows.Close(uvm.vmmemProcess) if uvm.hcsSystem != nil { + uvm.ReleaseCPUGroup(ctx) uvm.hcsSystem.Terminate(ctx) uvm.Wait() } diff --git a/internal/uvm/create_lcow.go b/internal/uvm/create_lcow.go index 15cfa1c147..3701ad9f13 100644 --- a/internal/uvm/create_lcow.go +++ b/internal/uvm/create_lcow.go @@ -187,7 +187,7 @@ func CreateLCOW(ctx context.Context, opts *OptionsLCOW) (_ *UtilityVM, err error return nil, errors.Wrap(err, errBadUVMOpts.Error()) } - processorTopology, err := hostProcessorInfo(ctx) + processorTopology, err := HostProcessorInfo(ctx) if err != nil { return nil, fmt.Errorf("failed to get host processor information: %s", err) } diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index d98819575f..f89761ba30 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -112,7 +112,7 @@ func CreateWCOW(ctx context.Context, opts *OptionsWCOW) (_ *UtilityVM, err error } } - processorTopology, err := hostProcessorInfo(ctx) + processorTopology, err := HostProcessorInfo(ctx) if err != nil { return nil, fmt.Errorf("failed to get host processor information: %s", err) } diff --git a/internal/uvm/host_information.go b/internal/uvm/host_information.go index 9f24ad2e14..75a16ec01a 100644 --- a/internal/uvm/host_information.go +++ b/internal/uvm/host_information.go @@ -10,10 +10,10 @@ import ( hcsschema "github.com/Microsoft/hcsshim/internal/schema2" ) -// hostProcessorInfo queries HCS for the UVM hosts processor information, including topology +// HostProcessorInfo queries HCS for the UVM hosts processor information, including topology // and NUMA configuration. This is also used to reliably get the hosts number of logical // processors in multi processor group settings. -func hostProcessorInfo(ctx context.Context) (*hcsschema.ProcessorTopology, error) { +func HostProcessorInfo(ctx context.Context) (*hcsschema.ProcessorTopology, error) { q := hcsschema.PropertyQuery{ PropertyTypes: []hcsschema.PropertyType{hcsschema.PTProcessorTopology}, } diff --git a/internal/uvm/resourcepaths.go b/internal/uvm/resourcepaths.go index 05abb726f6..c0af640d74 100644 --- a/internal/uvm/resourcepaths.go +++ b/internal/uvm/resourcepaths.go @@ -3,7 +3,7 @@ package uvm const ( gpuResourcePath string = "VirtualMachine/ComputeTopology/Gpu" memoryResourcePath string = "VirtualMachine/ComputeTopology/Memory/SizeInMB" - cpuGroupResourceFormat string = "VirtualMachine/ComputeTopology/Processor/CpuGroup/%s" + cpuGroupResourcePath string = "VirtualMachine/ComputeTopology/Processor/CpuGroup" idledResourcePath string = "VirtualMachine/ComputeTopology/Processor/IdledProcessors" cpuFrequencyPowerCapResourcePath string = "VirtualMachine/ComputeTopology/Processor/CpuFrequencyPowerCap" serialResourceFormat string = "VirtualMachine/Devices/ComPorts/%d" diff --git a/internal/uvm/types.go b/internal/uvm/types.go index abc74f457b..5fcb323ba6 100644 --- a/internal/uvm/types.go +++ b/internal/uvm/types.go @@ -113,4 +113,7 @@ type UtilityVM struct { // This is used in generating unique mount path inside UVM for every mount. // Access to this variable should be done atomically. mountCounter uint64 + + // cpuGroupID is the ID of the cpugroup on the host that this UVM is assigned to + cpuGroupID string } diff --git a/test/cri-containerd/runpodsandbox_test.go b/test/cri-containerd/runpodsandbox_test.go index d9eab6260e..95cec0cbef 100644 --- a/test/cri-containerd/runpodsandbox_test.go +++ b/test/cri-containerd/runpodsandbox_test.go @@ -1137,6 +1137,150 @@ func Test_RunPodSandbox_Mount_SandboxDir_LCOW(t *testing.T) { //TODO: Parse the output of the exec command to make sure the uvm mount was successful } +func Test_RunPodSandbox_CPUGroup_LCOW(t *testing.T) { + requireFeatures(t, featureLCOW) + + pullRequiredLcowImages(t, []string{imageLcowK8sPause}) + + annotations := []map[string]string{ + { + "io.microsoft.virtualmachine.cpugroup.id": "FA22A12C-36B3-486D-A3E9-BC526C2B450B", + // we believe it is reasonable to assume our test machines will have at least two LPs, otherwise + // this test will fail. + "io.microsoft.virtualmachine.cpugroup.logicalprocessors": "0,1", + "io.microsoft.virtualmachine.cpugroup.cap": "32768", + }, + { + "io.microsoft.virtualmachine.cpugroup.randomid": "true", + // we believe it is reasonable to assume our test machines will have at least two LPs, otherwise + // this test will fail. + "io.microsoft.virtualmachine.cpugroup.logicalprocessors": "0,1", + "io.microsoft.virtualmachine.cpugroup.cap": "32768", + }, + { + "io.microsoft.virtualmachine.cpugroup.randomid": "true", + }, + { + "io.microsoft.virtualmachine.cpugroup.id": "FA22A12C-36B3-486D-A3E9-BC526C2B450B", + }, + } + + for _, a := range annotations { + request := &runtime.RunPodSandboxRequest{ + Config: &runtime.PodSandboxConfig{ + Metadata: &runtime.PodSandboxMetadata{ + Name: t.Name(), + Uid: "0", + Namespace: testNamespace, + }, + Annotations: a, + }, + RuntimeHandler: lcowRuntimeHandler, + } + runPodSandboxTest(t, request) + } +} + +func Test_RunPodSandbox_CPUGroupSchedulingPriority_LCOW(t *testing.T) { + requireFeatures(t, featureLCOW) + + // TODO(katiewasnothere): update build version when this support is + // in an official release + if osversion.Get().Build < 20196 { + t.Skip("Requires build +20196") + } + + pullRequiredLcowImages(t, []string{imageLcowK8sPause}) + + request := &runtime.RunPodSandboxRequest{ + Config: &runtime.PodSandboxConfig{ + Metadata: &runtime.PodSandboxMetadata{ + Name: t.Name(), + Uid: "0", + Namespace: testNamespace, + }, + Annotations: map[string]string{ + "io.microsoft.virtualmachine.cpugroup.randomid": "true", + "io.microsoft.virtualmachine.cpugroup.priority": "2", + }, + }, + RuntimeHandler: lcowRuntimeHandler, + } + runPodSandboxTest(t, request) +} + +func Test_RunPodSandbox_CPUGroup_WCOW_Hypervisor(t *testing.T) { + requireFeatures(t, featureWCOWHypervisor) + + pullRequiredImages(t, []string{imageWindowsNanoserver}) + + annotations := []map[string]string{ + { + "io.microsoft.virtualmachine.cpugroup.id": "FA22A12C-36B3-486D-A3E9-BC526C2B450B", + // we believe it is reasonable to assume our test machines will have at least two LPs, otherwise + // this test will fail. + "io.microsoft.virtualmachine.cpugroup.logicalprocessors": "0,1", + "io.microsoft.virtualmachine.cpugroup.cap": "32768", + }, + { + "io.microsoft.virtualmachine.cpugroup.randomid": "true", + // we believe it is reasonable to assume our test machines will have at least two LPs, otherwise + // this test will fail. + "io.microsoft.virtualmachine.cpugroup.logicalprocessors": "0,1", + "io.microsoft.virtualmachine.cpugroup.cap": "32768", + }, + { + "io.microsoft.virtualmachine.cpugroup.randomid": "true", + }, + { + "io.microsoft.virtualmachine.cpugroup.id": "FA22A12C-36B3-486D-A3E9-BC526C2B450B", + }, + } + + for _, a := range annotations { + request := &runtime.RunPodSandboxRequest{ + Config: &runtime.PodSandboxConfig{ + Metadata: &runtime.PodSandboxMetadata{ + Name: t.Name(), + Uid: "0", + Namespace: testNamespace, + }, + Annotations: a, + }, + RuntimeHandler: wcowHypervisorRuntimeHandler, + } + runPodSandboxTest(t, request) + } +} + +func Test_RunPodSandbox_CPUGroupSchedulingPriority_WCOW_Hypervisor(t *testing.T) { + requireFeatures(t, featureWCOWHypervisor) + + // TODO(katiewasnothere): update build version when this support is + // in an official release + if osversion.Get().Build < 20196 { + t.Skip("Requires build +20196") + } + + pullRequiredImages(t, []string{imageWindowsNanoserver}) + + request := &runtime.RunPodSandboxRequest{ + Config: &runtime.PodSandboxConfig{ + Metadata: &runtime.PodSandboxMetadata{ + Name: t.Name(), + Uid: "0", + Namespace: testNamespace, + }, + Annotations: map[string]string{ + "io.microsoft.virtualmachine.cpugroup.randomid": "true", + "io.microsoft.virtualmachine.cpugroup.priority": "2", + }, + }, + RuntimeHandler: wcowHypervisorRuntimeHandler, + } + runPodSandboxTest(t, request) +} + func createExt4VHD(ctx context.Context, t *testing.T, path string) { uvm := testutilities.CreateLCOWUVM(ctx, t, t.Name()+"-createExt4VHD") defer uvm.Close() diff --git a/test/go.sum b/test/go.sum index 13975bdb09..afdb86004a 100644 --- a/test/go.sum +++ b/test/go.sum @@ -240,3 +240,4 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh k8s.io/cri-api v0.17.3 h1:jvjVvBqgZq3WcaPq07n0h5h9eCnIaR4dhKyHSoZG8Y8= k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.18.6 h1:dxhb+Ii0qThCgl3ZR+LO3wAy8RVzvppYVtyLOUC0fyI= +k8s.io/cri-api v0.18.8 h1:vdp2ExzqMhOaMQpMVpX7z4KTbqLodXMAnm6hVMCrsU8=