Support assigning cpugroup immediately after UVM start

internal/cpugroup/cpugroup.go

@@ -0,0 +1,76 @@
+package cpugroup
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/Microsoft/hcsshim/internal/hcs"
+	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
+)
+
+const NullGroupID = "00000000-0000-0000-0000-000000000000"
+
+// ErrHVStatusInvalidCPUGroupState corresponds to the internal error code for HV_STATUS_INVALID_CPU_GROUP_STATE
+var ErrHVStatusInvalidCPUGroupState = errors.New("The hypervisor could not perform the operation because the CPU group is entering or in an invalid state.")
+
+// Delete deletes the cpugroup from the host
+func Delete(ctx context.Context, id string) error {
+	operation := hcsschema.DeleteGroup
+	details := hcsschema.DeleteGroupOperation{
+		GroupId: id,
+	}
+
+	return modifyCPUGroupRequest(ctx, operation, details)
+}
+
+// modifyCPUGroupRequest is a helper function for making modify calls to a cpugroup
+func modifyCPUGroupRequest(ctx context.Context, operation hcsschema.CPUGroupOperation, details interface{}) error {
+	req := hcsschema.ModificationRequest{
+		PropertyType: hcsschema.PTCPUGroup,
+		Settings: &hcsschema.HostProcessorModificationRequest{
+			Operation:        operation,
+			OperationDetails: details,
+		},
+	}
+
+	return hcs.ModifyServiceSettings(ctx, req)
+}
+
+// Create creates a new cpugroup on the host with a prespecified id
+func Create(ctx context.Context, id string, logicalProcessors []uint32) error {
+	operation := hcsschema.CreateGroup
+	details := &hcsschema.CreateGroupOperation{
+		GroupId:               strings.ToLower(id),
+		LogicalProcessors:     logicalProcessors,
+		LogicalProcessorCount: uint32(len(logicalProcessors)),
+	}
+	if err := modifyCPUGroupRequest(ctx, operation, details); err != nil {
+		return fmt.Errorf("failed to make cpugroups CreateGroup request for details %+v with: %s", details, err)
+	}
+	return nil
+}
+
+// getCPUGroupConfig finds the cpugroup config information for group with `id`
+func getCPUGroupConfig(ctx context.Context, id string) (*hcsschema.CpuGroupConfig, error) {
+	query := hcsschema.PropertyQuery{
+		PropertyTypes: []hcsschema.PropertyType{hcsschema.PTCPUGroup},
+	}
+	cpuGroupsPresent, err := hcs.GetServiceProperties(ctx, query)
+	if err != nil {
+		return nil, err
+	}
+	groupConfigs := &hcsschema.CpuGroupConfigurations{}
+	if err := json.Unmarshal(cpuGroupsPresent.Properties[0], groupConfigs); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal host cpugroups: %v", err)
+	}
+
+	for _, c := range groupConfigs.CpuGroups {
+		if strings.ToLower(c.GroupId) == strings.ToLower(id) {
+			return &c, nil
+		}
+	}
+	return nil, fmt.Errorf("no cpugroup exists with id %v", id)
+}

internal/cpugroup/cpugroup_test.go

@@ -0,0 +1,35 @@
+package cpugroup
+
+import (
+	"context"
+	"testing"
+
+	"github.com/Microsoft/go-winio/pkg/guid"
+)
+
+// Unit tests for creating and deleting a CPU group on the host
+func TestCPUGroupCreateWithIDAndDelete(t *testing.T) {
+	t.Skip("only works on classic/core scheduler, skipping as we can't check this dynamically right now")
+
+	lps := []uint32{0, 1}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	id, err := guid.NewV4()
+	if err != nil {
+		t.Fatalf("failed to create cpugroup guid with: %v", err)
+	}
+	err = Create(ctx, id.String(), lps)
+	if err != nil {
+		t.Fatalf("failed to create cpugroup %s with: %v", id.String(), err)
+	}
+	defer func() {
+		if err := Delete(ctx, id.String()); err != nil {
+			t.Fatalf("failed to delete cpugroup %s with: %v", id.String(), err)
+		}
+	}()
+
+	_, err = getCPUGroupConfig(ctx, id.String())
+	if err != nil {
+		t.Fatalf("failed to find cpugroup on host with: %v", err)
+	}
+}

internal/oci/uvm.go

@@ -135,6 +135,9 @@ const (
 	// HCS-GCS bridge. Default value is true which means external bridge will be used
 	// by default.
 	annotationUseExternalGCSBridge = "io.microsoft.virtualmachine.useexternalgcsbridge"
+
+	// annotation used to specify the cpugroup ID that a UVM should be assigned to
+	annotationCPUGroupID = "io.microsoft.virtualmachine.cpugroup.id"
 )
 
 // parseAnnotationsBool searches `a` for `key` and if found verifies that the
@@ -391,6 +394,7 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) (
 		lopts.VPCIEnabled = parseAnnotationsBool(ctx, s.Annotations, annotationVPCIEnabled, lopts.VPCIEnabled)
 		lopts.BootFilesPath = parseAnnotationsString(s.Annotations, annotationBootFilesRootPath, lopts.BootFilesPath)
 		lopts.ExternalGuestConnection = parseAnnotationsBool(ctx, s.Annotations, annotationUseExternalGCSBridge, lopts.ExternalGuestConnection)
+		lopts.CPUGroupID = parseAnnotationsString(s.Annotations, annotationCPUGroupID, lopts.CPUGroupID)
 		handleAnnotationPreferredRootFSType(ctx, s.Annotations, lopts)
 		handleAnnotationKernelDirectBoot(ctx, s.Annotations, lopts)
 
@@ -413,6 +417,7 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) (
 		wopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotationStorageQoSIopsMaximum, wopts.StorageQoSIopsMaximum)
 		wopts.ExternalGuestConnection = parseAnnotationsBool(ctx, s.Annotations, annotationUseExternalGCSBridge, wopts.ExternalGuestConnection)
 		wopts.DisableCompartmentNamespace = parseAnnotationsBool(ctx, s.Annotations, annotationDisableCompartmentNamespace, wopts.DisableCompartmentNamespace)
+		wopts.CPUGroupID = parseAnnotationsString(s.Annotations, annotationCPUGroupID, wopts.CPUGroupID)
 		handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, wopts)
 		return wopts, nil
 	}

internal/uvm/host_information.go → internal/processorinfo/host_information.go

@@ -1,4 +1,4 @@
-package uvm
+package processorinfo
 
 import (
 	"context"
@@ -10,10 +10,10 @@ import (
 	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
 )
 
-// hostProcessorInfo queries HCS for the UVM hosts processor information, including topology
+// HostProcessorInfo queries HCS for the host's processor information, including topology
 // and NUMA configuration. This is also used to reliably get the hosts number of logical
 // processors in multi processor group settings.
-func hostProcessorInfo(ctx context.Context) (*hcsschema.ProcessorTopology, error) {
+func HostProcessorInfo(ctx context.Context) (*hcsschema.ProcessorTopology, error) {
 	q := hcsschema.PropertyQuery{
 		PropertyTypes: []hcsschema.PropertyType{hcsschema.PTProcessorTopology},
 	}

internal/schema2/cpu_group.go

@@ -0,0 +1,15 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+// CPU groups allow Hyper-V administrators to better manage and allocate the host's CPU resources across guest virtual machines
+type CpuGroup struct {
+	Id string `json:"Id,omitempty"`
+}

internal/schema2/cpu_group_affinity.go

@@ -0,0 +1,15 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type CpuGroupAffinity struct {
+	LogicalProcessorCount int32   `json:"LogicalProcessorCount,omitempty"`
+	LogicalProcessors     []int32 `json:"LogicalProcessors,omitempty"`
+}

internal/schema2/cpu_group_config.go

@@ -0,0 +1,18 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type CpuGroupConfig struct {
+	GroupId         string             `json:"GroupId,omitempty"`
+	Affinity        *CpuGroupAffinity  `json:"Affinity,omitempty"`
+	GroupProperties []CpuGroupProperty `json:"GroupProperties,omitempty"`
+	// Hypervisor CPU group IDs exposed to clients
+	HypervisorGroupId int32 `json:"HypervisorGroupId,omitempty"`
+}

internal/schema2/cpu_group_configurations.go

@@ -0,0 +1,15 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+// Structure used to return cpu groups for a Service property query
+type CpuGroupConfigurations struct {
+	CpuGroups []CpuGroupConfig `json:"CpuGroups,omitempty"`
+}

internal/schema2/cpu_group_operations.go

@@ -0,0 +1,18 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type CPUGroupOperation string
+
+const (
+	CreateGroup CPUGroupOperation = "CreateGroup"
+	DeleteGroup CPUGroupOperation = "DeleteGroup"
+	SetProperty CPUGroupOperation = "SetProperty"
+)

internal/schema2/cpu_group_property.go

@@ -0,0 +1,15 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type CpuGroupProperty struct {
+	PropertyCode  uint32 `json:"PropertyCode,omitempty"`
+	PropertyValue uint32 `json:"PropertyValue,omitempty"`
+}

internal/schema2/create_group_operation.go

@@ -0,0 +1,17 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+// Create group operation settings
+type CreateGroupOperation struct {
+	GroupId               string   `json:"GroupId,omitempty"`
+	LogicalProcessorCount uint32   `json:"LogicalProcessorCount,omitempty"`
+	LogicalProcessors     []uint32 `json:"LogicalProcessors,omitempty"`
+}

internal/schema2/delete_group_operation.go

@@ -0,0 +1,15 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+// Delete group operation settings
+type DeleteGroupOperation struct {
+	GroupId string `json:"GroupId,omitempty"`
+}

internal/schema2/host_processor_modify_request.go

@@ -0,0 +1,16 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+// Structure used to request a service processor modification
+type HostProcessorModificationRequest struct {
+	Operation        CPUGroupOperation `json:"Operation,omitempty"`
+	OperationDetails interface{}       `json:"OperationDetails,omitempty"`
+}

internal/schema2/property_type.go

@@ -22,4 +22,5 @@ const (
 	PTGuestConnection             PropertyType = "GuestConnection"
 	PTICHeartbeatStatus           PropertyType = "ICHeartbeatStatus"
 	PTProcessorTopology           PropertyType = "ProcessorTopology"
+	PTCPUGroup                    PropertyType = "CpuGroup"
 )

internal/uvm/cpugroups.go

@@ -0,0 +1,55 @@
+package uvm
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/Microsoft/hcsshim/internal/cpugroup"
+	hcsschema "github.com/Microsoft/hcsshim/internal/schema2"
+)
+
+// ReleaseCPUGroup unsets the cpugroup from the VM
+func (uvm *UtilityVM) ReleaseCPUGroup(ctx context.Context) error {
+	groupID := uvm.cpuGroupID
+	if groupID == "" {
+		// not set, don't try to do anything
+		return nil
+	}
+	if err := uvm.unsetCPUGroup(ctx); err != nil {
+		return fmt.Errorf("failed to remove VM %s from cpugroup %s", uvm.ID(), groupID)
+	}
+	return nil
+}
+
+// SetCPUGroup setups up the cpugroup for the VM with the requested id
+func (uvm *UtilityVM) SetCPUGroup(ctx context.Context, id string) error {
+	if id == "" {
+		return errors.New("must specify an ID to use when configuring a VM's cpugroup")
+	}
+	return uvm.setCPUGroup(ctx, id)
+}
+
+// setCPUGroup sets the VM's cpugroup
+func (uvm *UtilityVM) setCPUGroup(ctx context.Context, id string) error {
+	req := &hcsschema.ModifySettingRequest{
+		ResourcePath: cpuGroupResourcePath,
+		Settings: &hcsschema.CpuGroup{
+			Id: id,
+		},
+	}
+	if err := uvm.modify(ctx, req); err != nil {
+		return err
+	}
+	uvm.cpuGroupID = id
+	return nil
+}
+
+// unsetCPUGroup sets the VM's cpugroup to the null group ID
+// set groupID to 00000000-0000-0000-0000-000000000000 to remove the VM from a cpugroup
+//
+// Since a VM must be moved to the null group before potentially being added to a different
+// cpugroup, that means there may be a segment of time that the VM's cpu usage runs unrestricted.
+func (uvm *UtilityVM) unsetCPUGroup(ctx context.Context) error {
+	return uvm.setCPUGroup(ctx, cpugroup.NullGroupID)
+}

internal/uvm/create.go

@@ -81,6 +81,10 @@ type Options struct {
 	// unneccessary behavior and once this restriction is removed then we can remove the need for this variable
 	// and the associated annotation as well.
 	DisableCompartmentNamespace bool
+
+	// CPUGroupID set the ID of a CPUGroup on the host that the UVM should be added to on start.
+	// Defaults to an empty string which indicates the UVM should not be added to any CPUGroup.
+	CPUGroupID string
 }
 
 // Verifies that the final UVM options are correct and supported.
@@ -196,6 +200,9 @@ func (uvm *UtilityVM) Close() (err error) {
 	windows.Close(uvm.vmmemProcess)
 
 	if uvm.hcsSystem != nil {
+		if err := uvm.ReleaseCPUGroup(ctx); err != nil {
+			log.G(ctx).WithError(err).Warn("failed to release VM resource")
+		}
 		uvm.hcsSystem.Terminate(ctx)
 		uvm.Wait()
 	}