diff --git a/internal/uvm/virtual_device.go b/internal/uvm/virtual_device.go index 66d71e08d0..679941997f 100644 --- a/internal/uvm/virtual_device.go +++ b/internal/uvm/virtual_device.go @@ -95,7 +95,8 @@ func (uvm *UtilityVM) AssignDevice(ctx context.Context, deviceID string) (*VPCID request := &hcsschema.ModifySettingRequest{ ResourcePath: fmt.Sprintf(resourcepaths.VirtualPCIResourceFormat, vmBusGUID), RequestType: requesttype.Add, - Settings: targetDevice} + Settings: targetDevice, + } // WCOW (when supported) does not require a guest request as part of the // device assignment diff --git a/internal/vm/builder.go b/internal/vm/builder.go new file mode 100644 index 0000000000..4be3eee296 --- /dev/null +++ b/internal/vm/builder.go @@ -0,0 +1,74 @@ +package vm + +import ( + "context" +) + +type UVMBuilder interface { + // Create will create the Utility VM in a paused/powered off state with whatever is present in the implementation + // of the interfaces config at the time of the call. + Create(ctx context.Context) (UVM, error) +} + +type MemoryBackingType uint8 + +const ( + MemoryBackingTypeVirtual MemoryBackingType = iota + MemoryBackingTypePhysical +) + +// MemoryConfig holds the memory options that should be configurable for a Utility VM. +type MemoryConfig struct { + BackingType MemoryBackingType + DeferredCommit bool + HotHint bool + ColdHint bool + ColdDiscardHint bool +} + +// MemoryManager handles setting and managing memory configurations for the Utility VM. +type MemoryManager interface { + // SetMemoryLimit sets the amount of memory in megabytes that the Utility VM will be assigned. + SetMemoryLimit(memoryMB uint64) error + // SetMemoryConfig sets an array of different memory configuration options available. This includes things like the + // type of memory to back the VM (virtual/physical). + SetMemoryConfig(config *MemoryConfig) error + // SetMMIOConfig sets memory mapped IO configurations for the Utility VM. + SetMMIOConfig(lowGapMB uint64, highBaseMB uint64, highGapMB uint64) error +} + +// ProcessorManager handles setting and managing processor configurations for the Utility VM. +type ProcessorManager interface { + // SetProcessorCount sets the number of virtual processors that will be assigned to the Utility VM. + SetProcessorCount(count uint32) error +} + +// SerialManager manages setting up serial consoles for the Utility VM. +type SerialManager interface { + // SetSerialConsole sets up a serial console for `port`. Output will be relayed to the listener specified + // by `listenerPath`. For HCS `listenerPath` this is expected to be a path to a named pipe. + SetSerialConsole(port uint32, listenerPath string) error +} + +// BootManager manages boot configurations for the Utility VM. +type BootManager interface { + // SetUEFIBoot sets UEFI configurations for booting a Utility VM. + SetUEFIBoot(dir string, path string, args string) error + // SetLinuxKernelDirectBoot sets Linux direct boot configurations for booting a Utility VM. + SetLinuxKernelDirectBoot(kernel string, initRD string, cmd string) error +} + +// StorageQosManager manages setting storage limits on the Utility VM. +type StorageQosManager interface { + // SetStorageQos sets storage related options for the Utility VM + SetStorageQos(iopsMaximum int64, bandwidthMaximum int64) error +} + +// WindowsConfigManager manages options specific to a Windows host (cpugroups etc.) +type WindowsConfigManager interface { + // SetCPUGroup sets the CPU group that the Utility VM will belong to on a Windows host. + SetCPUGroup(id string) error +} + +// LinuxConfigManager manages options specific to a Linux host. +type LinuxConfigManager interface{} diff --git a/internal/vm/hcs/boot.go b/internal/vm/hcs/boot.go new file mode 100644 index 0000000000..425feebd2c --- /dev/null +++ b/internal/vm/hcs/boot.go @@ -0,0 +1,31 @@ +package hcs + +import ( + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/osversion" + "github.com/pkg/errors" +) + +func (uvmb *utilityVMBuilder) SetUEFIBoot(dir string, path string, args string) error { + uvmb.doc.VirtualMachine.Chipset.Uefi = &hcsschema.Uefi{ + BootThis: &hcsschema.UefiBootEntry{ + DevicePath: path, + DeviceType: "VmbFs", + VmbFsRootPath: dir, + OptionalData: args, + }, + } + return nil +} + +func (uvmb *utilityVMBuilder) SetLinuxKernelDirectBoot(kernel string, initRD string, cmd string) error { + if osversion.Get().Build < 18286 { + return errors.New("Linux kernel direct boot requires at least Windows version 18286") + } + uvmb.doc.VirtualMachine.Chipset.LinuxKernelDirect = &hcsschema.LinuxKernelDirect{ + KernelFilePath: kernel, + InitRdPath: initRD, + KernelCmdLine: cmd, + } + return nil +} diff --git a/internal/vm/hcs/builder.go b/internal/vm/hcs/builder.go new file mode 100644 index 0000000000..f37a0f08b8 --- /dev/null +++ b/internal/vm/hcs/builder.go @@ -0,0 +1,93 @@ +package hcs + +import ( + "context" + + "github.com/Microsoft/hcsshim/internal/hcs" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/schemaversion" + "github.com/Microsoft/hcsshim/internal/vm" + "github.com/pkg/errors" +) + +type utilityVMBuilder struct { + id string + guestOS vm.GuestOS + doc *hcsschema.ComputeSystem +} + +func NewUVMBuilder(id string, owner string, guestOS vm.GuestOS) (vm.UVMBuilder, error) { + doc := &hcsschema.ComputeSystem{ + Owner: owner, + SchemaVersion: schemaversion.SchemaV21(), + ShouldTerminateOnLastHandleClosed: true, + VirtualMachine: &hcsschema.VirtualMachine{ + StopOnReset: true, + Chipset: &hcsschema.Chipset{}, + ComputeTopology: &hcsschema.Topology{ + Memory: &hcsschema.Memory2{ + AllowOvercommit: true, + }, + Processor: &hcsschema.Processor2{}, + }, + Devices: &hcsschema.Devices{ + HvSocket: &hcsschema.HvSocket2{ + HvSocketConfig: &hcsschema.HvSocketSystemConfig{ + // Allow administrators and SYSTEM to bind to vsock sockets + // so that we can create a GCS log socket. + DefaultBindSecurityDescriptor: "D:P(A;;FA;;;SY)(A;;FA;;;BA)", + }, + }, + }, + }, + } + + switch guestOS { + case vm.Windows: + doc.VirtualMachine.Devices.VirtualSmb = &hcsschema.VirtualSmb{} + case vm.Linux: + doc.VirtualMachine.Devices.Plan9 = &hcsschema.Plan9{} + default: + return nil, vm.ErrUnknownGuestOS + } + + return &utilityVMBuilder{ + id: id, + guestOS: guestOS, + doc: doc, + }, nil +} + +func (uvmb *utilityVMBuilder) Create(ctx context.Context) (_ vm.UVM, err error) { + cs, err := hcs.CreateComputeSystem(ctx, uvmb.id, uvmb.doc) + if err != nil { + return nil, errors.Wrap(err, "failed to create hcs compute system") + } + + defer func() { + if err != nil { + _ = cs.Terminate(ctx) + _ = cs.Wait() + } + }() + + backingType := vm.MemoryBackingTypeVirtual + if !uvmb.doc.VirtualMachine.ComputeTopology.Memory.AllowOvercommit { + backingType = vm.MemoryBackingTypePhysical + } + + uvm := &utilityVM{ + id: uvmb.id, + guestOS: uvmb.guestOS, + cs: cs, + backingType: backingType, + state: vm.StateCreated, + } + + properties, err := cs.Properties(ctx) + if err != nil { + return nil, err + } + uvm.vmID = properties.RuntimeID + return uvm, nil +} diff --git a/internal/vm/hcs/hcs.go b/internal/vm/hcs/hcs.go new file mode 100644 index 0000000000..67c44a95d9 --- /dev/null +++ b/internal/vm/hcs/hcs.go @@ -0,0 +1,75 @@ +package hcs + +import ( + "context" + "sync" + + "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/hcsshim/internal/hcs" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/vm" + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +type utilityVM struct { + id string + state vm.State + guestOS vm.GuestOS + cs *hcs.System + backingType vm.MemoryBackingType + vmmemProcess windows.Handle + vmmemErr error + vmmemOnce sync.Once + vmID guid.GUID +} + +func (uvm *utilityVM) ID() string { + return uvm.id +} + +func (uvm *utilityVM) Start(ctx context.Context) (err error) { + if err := uvm.cs.Start(ctx); err != nil { + return errors.Wrap(err, "failed to start utility VM") + } + return nil +} + +func (uvm *utilityVM) Stop(ctx context.Context) error { + if err := uvm.cs.Terminate(ctx); err != nil { + return errors.Wrap(err, "failed to terminate utility VM") + } + return nil +} + +func (uvm *utilityVM) Pause(ctx context.Context) error { + if err := uvm.cs.Pause(ctx); err != nil { + return errors.Wrap(err, "failed to pause utility VM") + } + return nil +} + +func (uvm *utilityVM) Resume(ctx context.Context) error { + if err := uvm.cs.Resume(ctx); err != nil { + return errors.Wrap(err, "failed to resume utility VM") + } + return nil +} + +func (uvm *utilityVM) Save(ctx context.Context) error { + saveOptions := hcsschema.SaveOptions{ + SaveType: "AsTemplate", + } + if err := uvm.cs.Save(ctx, saveOptions); err != nil { + return errors.Wrap(err, "failed to save utility VM state") + } + return nil +} + +func (uvm *utilityVM) Wait() error { + return uvm.cs.Wait() +} + +func (uvm *utilityVM) ExitError() error { + return uvm.cs.ExitError() +} diff --git a/internal/vm/hcs/memory.go b/internal/vm/hcs/memory.go new file mode 100644 index 0000000000..afb9932628 --- /dev/null +++ b/internal/vm/hcs/memory.go @@ -0,0 +1,28 @@ +package hcs + +import ( + "github.com/Microsoft/hcsshim/internal/vm" +) + +func (uvmb *utilityVMBuilder) SetMemoryLimit(memoryMB uint64) error { + uvmb.doc.VirtualMachine.ComputeTopology.Memory.SizeInMB = memoryMB + return nil +} + +func (uvmb *utilityVMBuilder) SetMemoryConfig(config *vm.MemoryConfig) error { + memory := uvmb.doc.VirtualMachine.ComputeTopology.Memory + memory.AllowOvercommit = config.BackingType == vm.MemoryBackingTypeVirtual + memory.EnableDeferredCommit = config.DeferredCommit + memory.EnableHotHint = config.HotHint + memory.EnableColdHint = config.ColdHint + memory.EnableColdDiscardHint = config.ColdDiscardHint + return nil +} + +func (uvmb *utilityVMBuilder) SetMMIOConfig(lowGapMB uint64, highBaseMB uint64, highGapMB uint64) error { + memory := uvmb.doc.VirtualMachine.ComputeTopology.Memory + memory.LowMMIOGapInMB = lowGapMB + memory.HighMMIOBaseInMB = highBaseMB + memory.HighMMIOGapInMB = highGapMB + return nil +} diff --git a/internal/vm/hcs/network.go b/internal/vm/hcs/network.go new file mode 100644 index 0000000000..6610fffd39 --- /dev/null +++ b/internal/vm/hcs/network.go @@ -0,0 +1,34 @@ +package hcs + +import ( + "context" + "fmt" + + "github.com/Microsoft/hcsshim/internal/hcs/resourcepaths" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/requesttype" +) + +func (uvm *utilityVM) AddNIC(ctx context.Context, nicID, endpointID, macAddr string) error { + request := hcsschema.ModifySettingRequest{ + RequestType: requesttype.Add, + ResourcePath: fmt.Sprintf(resourcepaths.NetworkResourceFormat, nicID), + Settings: hcsschema.NetworkAdapter{ + EndpointId: endpointID, + MacAddress: macAddr, + }, + } + return uvm.cs.Modify(ctx, request) +} + +func (uvm *utilityVM) RemoveNIC(ctx context.Context, nicID, endpointID, macAddr string) error { + request := hcsschema.ModifySettingRequest{ + RequestType: requesttype.Remove, + ResourcePath: fmt.Sprintf(resourcepaths.NetworkResourceFormat, nicID), + Settings: hcsschema.NetworkAdapter{ + EndpointId: endpointID, + MacAddress: macAddr, + }, + } + return uvm.cs.Modify(ctx, request) +} diff --git a/internal/vm/hcs/pci.go b/internal/vm/hcs/pci.go new file mode 100644 index 0000000000..262823768f --- /dev/null +++ b/internal/vm/hcs/pci.go @@ -0,0 +1,33 @@ +package hcs + +import ( + "context" + "fmt" + + "github.com/Microsoft/hcsshim/internal/hcs/resourcepaths" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/requesttype" +) + +func (uvm *utilityVM) AddDevice(ctx context.Context, instanceID, vmbusGUID string) error { + request := &hcsschema.ModifySettingRequest{ + ResourcePath: fmt.Sprintf(resourcepaths.VirtualPCIResourceFormat, vmbusGUID), + RequestType: requesttype.Add, + Settings: hcsschema.VirtualPciDevice{ + Functions: []hcsschema.VirtualPciFunction{ + { + DeviceInstancePath: instanceID, + }, + }, + }, + } + return uvm.cs.Modify(ctx, request) +} + +func (uvm *utilityVM) RemoveDevice(ctx context.Context, instanceID, vmbusGUID string) error { + request := &hcsschema.ModifySettingRequest{ + ResourcePath: fmt.Sprintf(resourcepaths.VirtualPCIResourceFormat, vmbusGUID), + RequestType: requesttype.Remove, + } + return uvm.cs.Modify(ctx, request) +} diff --git a/internal/vm/hcs/plan9.go b/internal/vm/hcs/plan9.go new file mode 100644 index 0000000000..65d60c21ba --- /dev/null +++ b/internal/vm/hcs/plan9.go @@ -0,0 +1,38 @@ +package hcs + +import ( + "context" + + "github.com/Microsoft/hcsshim/internal/hcs/resourcepaths" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/requesttype" +) + +func (uvm *utilityVM) AddPlan9(ctx context.Context, path, name string, port int32, flags int32, allowed []string) error { + modification := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Add, + Settings: hcsschema.Plan9Share{ + Name: name, + AccessName: name, + Path: path, + Port: port, + Flags: flags, + AllowedFiles: allowed, + }, + ResourcePath: resourcepaths.Plan9ShareResourcePath, + } + return uvm.cs.Modify(ctx, modification) +} + +func (uvm *utilityVM) RemovePlan9(ctx context.Context, name string, port int32) error { + modification := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Remove, + Settings: hcsschema.Plan9Share{ + Name: name, + AccessName: name, + Port: port, + }, + ResourcePath: resourcepaths.Plan9ShareResourcePath, + } + return uvm.cs.Modify(ctx, modification) +} diff --git a/internal/vm/hcs/processor.go b/internal/vm/hcs/processor.go new file mode 100644 index 0000000000..fcbe12954e --- /dev/null +++ b/internal/vm/hcs/processor.go @@ -0,0 +1,6 @@ +package hcs + +func (uvmb *utilityVMBuilder) SetProcessorCount(count uint32) error { + uvmb.doc.VirtualMachine.ComputeTopology.Processor.Count = int32(count) + return nil +} diff --git a/internal/vm/hcs/scsi.go b/internal/vm/hcs/scsi.go new file mode 100644 index 0000000000..1fc2bba473 --- /dev/null +++ b/internal/vm/hcs/scsi.go @@ -0,0 +1,89 @@ +package hcs + +import ( + "context" + "fmt" + "strconv" + + "github.com/Microsoft/hcsshim/internal/hcs/resourcepaths" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/requesttype" + "github.com/Microsoft/hcsshim/internal/vm" + "github.com/pkg/errors" +) + +func (uvmb *utilityVMBuilder) AddSCSIController(id uint32) error { + if uvmb.doc.VirtualMachine.Devices.Scsi == nil { + uvmb.doc.VirtualMachine.Devices.Scsi = make(map[string]hcsschema.Scsi, 1) + } + uvmb.doc.VirtualMachine.Devices.Scsi[strconv.Itoa(int(id))] = hcsschema.Scsi{ + Attachments: make(map[string]hcsschema.Attachment), + } + return nil +} + +func (uvmb *utilityVMBuilder) AddSCSIDisk(ctx context.Context, controller uint32, lun uint32, path string, typ vm.SCSIDiskType, readOnly bool) error { + if uvmb.doc.VirtualMachine.Devices.Scsi == nil { + return errors.New("no SCSI controller found") + } + + ctrl, ok := uvmb.doc.VirtualMachine.Devices.Scsi[strconv.Itoa(int(controller))] + if !ok { + return fmt.Errorf("no scsi controller with index %d found", controller) + } + + ctrl.Attachments[strconv.Itoa(int(lun))] = hcsschema.Attachment{ + Path: path, + Type_: string(typ), + ReadOnly: readOnly, + } + + return nil +} + +func (uvmb *utilityVMBuilder) RemoveSCSIDisk(ctx context.Context, controller uint32, lun uint32, path string) error { + return vm.ErrNotSupported +} + +func (uvm *utilityVM) AddSCSIController(id uint32) error { + return vm.ErrNotSupported +} + +func (uvm *utilityVM) AddSCSIDisk(ctx context.Context, controller uint32, lun uint32, path string, typ vm.SCSIDiskType, readOnly bool) error { + diskTypeString, err := getSCSIDiskTypeString(typ) + if err != nil { + return err + } + request := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Add, + Settings: hcsschema.Attachment{ + Path: path, + Type_: diskTypeString, + ReadOnly: readOnly, + }, + ResourcePath: fmt.Sprintf(resourcepaths.SCSIResourceFormat, strconv.Itoa(int(controller)), lun), + } + return uvm.cs.Modify(ctx, request) +} + +func (uvm *utilityVM) RemoveSCSIDisk(ctx context.Context, controller uint32, lun uint32, path string) error { + request := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Remove, + ResourcePath: fmt.Sprintf(resourcepaths.SCSIResourceFormat, strconv.Itoa(int(controller)), lun), + } + + return uvm.cs.Modify(ctx, request) +} + +func getSCSIDiskTypeString(typ vm.SCSIDiskType) (string, error) { + switch typ { + case vm.SCSIDiskTypeVHD1: + fallthrough + case vm.SCSIDiskTypeVHDX: + return "VirtualDisk", nil + case vm.SCSIDiskTypePassThrough: + return "PassThru", nil + default: + return "", fmt.Errorf("unsupported SCSI disk type: %d", typ) + } +} diff --git a/internal/vm/hcs/serial.go b/internal/vm/hcs/serial.go new file mode 100644 index 0000000000..974493344a --- /dev/null +++ b/internal/vm/hcs/serial.go @@ -0,0 +1,22 @@ +package hcs + +import ( + "strconv" + "strings" + + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/pkg/errors" +) + +func (uvmb *utilityVMBuilder) SetSerialConsole(port uint32, listenerPath string) error { + if !strings.HasPrefix(listenerPath, `\\.\pipe\`) { + return errors.New("listener for serial console is not a named pipe") + } + + uvmb.doc.VirtualMachine.Devices.ComPorts = map[string]hcsschema.ComPort{ + strconv.Itoa(int(port)): { // "0" would be COM1 + NamedPipe: listenerPath, + }, + } + return nil +} diff --git a/internal/vm/hcs/stats.go b/internal/vm/hcs/stats.go new file mode 100644 index 0000000000..bb4cbfc59c --- /dev/null +++ b/internal/vm/hcs/stats.go @@ -0,0 +1,159 @@ +package hcs + +import ( + "context" + "strings" + + "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/go-winio/pkg/process" + "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/vm" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" +) + +// checkProcess checks if the process identified by the given pid has a name +// matching `desiredProcessName`, and is running as a user with domain +// `desiredDomain` and user name `desiredUser`. If the process matches, it +// returns a handle to the process. If the process does not match, it returns +// 0. +func checkProcess(ctx context.Context, pid uint32, desiredProcessName string, desiredDomain string, desiredUser string) (p windows.Handle, err error) { + desiredProcessName = strings.ToUpper(desiredProcessName) + desiredDomain = strings.ToUpper(desiredDomain) + desiredUser = strings.ToUpper(desiredUser) + + p, err = windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION|windows.PROCESS_VM_READ, false, pid) + if err != nil { + return 0, err + } + + defer func(openedProcess windows.Handle) { + // If we don't return this process handle, close it so it doesn't leak. + if p == 0 { + windows.Close(openedProcess) + } + }(p) + + // Querying vmmem's image name as a win32 path returns ERROR_GEN_FAILURE + // for some reason, so we query it as an NT path instead. + name, err := process.QueryFullProcessImageName(p, process.ImageNameFormatNTPath) + if err != nil { + return 0, err + } + if strings.ToUpper(name) == desiredProcessName { + var t windows.Token + if err := windows.OpenProcessToken(p, windows.TOKEN_QUERY, &t); err != nil { + return 0, err + } + defer t.Close() + tUser, err := t.GetTokenUser() + if err != nil { + return 0, err + } + user, domain, _, err := tUser.User.Sid.LookupAccount("") + if err != nil { + return 0, err + } + log.G(ctx).WithFields(logrus.Fields{ + "name": name, + "domain": domain, + "user": user, + }).Debug("checking vmmem process identity") + if strings.ToUpper(domain) == desiredDomain && strings.ToUpper(user) == desiredUser { + return p, nil + } + } + return 0, nil +} + +// lookupVMMEM locates the vmmem process for a VM given the VM ID. It returns +// a handle to the vmmem process. The lookup is implemented by enumerating all +// processes on the system, and finding a process with full name "vmmem", +// running as "NT VIRTUAL MACHINE\". +func lookupVMMEM(ctx context.Context, vmID guid.GUID) (proc windows.Handle, err error) { + vmIDStr := strings.ToUpper(vmID.String()) + log.G(ctx).WithField("vmID", vmIDStr).Debug("looking up vmmem") + + pids, err := process.EnumProcesses() + if err != nil { + return 0, errors.Wrap(err, "failed to enumerate processes") + } + for _, pid := range pids { + p, err := checkProcess(ctx, pid, "vmmem", "NT VIRTUAL MACHINE", vmIDStr) + if err != nil { + // Checking the process could fail for a variety of reasons, such as + // the process exiting since we called EnumProcesses, or not having + // access to open the process (even as SYSTEM). In the case of an + // error, we just log and continue looking at the other processes. + log.G(ctx).WithField("pid", pid).Debug("failed to check process") + continue + } + if p != 0 { + log.G(ctx).WithField("pid", pid).Debug("found vmmem match") + return p, nil + } + } + return 0, errors.New("failed to find matching vmmem process") +} + +// getVMMEMProcess returns a handle to the vmmem process associated with this +// UVM. It only does the actual process lookup once, after which it caches the +// process handle in the UVM object. +func (uvm *utilityVM) getVMMEMProcess(ctx context.Context) (windows.Handle, error) { + uvm.vmmemOnce.Do(func() { + uvm.vmmemProcess, uvm.vmmemErr = lookupVMMEM(ctx, uvm.vmID) + }) + return uvm.vmmemProcess, uvm.vmmemErr +} + +func (uvm *utilityVM) Stats(ctx context.Context) (*stats.VirtualMachineStatistics, error) { + s := &stats.VirtualMachineStatistics{} + props, err := uvm.cs.PropertiesV2(ctx, hcsschema.PTStatistics, hcsschema.PTMemory) + if err != nil { + return nil, err + } + + s.Processor = &stats.VirtualMachineProcessorStatistics{} + s.Processor.TotalRuntimeNS = uint64(props.Statistics.Processor.TotalRuntime100ns * 100) + s.Memory = &stats.VirtualMachineMemoryStatistics{} + + if uvm.backingType == vm.MemoryBackingTypePhysical { + // If the uvm is physically backed we set the working set to the total amount allocated + // to the UVM. AssignedMemory returns the number of 4KB pages. Will always be 4KB + // regardless of what the UVMs actual page size is so we don't need that information. + if props.Memory != nil { + s.Memory.WorkingSetBytes = props.Memory.VirtualMachineMemory.AssignedMemory * 4096 + } + } else { + // The HCS properties does not return sufficient information to calculate + // working set size for a VA-backed UVM. To work around this, we instead + // locate the vmmem process for the VM, and query that process's working set + // instead, which will be the working set for the VM. + vmmemProc, err := uvm.getVMMEMProcess(ctx) + if err != nil { + return nil, err + } + memCounters, err := process.GetProcessMemoryInfo(vmmemProc) + if err != nil { + return nil, err + } + s.Memory.WorkingSetBytes = uint64(memCounters.WorkingSetSize) + } + + if props.Memory != nil { + s.Memory.VirtualNodeCount = props.Memory.VirtualNodeCount + s.Memory.VmMemory = &stats.VirtualMachineMemory{} + s.Memory.VmMemory.AvailableMemory = props.Memory.VirtualMachineMemory.AvailableMemory + s.Memory.VmMemory.AvailableMemoryBuffer = props.Memory.VirtualMachineMemory.AvailableMemoryBuffer + s.Memory.VmMemory.ReservedMemory = props.Memory.VirtualMachineMemory.ReservedMemory + s.Memory.VmMemory.AssignedMemory = props.Memory.VirtualMachineMemory.AssignedMemory + s.Memory.VmMemory.SlpActive = props.Memory.VirtualMachineMemory.SlpActive + s.Memory.VmMemory.BalancingEnabled = props.Memory.VirtualMachineMemory.BalancingEnabled + s.Memory.VmMemory.DmOperationInProgress = props.Memory.VirtualMachineMemory.DmOperationInProgress + } + + return s, nil +} diff --git a/internal/vm/hcs/storage.go b/internal/vm/hcs/storage.go new file mode 100644 index 0000000000..9b2317a036 --- /dev/null +++ b/internal/vm/hcs/storage.go @@ -0,0 +1,7 @@ +package hcs + +func (uvmb *utilityVMBuilder) SetStorageQos(iopsMaximum int64, bandwidthMaximum int64) error { + uvmb.doc.VirtualMachine.StorageQoS.BandwidthMaximum = int32(bandwidthMaximum) + uvmb.doc.VirtualMachine.StorageQoS.IopsMaximum = int32(iopsMaximum) + return nil +} diff --git a/internal/vm/hcs/supported.go b/internal/vm/hcs/supported.go new file mode 100644 index 0000000000..e179c252c3 --- /dev/null +++ b/internal/vm/hcs/supported.go @@ -0,0 +1,8 @@ +package hcs + +import "github.com/Microsoft/hcsshim/internal/vm" + +func (uvm *utilityVM) Supported(resource vm.Resource, op vm.ResourceOperation) bool { + // For now at least HCS supports everything we care about. + return true +} diff --git a/internal/vm/hcs/vmsocket.go b/internal/vm/hcs/vmsocket.go new file mode 100644 index 0000000000..22e268858a --- /dev/null +++ b/internal/vm/hcs/vmsocket.go @@ -0,0 +1,41 @@ +package hcs + +import ( + "context" + "net" + + "github.com/Microsoft/go-winio" + "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/hcsshim/internal/vm" + "github.com/pkg/errors" +) + +func (uvm *utilityVM) VMSocketListen(ctx context.Context, listenType vm.VMSocketType, connID interface{}) (net.Listener, error) { + switch listenType { + case vm.HvSocket: + serviceGUID, ok := connID.(guid.GUID) + if !ok { + return nil, errors.New("parameter passed to hvsocketlisten is not a GUID") + } + return uvm.hvSocketListen(ctx, serviceGUID) + case vm.VSock: + port, ok := connID.(uint32) + if !ok { + return nil, errors.New("parameter passed to vsocklisten is not the right type") + } + return uvm.vsockListen(ctx, port) + default: + return nil, errors.New("unknown vmsocket type requested") + } +} + +func (uvm *utilityVM) hvSocketListen(ctx context.Context, serviceID guid.GUID) (net.Listener, error) { + return winio.ListenHvsock(&winio.HvsockAddr{ + VMID: uvm.vmID, + ServiceID: serviceID, + }) +} + +func (uvm *utilityVM) vsockListen(ctx context.Context, port uint32) (net.Listener, error) { + return nil, vm.ErrNotSupported +} diff --git a/internal/vm/hcs/vpmem.go b/internal/vm/hcs/vpmem.go new file mode 100644 index 0000000000..09287ab8f8 --- /dev/null +++ b/internal/vm/hcs/vpmem.go @@ -0,0 +1,82 @@ +package hcs + +import ( + "context" + "fmt" + "strconv" + + "github.com/Microsoft/hcsshim/internal/hcs/resourcepaths" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/requesttype" + "github.com/Microsoft/hcsshim/internal/vm" + "github.com/pkg/errors" +) + +func (uvmb *utilityVMBuilder) AddVPMemController(maximumDevices uint32, maximumSizeBytes uint64) error { + uvmb.doc.VirtualMachine.Devices.VirtualPMem = &hcsschema.VirtualPMemController{ + MaximumCount: maximumDevices, + MaximumSizeBytes: maximumSizeBytes, + } + uvmb.doc.VirtualMachine.Devices.VirtualPMem.Devices = make(map[string]hcsschema.VirtualPMemDevice) + return nil +} + +func (uvmb *utilityVMBuilder) AddVPMemDevice(ctx context.Context, id uint32, path string, readOnly bool, imageFormat vm.VPMemImageFormat) error { + if uvmb.doc.VirtualMachine.Devices.VirtualPMem == nil { + return errors.New("VPMem controller has not been added") + } + imageFormatString, err := getVPMemImageFormatString(imageFormat) + if err != nil { + return err + } + uvmb.doc.VirtualMachine.Devices.VirtualPMem.Devices[strconv.Itoa(int(id))] = hcsschema.VirtualPMemDevice{ + HostPath: path, + ReadOnly: readOnly, + ImageFormat: imageFormatString, + } + return nil +} + +func (uvmb *utilityVMBuilder) RemoveVPMemDevice(ctx context.Context, id uint32, path string) error { + return vm.ErrNotSupported +} + +func (uvm *utilityVM) AddVPMemController(maximumDevices uint32, maximumSizeBytes uint64) error { + return vm.ErrNotSupported +} + +func (uvm *utilityVM) AddVPMemDevice(ctx context.Context, id uint32, path string, readOnly bool, imageFormat vm.VPMemImageFormat) error { + imageFormatString, err := getVPMemImageFormatString(imageFormat) + if err != nil { + return err + } + request := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Add, + Settings: hcsschema.VirtualPMemDevice{ + HostPath: path, + ReadOnly: readOnly, + ImageFormat: imageFormatString, + }, + ResourcePath: fmt.Sprintf(resourcepaths.VPMemControllerResourceFormat, id), + } + return uvm.cs.Modify(ctx, request) +} + +func (uvm *utilityVM) RemoveVPMemDevice(ctx context.Context, id uint32, path string) error { + request := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Remove, + ResourcePath: fmt.Sprintf(resourcepaths.VPMemControllerResourceFormat, id), + } + return uvm.cs.Modify(ctx, request) +} + +func getVPMemImageFormatString(imageFormat vm.VPMemImageFormat) (string, error) { + switch imageFormat { + case vm.VPMemImageFormatVHD1: + return "Vhd1", nil + case vm.VPMemImageFormatVHDX: + return "Vhdx", nil + default: + return "", fmt.Errorf("unsupported VPMem image format: %d", imageFormat) + } +} diff --git a/internal/vm/hcs/vsmb.go b/internal/vm/hcs/vsmb.go new file mode 100644 index 0000000000..b5cbade5b7 --- /dev/null +++ b/internal/vm/hcs/vsmb.go @@ -0,0 +1,65 @@ +package hcs + +import ( + "context" + + "github.com/Microsoft/hcsshim/internal/hcs/resourcepaths" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/requesttype" + "github.com/Microsoft/hcsshim/internal/vm" +) + +func (uvmb *utilityVMBuilder) AddVSMB(ctx context.Context, path string, name string, allowed []string, options *vm.VSMBOptions) error { + uvmb.doc.VirtualMachine.Devices.VirtualSmb = &hcsschema.VirtualSmb{ + DirectFileMappingInMB: 1024, // Sensible default, but could be a tuning parameter somewhere + Shares: []hcsschema.VirtualSmbShare{ + { + Name: name, + Path: path, + AllowedFiles: allowed, + Options: vmVSMBOptionsToHCS(options), + }, + }, + } + return nil +} + +func (uvmb *utilityVMBuilder) RemoveVSMB(ctx context.Context, name string) error { + return vm.ErrNotSupported +} + +func vmVSMBOptionsToHCS(options *vm.VSMBOptions) *hcsschema.VirtualSmbShareOptions { + return &hcsschema.VirtualSmbShareOptions{ + ReadOnly: options.ReadOnly, + ShareRead: options.ShareRead, + CacheIo: options.CacheIo, + NoOplocks: options.NoOplocks, + NoDirectmap: options.NoDirectMap, + TakeBackupPrivilege: options.TakeBackupPrivilege, + PseudoOplocks: options.PseudoOplocks, + PseudoDirnotify: options.PseudoDirnotify, + } +} + +func (uvm *utilityVM) AddVSMB(ctx context.Context, path string, name string, allowed []string, options *vm.VSMBOptions) error { + modification := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Add, + Settings: hcsschema.VirtualSmbShare{ + Name: name, + Options: vmVSMBOptionsToHCS(options), + Path: path, + AllowedFiles: allowed, + }, + ResourcePath: resourcepaths.VSMBShareResourcePath, + } + return uvm.cs.Modify(ctx, modification) +} + +func (uvm *utilityVM) RemoveVSMB(ctx context.Context, name string) error { + modification := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Remove, + Settings: hcsschema.VirtualSmbShare{Name: name}, + ResourcePath: resourcepaths.VSMBShareResourcePath, + } + return uvm.cs.Modify(ctx, modification) +} diff --git a/internal/vm/hcs/windows.go b/internal/vm/hcs/windows.go new file mode 100644 index 0000000000..029c71d417 --- /dev/null +++ b/internal/vm/hcs/windows.go @@ -0,0 +1,10 @@ +package hcs + +import ( + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" +) + +func (uvmb *utilityVMBuilder) SetCPUGroup(id string) error { + uvmb.doc.VirtualMachine.ComputeTopology.Processor.CpuGroup = &hcsschema.CpuGroup{Id: id} + return nil +} diff --git a/internal/vm/vm.go b/internal/vm/vm.go new file mode 100644 index 0000000000..0799d933f1 --- /dev/null +++ b/internal/vm/vm.go @@ -0,0 +1,212 @@ +package vm + +import ( + "context" + "errors" + "net" + + "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats" +) + +var ( + ErrNotSupported = errors.New("virtstack does not support the operation") + ErrAlreadySet = errors.New("field has already been set") + ErrUnknownGuestOS = errors.New("unknown guest operating system supplied") + + ErrNotInPreCreatedState = errors.New("VM is not in pre-created state") + ErrNotInCreatedState = errors.New("VM is not in created state") + ErrNotInRunningState = errors.New("VM is not in running state") + ErrNotInPausedState = errors.New("VM is not in paused state") +) + +const ( + HCS = "hcs" + RemoteVM = "remotevm" +) + +// UVM is an abstraction around a lightweight virtual machine. It houses core lifecycle methods such as Create +// Start, and Stop and also several optional nested interfaces that can be used to determine what the virtual machine +// supports and to configure these resources. +type UVM interface { + // ID will return a string identifier for the Utility VM. + ID() string + + // Start will power on the Utility VM and put it into a running state. This will boot the guest OS and start all of the + // devices configured on the machine. + Start(ctx context.Context) error + + // Stop will shutdown the Utility VM and place it into a terminated state. + Stop(ctx context.Context) error + + // Pause will place the Utility VM into a paused state. The guest OS will be halted and any devices will have be in a + // a suspended state. Save can be used to snapshot the current state of the virtual machine, and Resume can be used to + // place the virtual machine back into a running state. + Pause(ctx context.Context) error + + // Resume will put a previously paused Utility VM back into a running state. The guest OS will resume operation from the point + // in time it was paused and all devices should be un-suspended. + Resume(ctx context.Context) error + + // Save will snapshot the state of the Utility VM at the point in time when the VM was paused. + Save(ctx context.Context) error + + // Wait synchronously waits for the Utility VM to shutdown or terminate. A call to stop will trigger this + // to unblock. + Wait() error + + // Stats returns statistics about the Utility VM. This includes things like assigned memory, available memory, + // processor runtime etc. + Stats(ctx context.Context) (*stats.VirtualMachineStatistics, error) + + // Supported returns if the virt stack supports a given operation on a resource. + Supported(resource Resource, operation ResourceOperation) bool + + // ExitError will return any error if the Utility VM exited unexpectedly, or if the Utility VM experienced an + // error after Wait returned, it will return the wait error. + ExitError() error +} + +// Resource refers to the type of a resource on a Utility VM. +type Resource uint8 + +const ( + VPMem = iota + SCSI + Network + VSMB + PCI + Plan9 + CPUGroup +) + +// Operation refers to the type of operation to perform on a given resource. +type ResourceOperation uint8 + +const ( + Add ResourceOperation = iota + Remove + Update +) + +// GuestOS signifies the guest operating system that a Utility VM will be running. +type GuestOS string + +const ( + Windows GuestOS = "windows" + Linux GuestOS = "linux" +) + +// State signifies the states that a Utility VM can be in. The state of the Utility VM should be the source of truth for what +// operations can be performed at a given moment. +type State uint8 + +const ( + StatePreCreated State = iota + StateCreated + StateRunning + StateTerminated + StatePaused +) + +// SCSIDiskType refers to the disk type of the scsi device. This is either a vhd, vhdx, or a physical disk. +type SCSIDiskType uint8 + +const ( + SCSIDiskTypeVHD1 SCSIDiskType = iota + SCSIDiskTypeVHDX + SCSIDiskTypePassThrough +) + +// SCSIManager manages adding and removing SCSI devices for a Utility VM. +type SCSIManager interface { + // AddSCSIController adds a SCSI controller to the Utility VM configuration document. + AddSCSIController(id uint32) error + // AddSCSIDisk adds a SCSI disk to the configuration document if in a precreated state, or hot adds a + // SCSI disk to the Utility VM if the VM is running. + AddSCSIDisk(ctx context.Context, controller uint32, lun uint32, path string, typ SCSIDiskType, readOnly bool) error + // RemoveSCSIDisk removes a SCSI disk from a Utility VM. + RemoveSCSIDisk(ctx context.Context, controller uint32, lun uint32, path string) error +} + +// VPMemImageFormat refers to the image type of the vpmem block device. This is either a vhd or vhdx. +type VPMemImageFormat uint8 + +const ( + VPMemImageFormatVHD1 VPMemImageFormat = iota + VPMemImageFormatVHDX +) + +// VPMemManager manages adding and removing virtual persistent memory devices for a Utility VM. +type VPMemManager interface { + // AddVPMemController adds a new virtual pmem controller to the Utility VM. + // `maximumDevices` specifies how many vpmem devices will be present in the guest. + // `maximumSizeBytes` specifies the maximum size allowed for a vpmem device. + AddVPMemController(maximumDevices uint32, maximumSizeBytes uint64) error + // AddVPMemDevice adds a virtual pmem device to the Utility VM. + AddVPMemDevice(ctx context.Context, id uint32, path string, readOnly bool, imageFormat VPMemImageFormat) error + // RemoveVpmemDevice removes a virtual pmem device from the Utility VM. + RemoveVPMemDevice(ctx context.Context, id uint32, path string) error +} + +// NetworkManager manages adding and removing network adapters for a Utility VM. +type NetworkManager interface { + // AddNIC adds a network adapter to the Utility VM. `nicID` should be a string representation of a + // Windows GUID. + AddNIC(ctx context.Context, nicID string, endpointID string, macAddr string) error + // RemoveNIC removes a network adapter from the Utility VM. `nicID` should be a string representation of a + // Windows GUID. + RemoveNIC(ctx context.Context, nicID string, endpointID string, macAddr string) error +} + +// PCIManager manages assiging pci devices to a Utility VM. This is Windows specific at the moment. +type PCIManager interface { + // AddDevice adds the pci device identified by `instanceID` to the Utility VM. + // https://docs.microsoft.com/en-us/windows-hardware/drivers/install/instance-ids + AddDevice(ctx context.Context, instanceID string, vmbusGUID string) error + // RemoveDevice removes the pci device identified by `instanceID` from the Utility VM. + RemoveDevice(ctx context.Context, instanceID string, vmbusGUID string) error +} + +// VMSocketType refers to which hypervisor socket transport type to use. +type VMSocketType uint8 + +const ( + HvSocket VMSocketType = iota + VSock +) + +// VMSocketManager manages configuration for a hypervisor socket transport. This includes sockets such as +// HvSocket and Vsock. +type VMSocketManager interface { + // VMSocketListen will create the requested vmsocket type and listen on the address specified by `connID`. + // For HvSocket the type expected is a GUID, for Vsock it's a port of type uint32. + VMSocketListen(ctx context.Context, socketType VMSocketType, connID interface{}) (net.Listener, error) +} + +// VSMBOptions +type VSMBOptions struct { + ReadOnly bool + CacheIo bool + NoDirectMap bool + PseudoOplocks bool + ShareRead bool + TakeBackupPrivilege bool + NoOplocks bool + PseudoDirnotify bool +} + +// VSMBManager manages adding virtual smb shares to a Utility VM. +type VSMBManager interface { + // AddVSMB adds a virtual smb share to a running Utility VM. + AddVSMB(ctx context.Context, hostPath string, name string, allowedFiles []string, options *VSMBOptions) error + // RemoveVSMB removes a virtual smb share from a running Utility VM. + RemoveVSMB(ctx context.Context, name string) error +} + +// Plan9Manager manages adding plan 9 shares to a Utility VM. +type Plan9Manager interface { + // AddPlan9 adds a plan 9 share to a running Utility VM. + AddPlan9(ctx context.Context, path, name string, port int32, flags int32, allowed []string) error + // RemovePlan9 removes a plan 9 share from a running Utility VM. + RemovePlan9(ctx context.Context, name string, port int32) error +}