From 4641e993ca02a8c98ef95a840bc2b24da28bee8a Mon Sep 17 00:00:00 2001 From: Daniel Canter Date: Thu, 22 Apr 2021 03:32:57 -0700 Subject: [PATCH] Add abstractions for direct HCS interactions Add vm package, uvm and uvmbuilder interfaces to abstract away the operations that we call directly into hcs for. This will be useful for having these operations be performed by a different virtstack so long as it supports what is needed for containers. Signed-off-by: Daniel Canter --- internal/uvm/virtual_device.go | 3 +- internal/vm/builder.go | 74 ++++++++++++ internal/vm/hcs/boot.go | 31 +++++ internal/vm/hcs/builder.go | 93 +++++++++++++++ internal/vm/hcs/hcs.go | 75 ++++++++++++ internal/vm/hcs/memory.go | 28 +++++ internal/vm/hcs/network.go | 34 ++++++ internal/vm/hcs/pci.go | 33 +++++ internal/vm/hcs/plan9.go | 38 ++++++ internal/vm/hcs/processor.go | 6 + internal/vm/hcs/scsi.go | 89 ++++++++++++++ internal/vm/hcs/serial.go | 22 ++++ internal/vm/hcs/stats.go | 159 +++++++++++++++++++++++++ internal/vm/hcs/storage.go | 7 ++ internal/vm/hcs/supported.go | 8 ++ internal/vm/hcs/vmsocket.go | 41 +++++++ internal/vm/hcs/vpmem.go | 82 +++++++++++++ internal/vm/hcs/vsmb.go | 65 ++++++++++ internal/vm/hcs/windows.go | 10 ++ internal/vm/vm.go | 212 +++++++++++++++++++++++++++++++++ 20 files changed, 1109 insertions(+), 1 deletion(-) create mode 100644 internal/vm/builder.go create mode 100644 internal/vm/hcs/boot.go create mode 100644 internal/vm/hcs/builder.go create mode 100644 internal/vm/hcs/hcs.go create mode 100644 internal/vm/hcs/memory.go create mode 100644 internal/vm/hcs/network.go create mode 100644 internal/vm/hcs/pci.go create mode 100644 internal/vm/hcs/plan9.go create mode 100644 internal/vm/hcs/processor.go create mode 100644 internal/vm/hcs/scsi.go create mode 100644 internal/vm/hcs/serial.go create mode 100644 internal/vm/hcs/stats.go create mode 100644 internal/vm/hcs/storage.go create mode 100644 internal/vm/hcs/supported.go create mode 100644 internal/vm/hcs/vmsocket.go create mode 100644 internal/vm/hcs/vpmem.go create mode 100644 internal/vm/hcs/vsmb.go create mode 100644 internal/vm/hcs/windows.go create mode 100644 internal/vm/vm.go diff --git a/internal/uvm/virtual_device.go b/internal/uvm/virtual_device.go index 66d71e08d0..679941997f 100644 --- a/internal/uvm/virtual_device.go +++ b/internal/uvm/virtual_device.go @@ -95,7 +95,8 @@ func (uvm *UtilityVM) AssignDevice(ctx context.Context, deviceID string) (*VPCID request := &hcsschema.ModifySettingRequest{ ResourcePath: fmt.Sprintf(resourcepaths.VirtualPCIResourceFormat, vmBusGUID), RequestType: requesttype.Add, - Settings: targetDevice} + Settings: targetDevice, + } // WCOW (when supported) does not require a guest request as part of the // device assignment diff --git a/internal/vm/builder.go b/internal/vm/builder.go new file mode 100644 index 0000000000..4be3eee296 --- /dev/null +++ b/internal/vm/builder.go @@ -0,0 +1,74 @@ +package vm + +import ( + "context" +) + +type UVMBuilder interface { + // Create will create the Utility VM in a paused/powered off state with whatever is present in the implementation + // of the interfaces config at the time of the call. + Create(ctx context.Context) (UVM, error) +} + +type MemoryBackingType uint8 + +const ( + MemoryBackingTypeVirtual MemoryBackingType = iota + MemoryBackingTypePhysical +) + +// MemoryConfig holds the memory options that should be configurable for a Utility VM. +type MemoryConfig struct { + BackingType MemoryBackingType + DeferredCommit bool + HotHint bool + ColdHint bool + ColdDiscardHint bool +} + +// MemoryManager handles setting and managing memory configurations for the Utility VM. +type MemoryManager interface { + // SetMemoryLimit sets the amount of memory in megabytes that the Utility VM will be assigned. + SetMemoryLimit(memoryMB uint64) error + // SetMemoryConfig sets an array of different memory configuration options available. This includes things like the + // type of memory to back the VM (virtual/physical). + SetMemoryConfig(config *MemoryConfig) error + // SetMMIOConfig sets memory mapped IO configurations for the Utility VM. + SetMMIOConfig(lowGapMB uint64, highBaseMB uint64, highGapMB uint64) error +} + +// ProcessorManager handles setting and managing processor configurations for the Utility VM. +type ProcessorManager interface { + // SetProcessorCount sets the number of virtual processors that will be assigned to the Utility VM. + SetProcessorCount(count uint32) error +} + +// SerialManager manages setting up serial consoles for the Utility VM. +type SerialManager interface { + // SetSerialConsole sets up a serial console for `port`. Output will be relayed to the listener specified + // by `listenerPath`. For HCS `listenerPath` this is expected to be a path to a named pipe. + SetSerialConsole(port uint32, listenerPath string) error +} + +// BootManager manages boot configurations for the Utility VM. +type BootManager interface { + // SetUEFIBoot sets UEFI configurations for booting a Utility VM. + SetUEFIBoot(dir string, path string, args string) error + // SetLinuxKernelDirectBoot sets Linux direct boot configurations for booting a Utility VM. + SetLinuxKernelDirectBoot(kernel string, initRD string, cmd string) error +} + +// StorageQosManager manages setting storage limits on the Utility VM. +type StorageQosManager interface { + // SetStorageQos sets storage related options for the Utility VM + SetStorageQos(iopsMaximum int64, bandwidthMaximum int64) error +} + +// WindowsConfigManager manages options specific to a Windows host (cpugroups etc.) +type WindowsConfigManager interface { + // SetCPUGroup sets the CPU group that the Utility VM will belong to on a Windows host. + SetCPUGroup(id string) error +} + +// LinuxConfigManager manages options specific to a Linux host. +type LinuxConfigManager interface{} diff --git a/internal/vm/hcs/boot.go b/internal/vm/hcs/boot.go new file mode 100644 index 0000000000..425feebd2c --- /dev/null +++ b/internal/vm/hcs/boot.go @@ -0,0 +1,31 @@ +package hcs + +import ( + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/osversion" + "github.com/pkg/errors" +) + +func (uvmb *utilityVMBuilder) SetUEFIBoot(dir string, path string, args string) error { + uvmb.doc.VirtualMachine.Chipset.Uefi = &hcsschema.Uefi{ + BootThis: &hcsschema.UefiBootEntry{ + DevicePath: path, + DeviceType: "VmbFs", + VmbFsRootPath: dir, + OptionalData: args, + }, + } + return nil +} + +func (uvmb *utilityVMBuilder) SetLinuxKernelDirectBoot(kernel string, initRD string, cmd string) error { + if osversion.Get().Build < 18286 { + return errors.New("Linux kernel direct boot requires at least Windows version 18286") + } + uvmb.doc.VirtualMachine.Chipset.LinuxKernelDirect = &hcsschema.LinuxKernelDirect{ + KernelFilePath: kernel, + InitRdPath: initRD, + KernelCmdLine: cmd, + } + return nil +} diff --git a/internal/vm/hcs/builder.go b/internal/vm/hcs/builder.go new file mode 100644 index 0000000000..f37a0f08b8 --- /dev/null +++ b/internal/vm/hcs/builder.go @@ -0,0 +1,93 @@ +package hcs + +import ( + "context" + + "github.com/Microsoft/hcsshim/internal/hcs" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/schemaversion" + "github.com/Microsoft/hcsshim/internal/vm" + "github.com/pkg/errors" +) + +type utilityVMBuilder struct { + id string + guestOS vm.GuestOS + doc *hcsschema.ComputeSystem +} + +func NewUVMBuilder(id string, owner string, guestOS vm.GuestOS) (vm.UVMBuilder, error) { + doc := &hcsschema.ComputeSystem{ + Owner: owner, + SchemaVersion: schemaversion.SchemaV21(), + ShouldTerminateOnLastHandleClosed: true, + VirtualMachine: &hcsschema.VirtualMachine{ + StopOnReset: true, + Chipset: &hcsschema.Chipset{}, + ComputeTopology: &hcsschema.Topology{ + Memory: &hcsschema.Memory2{ + AllowOvercommit: true, + }, + Processor: &hcsschema.Processor2{}, + }, + Devices: &hcsschema.Devices{ + HvSocket: &hcsschema.HvSocket2{ + HvSocketConfig: &hcsschema.HvSocketSystemConfig{ + // Allow administrators and SYSTEM to bind to vsock sockets + // so that we can create a GCS log socket. + DefaultBindSecurityDescriptor: "D:P(A;;FA;;;SY)(A;;FA;;;BA)", + }, + }, + }, + }, + } + + switch guestOS { + case vm.Windows: + doc.VirtualMachine.Devices.VirtualSmb = &hcsschema.VirtualSmb{} + case vm.Linux: + doc.VirtualMachine.Devices.Plan9 = &hcsschema.Plan9{} + default: + return nil, vm.ErrUnknownGuestOS + } + + return &utilityVMBuilder{ + id: id, + guestOS: guestOS, + doc: doc, + }, nil +} + +func (uvmb *utilityVMBuilder) Create(ctx context.Context) (_ vm.UVM, err error) { + cs, err := hcs.CreateComputeSystem(ctx, uvmb.id, uvmb.doc) + if err != nil { + return nil, errors.Wrap(err, "failed to create hcs compute system") + } + + defer func() { + if err != nil { + _ = cs.Terminate(ctx) + _ = cs.Wait() + } + }() + + backingType := vm.MemoryBackingTypeVirtual + if !uvmb.doc.VirtualMachine.ComputeTopology.Memory.AllowOvercommit { + backingType = vm.MemoryBackingTypePhysical + } + + uvm := &utilityVM{ + id: uvmb.id, + guestOS: uvmb.guestOS, + cs: cs, + backingType: backingType, + state: vm.StateCreated, + } + + properties, err := cs.Properties(ctx) + if err != nil { + return nil, err + } + uvm.vmID = properties.RuntimeID + return uvm, nil +} diff --git a/internal/vm/hcs/hcs.go b/internal/vm/hcs/hcs.go new file mode 100644 index 0000000000..67c44a95d9 --- /dev/null +++ b/internal/vm/hcs/hcs.go @@ -0,0 +1,75 @@ +package hcs + +import ( + "context" + "sync" + + "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/hcsshim/internal/hcs" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/vm" + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +type utilityVM struct { + id string + state vm.State + guestOS vm.GuestOS + cs *hcs.System + backingType vm.MemoryBackingType + vmmemProcess windows.Handle + vmmemErr error + vmmemOnce sync.Once + vmID guid.GUID +} + +func (uvm *utilityVM) ID() string { + return uvm.id +} + +func (uvm *utilityVM) Start(ctx context.Context) (err error) { + if err := uvm.cs.Start(ctx); err != nil { + return errors.Wrap(err, "failed to start utility VM") + } + return nil +} + +func (uvm *utilityVM) Stop(ctx context.Context) error { + if err := uvm.cs.Terminate(ctx); err != nil { + return errors.Wrap(err, "failed to terminate utility VM") + } + return nil +} + +func (uvm *utilityVM) Pause(ctx context.Context) error { + if err := uvm.cs.Pause(ctx); err != nil { + return errors.Wrap(err, "failed to pause utility VM") + } + return nil +} + +func (uvm *utilityVM) Resume(ctx context.Context) error { + if err := uvm.cs.Resume(ctx); err != nil { + return errors.Wrap(err, "failed to resume utility VM") + } + return nil +} + +func (uvm *utilityVM) Save(ctx context.Context) error { + saveOptions := hcsschema.SaveOptions{ + SaveType: "AsTemplate", + } + if err := uvm.cs.Save(ctx, saveOptions); err != nil { + return errors.Wrap(err, "failed to save utility VM state") + } + return nil +} + +func (uvm *utilityVM) Wait() error { + return uvm.cs.Wait() +} + +func (uvm *utilityVM) ExitError() error { + return uvm.cs.ExitError() +} diff --git a/internal/vm/hcs/memory.go b/internal/vm/hcs/memory.go new file mode 100644 index 0000000000..afb9932628 --- /dev/null +++ b/internal/vm/hcs/memory.go @@ -0,0 +1,28 @@ +package hcs + +import ( + "github.com/Microsoft/hcsshim/internal/vm" +) + +func (uvmb *utilityVMBuilder) SetMemoryLimit(memoryMB uint64) error { + uvmb.doc.VirtualMachine.ComputeTopology.Memory.SizeInMB = memoryMB + return nil +} + +func (uvmb *utilityVMBuilder) SetMemoryConfig(config *vm.MemoryConfig) error { + memory := uvmb.doc.VirtualMachine.ComputeTopology.Memory + memory.AllowOvercommit = config.BackingType == vm.MemoryBackingTypeVirtual + memory.EnableDeferredCommit = config.DeferredCommit + memory.EnableHotHint = config.HotHint + memory.EnableColdHint = config.ColdHint + memory.EnableColdDiscardHint = config.ColdDiscardHint + return nil +} + +func (uvmb *utilityVMBuilder) SetMMIOConfig(lowGapMB uint64, highBaseMB uint64, highGapMB uint64) error { + memory := uvmb.doc.VirtualMachine.ComputeTopology.Memory + memory.LowMMIOGapInMB = lowGapMB + memory.HighMMIOBaseInMB = highBaseMB + memory.HighMMIOGapInMB = highGapMB + return nil +} diff --git a/internal/vm/hcs/network.go b/internal/vm/hcs/network.go new file mode 100644 index 0000000000..6610fffd39 --- /dev/null +++ b/internal/vm/hcs/network.go @@ -0,0 +1,34 @@ +package hcs + +import ( + "context" + "fmt" + + "github.com/Microsoft/hcsshim/internal/hcs/resourcepaths" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/requesttype" +) + +func (uvm *utilityVM) AddNIC(ctx context.Context, nicID, endpointID, macAddr string) error { + request := hcsschema.ModifySettingRequest{ + RequestType: requesttype.Add, + ResourcePath: fmt.Sprintf(resourcepaths.NetworkResourceFormat, nicID), + Settings: hcsschema.NetworkAdapter{ + EndpointId: endpointID, + MacAddress: macAddr, + }, + } + return uvm.cs.Modify(ctx, request) +} + +func (uvm *utilityVM) RemoveNIC(ctx context.Context, nicID, endpointID, macAddr string) error { + request := hcsschema.ModifySettingRequest{ + RequestType: requesttype.Remove, + ResourcePath: fmt.Sprintf(resourcepaths.NetworkResourceFormat, nicID), + Settings: hcsschema.NetworkAdapter{ + EndpointId: endpointID, + MacAddress: macAddr, + }, + } + return uvm.cs.Modify(ctx, request) +} diff --git a/internal/vm/hcs/pci.go b/internal/vm/hcs/pci.go new file mode 100644 index 0000000000..262823768f --- /dev/null +++ b/internal/vm/hcs/pci.go @@ -0,0 +1,33 @@ +package hcs + +import ( + "context" + "fmt" + + "github.com/Microsoft/hcsshim/internal/hcs/resourcepaths" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/requesttype" +) + +func (uvm *utilityVM) AddDevice(ctx context.Context, instanceID, vmbusGUID string) error { + request := &hcsschema.ModifySettingRequest{ + ResourcePath: fmt.Sprintf(resourcepaths.VirtualPCIResourceFormat, vmbusGUID), + RequestType: requesttype.Add, + Settings: hcsschema.VirtualPciDevice{ + Functions: []hcsschema.VirtualPciFunction{ + { + DeviceInstancePath: instanceID, + }, + }, + }, + } + return uvm.cs.Modify(ctx, request) +} + +func (uvm *utilityVM) RemoveDevice(ctx context.Context, instanceID, vmbusGUID string) error { + request := &hcsschema.ModifySettingRequest{ + ResourcePath: fmt.Sprintf(resourcepaths.VirtualPCIResourceFormat, vmbusGUID), + RequestType: requesttype.Remove, + } + return uvm.cs.Modify(ctx, request) +} diff --git a/internal/vm/hcs/plan9.go b/internal/vm/hcs/plan9.go new file mode 100644 index 0000000000..65d60c21ba --- /dev/null +++ b/internal/vm/hcs/plan9.go @@ -0,0 +1,38 @@ +package hcs + +import ( + "context" + + "github.com/Microsoft/hcsshim/internal/hcs/resourcepaths" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/requesttype" +) + +func (uvm *utilityVM) AddPlan9(ctx context.Context, path, name string, port int32, flags int32, allowed []string) error { + modification := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Add, + Settings: hcsschema.Plan9Share{ + Name: name, + AccessName: name, + Path: path, + Port: port, + Flags: flags, + AllowedFiles: allowed, + }, + ResourcePath: resourcepaths.Plan9ShareResourcePath, + } + return uvm.cs.Modify(ctx, modification) +} + +func (uvm *utilityVM) RemovePlan9(ctx context.Context, name string, port int32) error { + modification := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Remove, + Settings: hcsschema.Plan9Share{ + Name: name, + AccessName: name, + Port: port, + }, + ResourcePath: resourcepaths.Plan9ShareResourcePath, + } + return uvm.cs.Modify(ctx, modification) +} diff --git a/internal/vm/hcs/processor.go b/internal/vm/hcs/processor.go new file mode 100644 index 0000000000..fcbe12954e --- /dev/null +++ b/internal/vm/hcs/processor.go @@ -0,0 +1,6 @@ +package hcs + +func (uvmb *utilityVMBuilder) SetProcessorCount(count uint32) error { + uvmb.doc.VirtualMachine.ComputeTopology.Processor.Count = int32(count) + return nil +} diff --git a/internal/vm/hcs/scsi.go b/internal/vm/hcs/scsi.go new file mode 100644 index 0000000000..1fc2bba473 --- /dev/null +++ b/internal/vm/hcs/scsi.go @@ -0,0 +1,89 @@ +package hcs + +import ( + "context" + "fmt" + "strconv" + + "github.com/Microsoft/hcsshim/internal/hcs/resourcepaths" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/requesttype" + "github.com/Microsoft/hcsshim/internal/vm" + "github.com/pkg/errors" +) + +func (uvmb *utilityVMBuilder) AddSCSIController(id uint32) error { + if uvmb.doc.VirtualMachine.Devices.Scsi == nil { + uvmb.doc.VirtualMachine.Devices.Scsi = make(map[string]hcsschema.Scsi, 1) + } + uvmb.doc.VirtualMachine.Devices.Scsi[strconv.Itoa(int(id))] = hcsschema.Scsi{ + Attachments: make(map[string]hcsschema.Attachment), + } + return nil +} + +func (uvmb *utilityVMBuilder) AddSCSIDisk(ctx context.Context, controller uint32, lun uint32, path string, typ vm.SCSIDiskType, readOnly bool) error { + if uvmb.doc.VirtualMachine.Devices.Scsi == nil { + return errors.New("no SCSI controller found") + } + + ctrl, ok := uvmb.doc.VirtualMachine.Devices.Scsi[strconv.Itoa(int(controller))] + if !ok { + return fmt.Errorf("no scsi controller with index %d found", controller) + } + + ctrl.Attachments[strconv.Itoa(int(lun))] = hcsschema.Attachment{ + Path: path, + Type_: string(typ), + ReadOnly: readOnly, + } + + return nil +} + +func (uvmb *utilityVMBuilder) RemoveSCSIDisk(ctx context.Context, controller uint32, lun uint32, path string) error { + return vm.ErrNotSupported +} + +func (uvm *utilityVM) AddSCSIController(id uint32) error { + return vm.ErrNotSupported +} + +func (uvm *utilityVM) AddSCSIDisk(ctx context.Context, controller uint32, lun uint32, path string, typ vm.SCSIDiskType, readOnly bool) error { + diskTypeString, err := getSCSIDiskTypeString(typ) + if err != nil { + return err + } + request := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Add, + Settings: hcsschema.Attachment{ + Path: path, + Type_: diskTypeString, + ReadOnly: readOnly, + }, + ResourcePath: fmt.Sprintf(resourcepaths.SCSIResourceFormat, strconv.Itoa(int(controller)), lun), + } + return uvm.cs.Modify(ctx, request) +} + +func (uvm *utilityVM) RemoveSCSIDisk(ctx context.Context, controller uint32, lun uint32, path string) error { + request := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Remove, + ResourcePath: fmt.Sprintf(resourcepaths.SCSIResourceFormat, strconv.Itoa(int(controller)), lun), + } + + return uvm.cs.Modify(ctx, request) +} + +func getSCSIDiskTypeString(typ vm.SCSIDiskType) (string, error) { + switch typ { + case vm.SCSIDiskTypeVHD1: + fallthrough + case vm.SCSIDiskTypeVHDX: + return "VirtualDisk", nil + case vm.SCSIDiskTypePassThrough: + return "PassThru", nil + default: + return "", fmt.Errorf("unsupported SCSI disk type: %d", typ) + } +} diff --git a/internal/vm/hcs/serial.go b/internal/vm/hcs/serial.go new file mode 100644 index 0000000000..974493344a --- /dev/null +++ b/internal/vm/hcs/serial.go @@ -0,0 +1,22 @@ +package hcs + +import ( + "strconv" + "strings" + + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/pkg/errors" +) + +func (uvmb *utilityVMBuilder) SetSerialConsole(port uint32, listenerPath string) error { + if !strings.HasPrefix(listenerPath, `\\.\pipe\`) { + return errors.New("listener for serial console is not a named pipe") + } + + uvmb.doc.VirtualMachine.Devices.ComPorts = map[string]hcsschema.ComPort{ + strconv.Itoa(int(port)): { // "0" would be COM1 + NamedPipe: listenerPath, + }, + } + return nil +} diff --git a/internal/vm/hcs/stats.go b/internal/vm/hcs/stats.go new file mode 100644 index 0000000000..bb4cbfc59c --- /dev/null +++ b/internal/vm/hcs/stats.go @@ -0,0 +1,159 @@ +package hcs + +import ( + "context" + "strings" + + "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/go-winio/pkg/process" + "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/vm" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/windows" +) + +// checkProcess checks if the process identified by the given pid has a name +// matching `desiredProcessName`, and is running as a user with domain +// `desiredDomain` and user name `desiredUser`. If the process matches, it +// returns a handle to the process. If the process does not match, it returns +// 0. +func checkProcess(ctx context.Context, pid uint32, desiredProcessName string, desiredDomain string, desiredUser string) (p windows.Handle, err error) { + desiredProcessName = strings.ToUpper(desiredProcessName) + desiredDomain = strings.ToUpper(desiredDomain) + desiredUser = strings.ToUpper(desiredUser) + + p, err = windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION|windows.PROCESS_VM_READ, false, pid) + if err != nil { + return 0, err + } + + defer func(openedProcess windows.Handle) { + // If we don't return this process handle, close it so it doesn't leak. + if p == 0 { + windows.Close(openedProcess) + } + }(p) + + // Querying vmmem's image name as a win32 path returns ERROR_GEN_FAILURE + // for some reason, so we query it as an NT path instead. + name, err := process.QueryFullProcessImageName(p, process.ImageNameFormatNTPath) + if err != nil { + return 0, err + } + if strings.ToUpper(name) == desiredProcessName { + var t windows.Token + if err := windows.OpenProcessToken(p, windows.TOKEN_QUERY, &t); err != nil { + return 0, err + } + defer t.Close() + tUser, err := t.GetTokenUser() + if err != nil { + return 0, err + } + user, domain, _, err := tUser.User.Sid.LookupAccount("") + if err != nil { + return 0, err + } + log.G(ctx).WithFields(logrus.Fields{ + "name": name, + "domain": domain, + "user": user, + }).Debug("checking vmmem process identity") + if strings.ToUpper(domain) == desiredDomain && strings.ToUpper(user) == desiredUser { + return p, nil + } + } + return 0, nil +} + +// lookupVMMEM locates the vmmem process for a VM given the VM ID. It returns +// a handle to the vmmem process. The lookup is implemented by enumerating all +// processes on the system, and finding a process with full name "vmmem", +// running as "NT VIRTUAL MACHINE\". +func lookupVMMEM(ctx context.Context, vmID guid.GUID) (proc windows.Handle, err error) { + vmIDStr := strings.ToUpper(vmID.String()) + log.G(ctx).WithField("vmID", vmIDStr).Debug("looking up vmmem") + + pids, err := process.EnumProcesses() + if err != nil { + return 0, errors.Wrap(err, "failed to enumerate processes") + } + for _, pid := range pids { + p, err := checkProcess(ctx, pid, "vmmem", "NT VIRTUAL MACHINE", vmIDStr) + if err != nil { + // Checking the process could fail for a variety of reasons, such as + // the process exiting since we called EnumProcesses, or not having + // access to open the process (even as SYSTEM). In the case of an + // error, we just log and continue looking at the other processes. + log.G(ctx).WithField("pid", pid).Debug("failed to check process") + continue + } + if p != 0 { + log.G(ctx).WithField("pid", pid).Debug("found vmmem match") + return p, nil + } + } + return 0, errors.New("failed to find matching vmmem process") +} + +// getVMMEMProcess returns a handle to the vmmem process associated with this +// UVM. It only does the actual process lookup once, after which it caches the +// process handle in the UVM object. +func (uvm *utilityVM) getVMMEMProcess(ctx context.Context) (windows.Handle, error) { + uvm.vmmemOnce.Do(func() { + uvm.vmmemProcess, uvm.vmmemErr = lookupVMMEM(ctx, uvm.vmID) + }) + return uvm.vmmemProcess, uvm.vmmemErr +} + +func (uvm *utilityVM) Stats(ctx context.Context) (*stats.VirtualMachineStatistics, error) { + s := &stats.VirtualMachineStatistics{} + props, err := uvm.cs.PropertiesV2(ctx, hcsschema.PTStatistics, hcsschema.PTMemory) + if err != nil { + return nil, err + } + + s.Processor = &stats.VirtualMachineProcessorStatistics{} + s.Processor.TotalRuntimeNS = uint64(props.Statistics.Processor.TotalRuntime100ns * 100) + s.Memory = &stats.VirtualMachineMemoryStatistics{} + + if uvm.backingType == vm.MemoryBackingTypePhysical { + // If the uvm is physically backed we set the working set to the total amount allocated + // to the UVM. AssignedMemory returns the number of 4KB pages. Will always be 4KB + // regardless of what the UVMs actual page size is so we don't need that information. + if props.Memory != nil { + s.Memory.WorkingSetBytes = props.Memory.VirtualMachineMemory.AssignedMemory * 4096 + } + } else { + // The HCS properties does not return sufficient information to calculate + // working set size for a VA-backed UVM. To work around this, we instead + // locate the vmmem process for the VM, and query that process's working set + // instead, which will be the working set for the VM. + vmmemProc, err := uvm.getVMMEMProcess(ctx) + if err != nil { + return nil, err + } + memCounters, err := process.GetProcessMemoryInfo(vmmemProc) + if err != nil { + return nil, err + } + s.Memory.WorkingSetBytes = uint64(memCounters.WorkingSetSize) + } + + if props.Memory != nil { + s.Memory.VirtualNodeCount = props.Memory.VirtualNodeCount + s.Memory.VmMemory = &stats.VirtualMachineMemory{} + s.Memory.VmMemory.AvailableMemory = props.Memory.VirtualMachineMemory.AvailableMemory + s.Memory.VmMemory.AvailableMemoryBuffer = props.Memory.VirtualMachineMemory.AvailableMemoryBuffer + s.Memory.VmMemory.ReservedMemory = props.Memory.VirtualMachineMemory.ReservedMemory + s.Memory.VmMemory.AssignedMemory = props.Memory.VirtualMachineMemory.AssignedMemory + s.Memory.VmMemory.SlpActive = props.Memory.VirtualMachineMemory.SlpActive + s.Memory.VmMemory.BalancingEnabled = props.Memory.VirtualMachineMemory.BalancingEnabled + s.Memory.VmMemory.DmOperationInProgress = props.Memory.VirtualMachineMemory.DmOperationInProgress + } + + return s, nil +} diff --git a/internal/vm/hcs/storage.go b/internal/vm/hcs/storage.go new file mode 100644 index 0000000000..9b2317a036 --- /dev/null +++ b/internal/vm/hcs/storage.go @@ -0,0 +1,7 @@ +package hcs + +func (uvmb *utilityVMBuilder) SetStorageQos(iopsMaximum int64, bandwidthMaximum int64) error { + uvmb.doc.VirtualMachine.StorageQoS.BandwidthMaximum = int32(bandwidthMaximum) + uvmb.doc.VirtualMachine.StorageQoS.IopsMaximum = int32(iopsMaximum) + return nil +} diff --git a/internal/vm/hcs/supported.go b/internal/vm/hcs/supported.go new file mode 100644 index 0000000000..e179c252c3 --- /dev/null +++ b/internal/vm/hcs/supported.go @@ -0,0 +1,8 @@ +package hcs + +import "github.com/Microsoft/hcsshim/internal/vm" + +func (uvm *utilityVM) Supported(resource vm.Resource, op vm.ResourceOperation) bool { + // For now at least HCS supports everything we care about. + return true +} diff --git a/internal/vm/hcs/vmsocket.go b/internal/vm/hcs/vmsocket.go new file mode 100644 index 0000000000..22e268858a --- /dev/null +++ b/internal/vm/hcs/vmsocket.go @@ -0,0 +1,41 @@ +package hcs + +import ( + "context" + "net" + + "github.com/Microsoft/go-winio" + "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/hcsshim/internal/vm" + "github.com/pkg/errors" +) + +func (uvm *utilityVM) VMSocketListen(ctx context.Context, listenType vm.VMSocketType, connID interface{}) (net.Listener, error) { + switch listenType { + case vm.HvSocket: + serviceGUID, ok := connID.(guid.GUID) + if !ok { + return nil, errors.New("parameter passed to hvsocketlisten is not a GUID") + } + return uvm.hvSocketListen(ctx, serviceGUID) + case vm.VSock: + port, ok := connID.(uint32) + if !ok { + return nil, errors.New("parameter passed to vsocklisten is not the right type") + } + return uvm.vsockListen(ctx, port) + default: + return nil, errors.New("unknown vmsocket type requested") + } +} + +func (uvm *utilityVM) hvSocketListen(ctx context.Context, serviceID guid.GUID) (net.Listener, error) { + return winio.ListenHvsock(&winio.HvsockAddr{ + VMID: uvm.vmID, + ServiceID: serviceID, + }) +} + +func (uvm *utilityVM) vsockListen(ctx context.Context, port uint32) (net.Listener, error) { + return nil, vm.ErrNotSupported +} diff --git a/internal/vm/hcs/vpmem.go b/internal/vm/hcs/vpmem.go new file mode 100644 index 0000000000..09287ab8f8 --- /dev/null +++ b/internal/vm/hcs/vpmem.go @@ -0,0 +1,82 @@ +package hcs + +import ( + "context" + "fmt" + "strconv" + + "github.com/Microsoft/hcsshim/internal/hcs/resourcepaths" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/requesttype" + "github.com/Microsoft/hcsshim/internal/vm" + "github.com/pkg/errors" +) + +func (uvmb *utilityVMBuilder) AddVPMemController(maximumDevices uint32, maximumSizeBytes uint64) error { + uvmb.doc.VirtualMachine.Devices.VirtualPMem = &hcsschema.VirtualPMemController{ + MaximumCount: maximumDevices, + MaximumSizeBytes: maximumSizeBytes, + } + uvmb.doc.VirtualMachine.Devices.VirtualPMem.Devices = make(map[string]hcsschema.VirtualPMemDevice) + return nil +} + +func (uvmb *utilityVMBuilder) AddVPMemDevice(ctx context.Context, id uint32, path string, readOnly bool, imageFormat vm.VPMemImageFormat) error { + if uvmb.doc.VirtualMachine.Devices.VirtualPMem == nil { + return errors.New("VPMem controller has not been added") + } + imageFormatString, err := getVPMemImageFormatString(imageFormat) + if err != nil { + return err + } + uvmb.doc.VirtualMachine.Devices.VirtualPMem.Devices[strconv.Itoa(int(id))] = hcsschema.VirtualPMemDevice{ + HostPath: path, + ReadOnly: readOnly, + ImageFormat: imageFormatString, + } + return nil +} + +func (uvmb *utilityVMBuilder) RemoveVPMemDevice(ctx context.Context, id uint32, path string) error { + return vm.ErrNotSupported +} + +func (uvm *utilityVM) AddVPMemController(maximumDevices uint32, maximumSizeBytes uint64) error { + return vm.ErrNotSupported +} + +func (uvm *utilityVM) AddVPMemDevice(ctx context.Context, id uint32, path string, readOnly bool, imageFormat vm.VPMemImageFormat) error { + imageFormatString, err := getVPMemImageFormatString(imageFormat) + if err != nil { + return err + } + request := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Add, + Settings: hcsschema.VirtualPMemDevice{ + HostPath: path, + ReadOnly: readOnly, + ImageFormat: imageFormatString, + }, + ResourcePath: fmt.Sprintf(resourcepaths.VPMemControllerResourceFormat, id), + } + return uvm.cs.Modify(ctx, request) +} + +func (uvm *utilityVM) RemoveVPMemDevice(ctx context.Context, id uint32, path string) error { + request := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Remove, + ResourcePath: fmt.Sprintf(resourcepaths.VPMemControllerResourceFormat, id), + } + return uvm.cs.Modify(ctx, request) +} + +func getVPMemImageFormatString(imageFormat vm.VPMemImageFormat) (string, error) { + switch imageFormat { + case vm.VPMemImageFormatVHD1: + return "Vhd1", nil + case vm.VPMemImageFormatVHDX: + return "Vhdx", nil + default: + return "", fmt.Errorf("unsupported VPMem image format: %d", imageFormat) + } +} diff --git a/internal/vm/hcs/vsmb.go b/internal/vm/hcs/vsmb.go new file mode 100644 index 0000000000..b5cbade5b7 --- /dev/null +++ b/internal/vm/hcs/vsmb.go @@ -0,0 +1,65 @@ +package hcs + +import ( + "context" + + "github.com/Microsoft/hcsshim/internal/hcs/resourcepaths" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/requesttype" + "github.com/Microsoft/hcsshim/internal/vm" +) + +func (uvmb *utilityVMBuilder) AddVSMB(ctx context.Context, path string, name string, allowed []string, options *vm.VSMBOptions) error { + uvmb.doc.VirtualMachine.Devices.VirtualSmb = &hcsschema.VirtualSmb{ + DirectFileMappingInMB: 1024, // Sensible default, but could be a tuning parameter somewhere + Shares: []hcsschema.VirtualSmbShare{ + { + Name: name, + Path: path, + AllowedFiles: allowed, + Options: vmVSMBOptionsToHCS(options), + }, + }, + } + return nil +} + +func (uvmb *utilityVMBuilder) RemoveVSMB(ctx context.Context, name string) error { + return vm.ErrNotSupported +} + +func vmVSMBOptionsToHCS(options *vm.VSMBOptions) *hcsschema.VirtualSmbShareOptions { + return &hcsschema.VirtualSmbShareOptions{ + ReadOnly: options.ReadOnly, + ShareRead: options.ShareRead, + CacheIo: options.CacheIo, + NoOplocks: options.NoOplocks, + NoDirectmap: options.NoDirectMap, + TakeBackupPrivilege: options.TakeBackupPrivilege, + PseudoOplocks: options.PseudoOplocks, + PseudoDirnotify: options.PseudoDirnotify, + } +} + +func (uvm *utilityVM) AddVSMB(ctx context.Context, path string, name string, allowed []string, options *vm.VSMBOptions) error { + modification := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Add, + Settings: hcsschema.VirtualSmbShare{ + Name: name, + Options: vmVSMBOptionsToHCS(options), + Path: path, + AllowedFiles: allowed, + }, + ResourcePath: resourcepaths.VSMBShareResourcePath, + } + return uvm.cs.Modify(ctx, modification) +} + +func (uvm *utilityVM) RemoveVSMB(ctx context.Context, name string) error { + modification := &hcsschema.ModifySettingRequest{ + RequestType: requesttype.Remove, + Settings: hcsschema.VirtualSmbShare{Name: name}, + ResourcePath: resourcepaths.VSMBShareResourcePath, + } + return uvm.cs.Modify(ctx, modification) +} diff --git a/internal/vm/hcs/windows.go b/internal/vm/hcs/windows.go new file mode 100644 index 0000000000..029c71d417 --- /dev/null +++ b/internal/vm/hcs/windows.go @@ -0,0 +1,10 @@ +package hcs + +import ( + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" +) + +func (uvmb *utilityVMBuilder) SetCPUGroup(id string) error { + uvmb.doc.VirtualMachine.ComputeTopology.Processor.CpuGroup = &hcsschema.CpuGroup{Id: id} + return nil +} diff --git a/internal/vm/vm.go b/internal/vm/vm.go new file mode 100644 index 0000000000..0799d933f1 --- /dev/null +++ b/internal/vm/vm.go @@ -0,0 +1,212 @@ +package vm + +import ( + "context" + "errors" + "net" + + "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats" +) + +var ( + ErrNotSupported = errors.New("virtstack does not support the operation") + ErrAlreadySet = errors.New("field has already been set") + ErrUnknownGuestOS = errors.New("unknown guest operating system supplied") + + ErrNotInPreCreatedState = errors.New("VM is not in pre-created state") + ErrNotInCreatedState = errors.New("VM is not in created state") + ErrNotInRunningState = errors.New("VM is not in running state") + ErrNotInPausedState = errors.New("VM is not in paused state") +) + +const ( + HCS = "hcs" + RemoteVM = "remotevm" +) + +// UVM is an abstraction around a lightweight virtual machine. It houses core lifecycle methods such as Create +// Start, and Stop and also several optional nested interfaces that can be used to determine what the virtual machine +// supports and to configure these resources. +type UVM interface { + // ID will return a string identifier for the Utility VM. + ID() string + + // Start will power on the Utility VM and put it into a running state. This will boot the guest OS and start all of the + // devices configured on the machine. + Start(ctx context.Context) error + + // Stop will shutdown the Utility VM and place it into a terminated state. + Stop(ctx context.Context) error + + // Pause will place the Utility VM into a paused state. The guest OS will be halted and any devices will have be in a + // a suspended state. Save can be used to snapshot the current state of the virtual machine, and Resume can be used to + // place the virtual machine back into a running state. + Pause(ctx context.Context) error + + // Resume will put a previously paused Utility VM back into a running state. The guest OS will resume operation from the point + // in time it was paused and all devices should be un-suspended. + Resume(ctx context.Context) error + + // Save will snapshot the state of the Utility VM at the point in time when the VM was paused. + Save(ctx context.Context) error + + // Wait synchronously waits for the Utility VM to shutdown or terminate. A call to stop will trigger this + // to unblock. + Wait() error + + // Stats returns statistics about the Utility VM. This includes things like assigned memory, available memory, + // processor runtime etc. + Stats(ctx context.Context) (*stats.VirtualMachineStatistics, error) + + // Supported returns if the virt stack supports a given operation on a resource. + Supported(resource Resource, operation ResourceOperation) bool + + // ExitError will return any error if the Utility VM exited unexpectedly, or if the Utility VM experienced an + // error after Wait returned, it will return the wait error. + ExitError() error +} + +// Resource refers to the type of a resource on a Utility VM. +type Resource uint8 + +const ( + VPMem = iota + SCSI + Network + VSMB + PCI + Plan9 + CPUGroup +) + +// Operation refers to the type of operation to perform on a given resource. +type ResourceOperation uint8 + +const ( + Add ResourceOperation = iota + Remove + Update +) + +// GuestOS signifies the guest operating system that a Utility VM will be running. +type GuestOS string + +const ( + Windows GuestOS = "windows" + Linux GuestOS = "linux" +) + +// State signifies the states that a Utility VM can be in. The state of the Utility VM should be the source of truth for what +// operations can be performed at a given moment. +type State uint8 + +const ( + StatePreCreated State = iota + StateCreated + StateRunning + StateTerminated + StatePaused +) + +// SCSIDiskType refers to the disk type of the scsi device. This is either a vhd, vhdx, or a physical disk. +type SCSIDiskType uint8 + +const ( + SCSIDiskTypeVHD1 SCSIDiskType = iota + SCSIDiskTypeVHDX + SCSIDiskTypePassThrough +) + +// SCSIManager manages adding and removing SCSI devices for a Utility VM. +type SCSIManager interface { + // AddSCSIController adds a SCSI controller to the Utility VM configuration document. + AddSCSIController(id uint32) error + // AddSCSIDisk adds a SCSI disk to the configuration document if in a precreated state, or hot adds a + // SCSI disk to the Utility VM if the VM is running. + AddSCSIDisk(ctx context.Context, controller uint32, lun uint32, path string, typ SCSIDiskType, readOnly bool) error + // RemoveSCSIDisk removes a SCSI disk from a Utility VM. + RemoveSCSIDisk(ctx context.Context, controller uint32, lun uint32, path string) error +} + +// VPMemImageFormat refers to the image type of the vpmem block device. This is either a vhd or vhdx. +type VPMemImageFormat uint8 + +const ( + VPMemImageFormatVHD1 VPMemImageFormat = iota + VPMemImageFormatVHDX +) + +// VPMemManager manages adding and removing virtual persistent memory devices for a Utility VM. +type VPMemManager interface { + // AddVPMemController adds a new virtual pmem controller to the Utility VM. + // `maximumDevices` specifies how many vpmem devices will be present in the guest. + // `maximumSizeBytes` specifies the maximum size allowed for a vpmem device. + AddVPMemController(maximumDevices uint32, maximumSizeBytes uint64) error + // AddVPMemDevice adds a virtual pmem device to the Utility VM. + AddVPMemDevice(ctx context.Context, id uint32, path string, readOnly bool, imageFormat VPMemImageFormat) error + // RemoveVpmemDevice removes a virtual pmem device from the Utility VM. + RemoveVPMemDevice(ctx context.Context, id uint32, path string) error +} + +// NetworkManager manages adding and removing network adapters for a Utility VM. +type NetworkManager interface { + // AddNIC adds a network adapter to the Utility VM. `nicID` should be a string representation of a + // Windows GUID. + AddNIC(ctx context.Context, nicID string, endpointID string, macAddr string) error + // RemoveNIC removes a network adapter from the Utility VM. `nicID` should be a string representation of a + // Windows GUID. + RemoveNIC(ctx context.Context, nicID string, endpointID string, macAddr string) error +} + +// PCIManager manages assiging pci devices to a Utility VM. This is Windows specific at the moment. +type PCIManager interface { + // AddDevice adds the pci device identified by `instanceID` to the Utility VM. + // https://docs.microsoft.com/en-us/windows-hardware/drivers/install/instance-ids + AddDevice(ctx context.Context, instanceID string, vmbusGUID string) error + // RemoveDevice removes the pci device identified by `instanceID` from the Utility VM. + RemoveDevice(ctx context.Context, instanceID string, vmbusGUID string) error +} + +// VMSocketType refers to which hypervisor socket transport type to use. +type VMSocketType uint8 + +const ( + HvSocket VMSocketType = iota + VSock +) + +// VMSocketManager manages configuration for a hypervisor socket transport. This includes sockets such as +// HvSocket and Vsock. +type VMSocketManager interface { + // VMSocketListen will create the requested vmsocket type and listen on the address specified by `connID`. + // For HvSocket the type expected is a GUID, for Vsock it's a port of type uint32. + VMSocketListen(ctx context.Context, socketType VMSocketType, connID interface{}) (net.Listener, error) +} + +// VSMBOptions +type VSMBOptions struct { + ReadOnly bool + CacheIo bool + NoDirectMap bool + PseudoOplocks bool + ShareRead bool + TakeBackupPrivilege bool + NoOplocks bool + PseudoDirnotify bool +} + +// VSMBManager manages adding virtual smb shares to a Utility VM. +type VSMBManager interface { + // AddVSMB adds a virtual smb share to a running Utility VM. + AddVSMB(ctx context.Context, hostPath string, name string, allowedFiles []string, options *VSMBOptions) error + // RemoveVSMB removes a virtual smb share from a running Utility VM. + RemoveVSMB(ctx context.Context, name string) error +} + +// Plan9Manager manages adding plan 9 shares to a Utility VM. +type Plan9Manager interface { + // AddPlan9 adds a plan 9 share to a running Utility VM. + AddPlan9(ctx context.Context, path, name string, port int32, flags int32, allowed []string) error + // RemovePlan9 removes a plan 9 share from a running Utility VM. + RemovePlan9(ctx context.Context, name string, port int32) error +}