Remotevm UVM implementation

internal/cpugroup/cpugroup.go

@@ -53,9 +53,8 @@ func Create(ctx context.Context, id string, logicalProcessors []uint32) error {
 	return nil
 }
 
-// getCPUGroupConfig finds the cpugroup config information for group with `id`
-//nolint:unused
-func getCPUGroupConfig(ctx context.Context, id string) (*hcsschema.CpuGroupConfig, error) {
+// GetCPUGroupConfig finds the cpugroup config information for group with `id`
+func GetCPUGroupConfig(ctx context.Context, id string) (*hcsschema.CpuGroupConfig, error) {
 	query := hcsschema.PropertyQuery{
 		PropertyTypes: []hcsschema.PropertyType{hcsschema.PTCPUGroup},
 	}

internal/cpugroup/cpugroup_test.go

@@ -31,7 +31,7 @@ func TestCPUGroupCreateWithIDAndDelete(t *testing.T) {
 		}
 	}()
 
-	_, err = getCPUGroupConfig(ctx, id.String())
+	_, err = GetCPUGroupConfig(ctx, id.String())
 	if err != nil {
 		t.Fatalf("failed to find cpugroup on host with: %v", err)
 	}

internal/hcs/schema2/cpu_group_config.go

@@ -14,5 +14,5 @@ type CpuGroupConfig struct {
 	Affinity        *CpuGroupAffinity  `json:"Affinity,omitempty"`
 	GroupProperties []CpuGroupProperty `json:"GroupProperties,omitempty"`
 	// Hypervisor CPU group IDs exposed to clients
-	HypervisorGroupId int32 `json:"HypervisorGroupId,omitempty"`
+	HypervisorGroupId uint64 `json:"HypervisorGroupId,omitempty"`
 }

internal/vm/builder.go

@@ -67,7 +67,7 @@ type StorageQosManager interface {
 // WindowsConfigManager manages options specific to a Windows host (cpugroups etc.)
 type WindowsConfigManager interface {
 	// SetCPUGroup sets the CPU group that the Utility VM will belong to on a Windows host.
-	SetCPUGroup(id string) error
+	SetCPUGroup(ctx context.Context, id string) error
 }
 
 // LinuxConfigManager manages options specific to a Linux host.

internal/vm/hcs/builder.go

@@ -10,6 +10,8 @@ import (
 	"github.com/pkg/errors"
 )
 
+var _ vm.UVMBuilder = &utilityVMBuilder{}
+
 type utilityVMBuilder struct {
 	id      string
 	guestOS vm.GuestOS
@@ -81,7 +83,6 @@ func (uvmb *utilityVMBuilder) Create(ctx context.Context) (_ vm.UVM, err error)
 		guestOS:     uvmb.guestOS,
 		cs:          cs,
 		backingType: backingType,
-		state:       vm.StateCreated,
 	}
 
 	properties, err := cs.Properties(ctx)

internal/vm/hcs/hcs.go

@@ -12,9 +12,10 @@ import (
 	"golang.org/x/sys/windows"
 )
 
+var _ vm.UVM = &utilityVM{}
+
 type utilityVM struct {
 	id           string
-	state        vm.State
 	guestOS      vm.GuestOS
 	cs           *hcs.System
 	backingType  vm.MemoryBackingType

internal/vm/hcs/windows.go

@@ -1,10 +1,12 @@
 package hcs
 
 import (
+	"context"
+
 	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
 )
 
-func (uvmb *utilityVMBuilder) SetCPUGroup(id string) error {
+func (uvmb *utilityVMBuilder) SetCPUGroup(ctx context.Context, id string) error {
 	uvmb.doc.VirtualMachine.ComputeTopology.Processor.CpuGroup = &hcsschema.CpuGroup{Id: id}
 	return nil
 }

internal/vm/remotevm/boot.go

@@ -0,0 +1,27 @@
+package remotevm
+
+import (
+	"github.com/Microsoft/hcsshim/internal/vmservice"
+)
+
+func (uvmb *utilityVMBuilder) SetLinuxKernelDirectBoot(kernel, initRD, cmd string) error {
+	uvmb.config.BootConfig = &vmservice.VMConfig_DirectBoot{
+		DirectBoot: &vmservice.DirectBoot{
+			KernelPath:    kernel,
+			InitrdPath:    initRD,
+			KernelCmdline: cmd,
+		},
+	}
+	return nil
+}
+
+func (uvmb *utilityVMBuilder) SetUEFIBoot(firmware, devPath, args string) error {
+	uvmb.config.BootConfig = &vmservice.VMConfig_Uefi{
+		Uefi: &vmservice.UEFI{
+			FirmwarePath: firmware,
+			DevicePath:   devPath,
+			OptionalData: args,
+		},
+	}
+	return nil
+}

internal/vm/remotevm/builder.go

@@ -0,0 +1,109 @@
+package remotevm
+
+import (
+	"context"
+	"io"
+	"io/ioutil"
+	"net"
+	"os/exec"
+
+	"github.com/Microsoft/hcsshim/internal/jobobject"
+	"github.com/Microsoft/hcsshim/internal/log"
+	"github.com/Microsoft/hcsshim/internal/vm"
+	"github.com/Microsoft/hcsshim/internal/vmservice"
+	"github.com/containerd/ttrpc"
+	ptypes "github.com/gogo/protobuf/types"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+var _ vm.UVMBuilder = &utilityVMBuilder{}
+
+type utilityVMBuilder struct {
+	id      string
+	guestOS vm.GuestOS
+	job     *jobobject.JobObject
+	config  *vmservice.VMConfig
+	client  vmservice.VMService
+}
+
+func NewUVMBuilder(ctx context.Context, id, owner, binPath, addr string, guestOS vm.GuestOS) (vm.UVMBuilder, error) {
+	var job *jobobject.JobObject
+	if binPath != "" {
+		log.G(ctx).WithFields(logrus.Fields{
+			"binary":  binPath,
+			"address": addr,
+		}).Debug("starting remotevm server process")
+
+		opts := &jobobject.Options{
+			Name: id,
+		}
+		job, err := jobobject.Create(ctx, opts)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create job object for remotevm process")
+		}
+
+		cmd := exec.Command(binPath, "--ttrpc", addr)
+		p, err := cmd.StdoutPipe()
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create stdout pipe")
+		}
+
+		if err := cmd.Start(); err != nil {
+			return nil, errors.Wrap(err, "failed to start remotevm server process")
+		}
+
+		if err := job.Assign(uint32(cmd.Process.Pid)); err != nil {
+			return nil, errors.Wrap(err, "failed to assign remotevm process to job")
+		}
+
+		if err := job.SetTerminateOnLastHandleClose(); err != nil {
+			return nil, errors.Wrap(err, "failed to set terminate on last handle closed for remotevm job object")
+		}
+
+		// Wait for stdout to close. This is our signal that the server is successfully up and running.
+		_, _ = io.Copy(ioutil.Discard, p)
+	}
+
+	conn, err := net.Dial("unix", addr)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to dial remotevm address %q", addr)
+	}
+
+	c := ttrpc.NewClient(conn, ttrpc.WithOnClose(func() { conn.Close() }))
+	vmClient := vmservice.NewVMClient(c)
+
+	return &utilityVMBuilder{
+		id:      id,
+		guestOS: guestOS,
+		config: &vmservice.VMConfig{
+			MemoryConfig:    &vmservice.MemoryConfig{},
+			DevicesConfig:   &vmservice.DevicesConfig{},
+			ProcessorConfig: &vmservice.ProcessorConfig{},
+			SerialConfig:    &vmservice.SerialConfig{},
+			ExtraData:       make(map[string]string),
+		},
+		job:    job,
+		client: vmClient,
+	}, nil
+}
+
+func (uvmb *utilityVMBuilder) Create(ctx context.Context) (vm.UVM, error) {
+	// Grab what capabilities the virtstack supports up front.
+	capabilities, err := uvmb.client.CapabilitiesVM(ctx, &ptypes.Empty{})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get virtstack capabilities from vmservice")
+	}
+
+	if _, err := uvmb.client.CreateVM(ctx, &vmservice.CreateVMRequest{Config: uvmb.config, LogID: uvmb.id}); err != nil {
+		return nil, errors.Wrap(err, "failed to create remote VM")
+	}
+
+	return &utilityVM{
+		id:           uvmb.id,
+		job:          uvmb.job,
+		config:       uvmb.config,
+		client:       uvmb.client,
+		capabilities: capabilities,
+	}, nil
+}

internal/vm/remotevm/memory.go

@@ -0,0 +1,36 @@
+package remotevm
+
+import (
+	"github.com/Microsoft/hcsshim/internal/vm"
+	"github.com/Microsoft/hcsshim/internal/vmservice"
+)
+
+func (uvmb *utilityVMBuilder) SetMemoryLimit(memoryMB uint64) error {
+	if uvmb.config.MemoryConfig == nil {
+		uvmb.config.MemoryConfig = &vmservice.MemoryConfig{}
+	}
+	uvmb.config.MemoryConfig.MemoryMb = memoryMB
+	return nil
+}
+
+func (uvmb *utilityVMBuilder) SetMemoryConfig(config *vm.MemoryConfig) error {
+	if uvmb.config.MemoryConfig == nil {
+		uvmb.config.MemoryConfig = &vmservice.MemoryConfig{}
+	}
+	uvmb.config.MemoryConfig.AllowOvercommit = config.BackingType == vm.MemoryBackingTypeVirtual
+	uvmb.config.MemoryConfig.ColdHint = config.ColdHint
+	uvmb.config.MemoryConfig.ColdDiscardHint = config.ColdDiscardHint
+	uvmb.config.MemoryConfig.DeferredCommit = config.DeferredCommit
+	uvmb.config.MemoryConfig.HotHint = config.HotHint
+	return vm.ErrNotSupported
+}
+
+func (uvmb *utilityVMBuilder) SetMMIOConfig(lowGapMB uint64, highBaseMB uint64, highGapMB uint64) error {
+	if uvmb.config.MemoryConfig == nil {
+		uvmb.config.MemoryConfig = &vmservice.MemoryConfig{}
+	}
+	uvmb.config.MemoryConfig.HighMmioBaseInMb = highBaseMB
+	uvmb.config.MemoryConfig.LowMmioGapInMb = lowGapMB
+	uvmb.config.MemoryConfig.HighMmioGapInMb = highGapMB
+	return nil
+}

internal/vm/remotevm/network.go

@@ -0,0 +1,120 @@
+package remotevm
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/Microsoft/go-winio/pkg/guid"
+	"github.com/Microsoft/hcsshim/hcn"
+	"github.com/Microsoft/hcsshim/internal/vmservice"
+	"github.com/pkg/errors"
+)
+
+func getSwitchID(endpointID, portID string) (string, error) {
+	// Get updated endpoint with new fields (need switch ID)
+	ep, err := hcn.GetEndpointByID(endpointID)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to get endpoint %q", endpointID)
+	}
+
+	type ExtraInfo struct {
+		Allocators []struct {
+			SwitchId         string
+			EndpointPortGuid string
+		}
+	}
+
+	var exi ExtraInfo
+	if err := json.Unmarshal(ep.Health.Extra.Resources, &exi); err != nil {
+		return "", errors.Wrapf(err, "failed to unmarshal resource data from endpoint %q", endpointID)
+	}
+
+	if len(exi.Allocators) == 0 {
+		return "", errors.New("no resource data found for endpoint")
+	}
+
+	// NIC should only ever belong to one switch but there are cases where there's more than one allocator
+	// in the returned data. It seems they only ever contain empty strings so search for the first entry
+	// that actually contains a switch ID and that has the matching port GUID we made earlier.
+	var switchID string
+	for _, allocator := range exi.Allocators {
+		if allocator.SwitchId != "" && strings.ToLower(allocator.EndpointPortGuid) == portID {
+			switchID = allocator.SwitchId
+			break
+		}
+	}
+	return switchID, nil
+}
+
+func (uvm *utilityVM) AddNIC(ctx context.Context, nicID, endpointID, macAddr string) error {
+	portID, err := guid.NewV4()
+	if err != nil {
+		return errors.Wrap(err, "failed to generate guid for port")
+	}
+
+	vmEndpointRequest := hcn.VmEndpointRequest{
+		PortId:           portID,
+		VirtualNicName:   fmt.Sprintf("%s--%s", nicID, portID),
+		VirtualMachineId: guid.GUID{},
+	}
+
+	m, err := json.Marshal(vmEndpointRequest)
+	if err != nil {
+		return errors.Wrap(err, "failed to marshal endpoint request json")
+	}
+
+	if err := hcn.ModifyEndpointSettings(endpointID, &hcn.ModifyEndpointSettingRequest{
+		ResourceType: hcn.EndpointResourceTypePort,
+		RequestType:  hcn.RequestTypeAdd,
+		Settings:     json.RawMessage(m),
+	}); err != nil {
+		return errors.Wrap(err, "failed to configure switch port")
+	}
+
+	switchID, err := getSwitchID(endpointID, portID.String())
+	if err != nil {
+		return err
+	}
+
+	nic := &vmservice.NICConfig{
+		NicID:      nicID,
+		MacAddress: macAddr,
+		PortID:     portID.String(),
+		SwitchID:   switchID,
+	}
+
+	if _, err := uvm.client.ModifyResource(ctx,
+		&vmservice.ModifyResourceRequest{
+			Type: vmservice.ModifyType_ADD,
+			Resource: &vmservice.ModifyResourceRequest_NicConfig{
+				NicConfig: nic,
+			},
+		},
+	); err != nil {
+		return errors.Wrap(err, "failed to add network adapter")
+	}
+
+	return nil
+}
+
+func (uvm *utilityVM) RemoveNIC(ctx context.Context, nicID, endpointID, macAddr string) error {
+	nic := &vmservice.NICConfig{
+		NicID:      nicID,
+		MacAddress: macAddr,
+	}
+
+	if _, err := uvm.client.ModifyResource(ctx,
+		&vmservice.ModifyResourceRequest{
+			Type: vmservice.ModifyType_REMOVE,
+			Resource: &vmservice.ModifyResourceRequest_NicConfig{
+				NicConfig: nic,
+			},
+		},
+	); err != nil {
+		return errors.Wrap(err, "failed to remove network adapter")
+	}
+
+	return nil
+}

internal/vm/remotevm/processor.go

@@ -0,0 +1,15 @@
+package remotevm
+
+import (
+	"context"
+
+	"github.com/Microsoft/hcsshim/internal/vmservice"
+)
+
+func (uvmb *utilityVMBuilder) SetProcessorCount(ctx context.Context, count uint32) error {
+	if uvmb.config.ProcessorConfig == nil {
+		uvmb.config.ProcessorConfig = &vmservice.ProcessorConfig{}
+	}
+	uvmb.config.ProcessorConfig.ProcessorCount = count
+	return nil
+}