From 2a1685eb8478006f206bf2bd8275bd0b9bc7435e Mon Sep 17 00:00:00 2001 From: Amit Barve Date: Thu, 27 May 2021 13:59:52 -0700 Subject: [PATCH] Support for extensible virtual disks as data disks. This commit adds support in hcsshim to mount an extensible virtual disk data disk into a container. In container config the host_path in the mount entry should use the format evd:/// to specify an extensible virtual disk. Signed-off-by: Amit Barve --- internal/hcs/schema2/attachment.go | 6 + internal/hcsoci/hcsdoc_wcow.go | 12 +- internal/hcsoci/resources_wcow.go | 8 ++ internal/uvm/create_wcow.go | 10 +- internal/uvm/scsi.go | 185 ++++++++++++++++++++++------- 5 files changed, 173 insertions(+), 48 deletions(-) diff --git a/internal/hcs/schema2/attachment.go b/internal/hcs/schema2/attachment.go index bcfeb34d54..70884aad75 100644 --- a/internal/hcs/schema2/attachment.go +++ b/internal/hcs/schema2/attachment.go @@ -27,4 +27,10 @@ type Attachment struct { CaptureIoAttributionContext bool `json:"CaptureIoAttributionContext,omitempty"` ReadOnly bool `json:"ReadOnly,omitempty"` + + SupportCompressedVolumes bool `json:"SupportCompressedVolumes,omitempty"` + + AlwaysAllowSparseFiles bool `json:"AlwaysAllowSparseFiles,omitempty"` + + ExtensibleVirtualDiskType string `json:"ExtensibleVirtualDiskType,omitempty"` } diff --git a/internal/hcsoci/hcsdoc_wcow.go b/internal/hcsoci/hcsdoc_wcow.go index 7423668dda..90d545d9cc 100644 --- a/internal/hcsoci/hcsdoc_wcow.go +++ b/internal/hcsoci/hcsdoc_wcow.go @@ -61,8 +61,16 @@ func createMountsConfig(ctx context.Context, coi *createOptionsInternal) (*mount return nil, fmt.Errorf("failed to eval symlinks for mount source %q: %s", mount.Source, err) } mdv2.HostPath = src - } else if mount.Type == "virtual-disk" || mount.Type == "physical-disk" { - uvmPath, err := coi.HostingSystem.GetScsiUvmPath(ctx, mount.Source) + } else if mount.Type == "virtual-disk" || mount.Type == "physical-disk" || mount.Type == "extensible-virtual-disk" { + mountPath := mount.Source + var err error + if mount.Type == "extensible-virtual-disk" { + _, mountPath, err = uvm.ParseExtensibleVirtualDiskPath(mount.Source) + if err != nil { + return nil, err + } + } + uvmPath, err := coi.HostingSystem.GetScsiUvmPath(ctx, mountPath) if err != nil { return nil, err } diff --git a/internal/hcsoci/resources_wcow.go b/internal/hcsoci/resources_wcow.go index 46599be037..29d6ac526c 100644 --- a/internal/hcsoci/resources_wcow.go +++ b/internal/hcsoci/resources_wcow.go @@ -119,6 +119,7 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R case "": case "physical-disk": case "virtual-disk": + case "extensible-virtual-disk": default: return fmt.Errorf("invalid OCI spec - Type '%s' not supported", mount.Type) } @@ -147,6 +148,13 @@ func setupMounts(ctx context.Context, coi *createOptionsInternal, r *resources.R return errors.Wrapf(err, "adding SCSI virtual disk mount %+v", mount) } r.Add(scsiMount) + } else if mount.Type == "extensible-virtual-disk" { + l.Debug("hcsshim::allocateWindowsResource Hot-adding ExtensibleVirtualDisk") + scsiMount, err := coi.HostingSystem.AddSCSIExtensibleVirtualDisk(ctx, mount.Source, uvmPath, readOnly) + if err != nil { + return errors.Wrapf(err, "adding SCSI EVD mount failed %+v", mount) + } + r.Add(scsiMount) } else { if uvm.IsPipe(mount.Source) { pipe, err := coi.HostingSystem.AddPipe(ctx, mount.Source) diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index 04eb1dd9d1..d1866b070e 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -292,7 +292,15 @@ func CreateWCOW(ctx context.Context, opts *OptionsWCOW) (_ *UtilityVM, err error }, } - uvm.scsiLocations[0][0] = newSCSIMount(uvm, doc.VirtualMachine.Devices.Scsi["0"].Attachments["0"].Path, "", "", 1, 0, 0, false) + uvm.scsiLocations[0][0] = newSCSIMount(uvm, + doc.VirtualMachine.Devices.Scsi["0"].Attachments["0"].Path, + "", + doc.VirtualMachine.Devices.Scsi["0"].Attachments["0"].Type_, + "", + 1, + 0, + 0, + false) } else { doc.VirtualMachine.RestoreState = &hcsschema.RestoreState{} doc.VirtualMachine.RestoreState.TemplateSystemId = opts.TemplateConfig.UVMID diff --git a/internal/uvm/scsi.go b/internal/uvm/scsi.go index fdb9c8bc13..2e0e72934b 100644 --- a/internal/uvm/scsi.go +++ b/internal/uvm/scsi.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "github.com/Microsoft/go-winio/pkg/security" "github.com/Microsoft/hcsshim/internal/copyfile" @@ -39,7 +40,7 @@ const ( VMAccessTypeIndividual ) -const scsiCurrentSerialVersionID = 1 +const scsiCurrentSerialVersionID = 2 var ( ErrNoAvailableLocation = fmt.Errorf("no available location") @@ -78,10 +79,40 @@ type SCSIMount struct { refCount uint32 // specifies if this is a readonly layer readOnly bool - // "VirtualDisk" or "PassThru" disk attachment type. + // "VirtualDisk" or "PassThru" or "ExtensibleVirtualDisk" disk attachment type. attachmentType string + // If attachmentType is "ExtensibleVirtualDisk" then extensibleVirtualDiskType should + // specify the type of it (for e.g "space" for storage spaces). Otherwise this should be + // empty. + extensibleVirtualDiskType string // serialization ID serialVersionID uint32 + // Make sure that serialVersionID is always the last field and its value is + // incremented every time this structure is updated +} + +// addSCSIRequest is an internal struct used to hold all the parameters that are sent to +// the addSCSIActual method. +type addSCSIRequest struct { + // host path to the disk that should be added as a SCSI disk. + hostPath string + // the path inside the uvm at which this disk should show up. Can be empty. + uvmPath string + // attachmentType is required and `must` be `VirtualDisk` for vhd/vhdx + // attachments, `PassThru` for physical disk and `ExtensibleVirtualDisk` for + // Extensible virtual disks. + attachmentType string + // indicates if the attachment should be added read only. + readOnly bool + // guestOptions is a slice that contains optional information to pass to the guest + // service. + guestOptions []string + // indicates what access to grant the vm for the hostpath. Only required for + // `VirtualDisk` and `PassThru` disk types. + vmAccess VMAccessType + // `evdType` indicates the type of the extensible virtual disk if `attachmentType` + // is "ExtensibleVirtualDisk" should be empty otherwise. + evdType string } // RefCount returns the current refcount for the SCSI mount. @@ -91,27 +122,29 @@ func (sm *SCSIMount) RefCount() uint32 { func (sm *SCSIMount) logFormat() logrus.Fields { return logrus.Fields{ - "HostPath": sm.HostPath, - "UVMPath": sm.UVMPath, - "isLayer": sm.isLayer, - "refCount": sm.refCount, - "Controller": sm.Controller, - "LUN": sm.LUN, - "SerialVersionID": sm.serialVersionID, + "HostPath": sm.HostPath, + "UVMPath": sm.UVMPath, + "isLayer": sm.isLayer, + "refCount": sm.refCount, + "Controller": sm.Controller, + "LUN": sm.LUN, + "ExtensibleVirtualDiskType": sm.extensibleVirtualDiskType, + "SerialVersionID": sm.serialVersionID, } } -func newSCSIMount(uvm *UtilityVM, hostPath, uvmPath, attachmentType string, refCount uint32, controller int, lun int32, readOnly bool) *SCSIMount { +func newSCSIMount(uvm *UtilityVM, hostPath, uvmPath, attachmentType, evdType string, refCount uint32, controller int, lun int32, readOnly bool) *SCSIMount { return &SCSIMount{ - vm: uvm, - HostPath: hostPath, - UVMPath: uvmPath, - refCount: refCount, - Controller: controller, - LUN: int32(lun), - readOnly: readOnly, - attachmentType: attachmentType, - serialVersionID: scsiCurrentSerialVersionID, + vm: uvm, + HostPath: hostPath, + UVMPath: uvmPath, + refCount: refCount, + Controller: controller, + LUN: int32(lun), + readOnly: readOnly, + attachmentType: attachmentType, + extensibleVirtualDiskType: evdType, + serialVersionID: scsiCurrentSerialVersionID, } } @@ -227,7 +260,15 @@ func (uvm *UtilityVM) RemoveSCSI(ctx context.Context, hostPath string) error { // // `vmAccess` indicates what access to grant the vm for the hostpath func (uvm *UtilityVM) AddSCSI(ctx context.Context, hostPath string, uvmPath string, readOnly bool, guestOptions []string, vmAccess VMAccessType) (*SCSIMount, error) { - return uvm.addSCSIActual(ctx, hostPath, uvmPath, "VirtualDisk", readOnly, guestOptions, vmAccess) + addReq := &addSCSIRequest{ + hostPath: hostPath, + uvmPath: uvmPath, + attachmentType: "VirtualDisk", + readOnly: readOnly, + guestOptions: guestOptions, + vmAccess: VMAccessTypeIndividual, + } + return uvm.addSCSIActual(ctx, addReq) } // AddSCSIPhysicalDisk attaches a physical disk from the host directly to the @@ -242,28 +283,60 @@ func (uvm *UtilityVM) AddSCSI(ctx context.Context, hostPath string, uvmPath stri // `guestOptions` is a slice that contains optional information to pass // to the guest service func (uvm *UtilityVM) AddSCSIPhysicalDisk(ctx context.Context, hostPath, uvmPath string, readOnly bool, guestOptions []string) (*SCSIMount, error) { - return uvm.addSCSIActual(ctx, hostPath, uvmPath, "PassThru", readOnly, guestOptions, VMAccessTypeIndividual) + addReq := &addSCSIRequest{ + hostPath: hostPath, + uvmPath: uvmPath, + attachmentType: "PassThru", + readOnly: readOnly, + guestOptions: guestOptions, + vmAccess: VMAccessTypeIndividual, + } + return uvm.addSCSIActual(ctx, addReq) } -// addSCSIActual is the implementation behind the external functions AddSCSI and -// AddSCSIPhysicalDisk. +// AddSCSIExtensibleVirtualDisk adds an extensible virtual disk as a SCSI mount +// to the utility VM at the next available location. All such disks which are not actual virtual disks +// but provide the same SCSI interface are added to the UVM as Extensible Virtual disks. // -// We are in control of everything ourselves. Hence we have ref- counting and -// so-on tracking what SCSI locations are available or used. +// `hostPath` is required. Depending on the type of the extensible virtual disk the format of `hostPath` can +// be different. +// For example, in case of storage spaces the host path must be in the +// `evd://space/{storage_pool_unique_ID}{virtual_disk_unique_ID}` format. // -// `attachmentType` is required and `must` be `VirtualDisk` for vhd/vhdx -// attachments and `PassThru` for physical disk. +// `uvmPath` must be provided in order to be able to use this disk in a container. // -// `readOnly` indicates the attachment should be added read only. -// -// `guestOptions` is a slice that contains optional information to pass -// to the guest service +// `readOnly` set to `true` if the virtual disk should be attached read only. // // `vmAccess` indicates what access to grant the vm for the hostpath +func (uvm *UtilityVM) AddSCSIExtensibleVirtualDisk(ctx context.Context, hostPath, uvmPath string, readOnly bool) (*SCSIMount, error) { + if uvmPath == "" { + return nil, errors.New("uvmPath can not be empty for extensible virtual disk") + } + evdType, mountPath, err := ParseExtensibleVirtualDiskPath(hostPath) + if err != nil { + return nil, err + } + addReq := &addSCSIRequest{ + hostPath: mountPath, + uvmPath: uvmPath, + attachmentType: "ExtensibleVirtualDisk", + readOnly: readOnly, + guestOptions: []string{}, + vmAccess: VMAccessTypeIndividual, + evdType: evdType, + } + return uvm.addSCSIActual(ctx, addReq) +} + +// addSCSIActual is the implementation behind the external functions AddSCSI, +// AddSCSIPhysicalDisk, AddSCSIExtensibleVirtualDisk. +// +// We are in control of everything ourselves. Hence we have ref- counting and +// so-on tracking what SCSI locations are available or used. // // Returns result from calling modify with the given scsi mount -func (uvm *UtilityVM) addSCSIActual(ctx context.Context, hostPath, uvmPath, attachmentType string, readOnly bool, guestOptions []string, vmAccess VMAccessType) (sm *SCSIMount, err error) { - sm, existed, err := uvm.allocateSCSIMount(ctx, readOnly, hostPath, uvmPath, attachmentType, vmAccess) +func (uvm *UtilityVM) addSCSIActual(ctx context.Context, addReq *addSCSIRequest) (sm *SCSIMount, err error) { + sm, existed, err := uvm.allocateSCSIMount(ctx, addReq.readOnly, addReq.hostPath, addReq.uvmPath, addReq.attachmentType, addReq.evdType, addReq.vmAccess) if err != nil { return nil, err } @@ -290,9 +363,10 @@ func (uvm *UtilityVM) addSCSIActual(ctx context.Context, hostPath, uvmPath, atta SCSIModification := &hcsschema.ModifySettingRequest{ RequestType: requesttype.Add, Settings: hcsschema.Attachment{ - Path: sm.HostPath, - Type_: attachmentType, - ReadOnly: readOnly, + Path: sm.HostPath, + Type_: addReq.attachmentType, + ReadOnly: addReq.readOnly, + ExtensibleVirtualDiskType: addReq.evdType, }, ResourcePath: fmt.Sprintf(resourcepaths.SCSIResourceFormat, strconv.Itoa(sm.Controller), sm.LUN), } @@ -313,8 +387,8 @@ func (uvm *UtilityVM) addSCSIActual(ctx context.Context, hostPath, uvmPath, atta MountPath: sm.UVMPath, Lun: uint8(sm.LUN), Controller: uint8(sm.Controller), - ReadOnly: readOnly, - Options: guestOptions, + ReadOnly: addReq.readOnly, + Options: addReq.guestOptions, } } SCSIModification.GuestRequest = guestReq @@ -330,11 +404,13 @@ func (uvm *UtilityVM) addSCSIActual(ctx context.Context, hostPath, uvmPath, atta // device or allocates a new one if not already present. // Returns the resulting *SCSIMount, a bool indicating if the scsi device was already present, // and error if any. -func (uvm *UtilityVM) allocateSCSIMount(ctx context.Context, readOnly bool, hostPath, uvmPath, attachmentType string, vmAccess VMAccessType) (*SCSIMount, bool, error) { - // Ensure the utility VM has access - err := grantAccess(ctx, uvm.id, hostPath, vmAccess) - if err != nil { - return nil, false, errors.Wrapf(err, "failed to grant VM access for SCSI mount") +func (uvm *UtilityVM) allocateSCSIMount(ctx context.Context, readOnly bool, hostPath, uvmPath, attachmentType, evdType string, vmAccess VMAccessType) (*SCSIMount, bool, error) { + if attachmentType != "ExtensibleVirtualDisk" { + // Ensure the utility VM has access + err := grantAccess(ctx, uvm.id, hostPath, vmAccess) + if err != nil { + return nil, false, errors.Wrapf(err, "failed to grant VM access for SCSI mount") + } } // We must hold the lock throughout the lookup (findSCSIAttachment) until // after the possible allocation (allocateSCSISlot) has been completed to ensure @@ -352,7 +428,8 @@ func (uvm *UtilityVM) allocateSCSIMount(ctx context.Context, readOnly bool, host return nil, false, err } - uvm.scsiLocations[controller][lun] = newSCSIMount(uvm, hostPath, uvmPath, attachmentType, 1, controller, int32(lun), readOnly) + uvm.scsiLocations[controller][lun] = newSCSIMount(uvm, hostPath, uvmPath, attachmentType, evdType, 1, controller, int32(lun), readOnly) + log.G(ctx).WithFields(uvm.scsiLocations[controller][lun].logFormat()).Debug("allocated SCSI mount") return uvm.scsiLocations[controller][lun], false, nil @@ -413,6 +490,9 @@ func (sm *SCSIMount) GobEncode() ([]byte, error) { if err := encoder.Encode(sm.attachmentType); err != nil { return nil, fmt.Errorf(errMsgFmt, err) } + if err := encoder.Encode(sm.extensibleVirtualDiskType); err != nil { + return nil, fmt.Errorf(errMsgFmt, err) + } return buf.Bytes(), nil } @@ -447,6 +527,9 @@ func (sm *SCSIMount) GobDecode(data []byte) error { if err := decoder.Decode(&sm.attachmentType); err != nil { return fmt.Errorf(errMsgFmt, err) } + if err := decoder.Decode(&sm.extensibleVirtualDiskType); err != nil { + return fmt.Errorf(errMsgFmt, err) + } return nil } @@ -524,7 +607,7 @@ func (sm *SCSIMount) Clone(ctx context.Context, vm *UtilityVM, cd *cloneData) er Type_: sm.attachmentType, } - clonedScsiMount := newSCSIMount(vm, dstVhdPath, sm.UVMPath, sm.attachmentType, 1, sm.Controller, sm.LUN, sm.readOnly) + clonedScsiMount := newSCSIMount(vm, dstVhdPath, sm.UVMPath, sm.attachmentType, sm.extensibleVirtualDiskType, 1, sm.Controller, sm.LUN, sm.readOnly) vm.scsiLocations[sm.Controller][sm.LUN] = clonedScsiMount @@ -534,3 +617,15 @@ func (sm *SCSIMount) Clone(ctx context.Context, vm *UtilityVM, cd *cloneData) er func (sm *SCSIMount) GetSerialVersionID() uint32 { return scsiCurrentSerialVersionID } + +// ParseExtensibleVirtualDiskPath parses the evd path provided in the config. +// extensible virtual disk path has format "evd:///" +// this function parses that and returns the `evdType` and `evd-mount-path`. +func ParseExtensibleVirtualDiskPath(hostPath string) (evdType, mountPath string, err error) { + trimmedPath := strings.TrimPrefix(hostPath, "evd://") + separatorIndex := strings.Index(trimmedPath, "/") + if separatorIndex <= 0 { + return "", "", errors.Errorf("invalid extensible vhd path: %s", hostPath) + } + return trimmedPath[:separatorIndex], trimmedPath[separatorIndex+1:], nil +}