From 03422a5a93ecc21904c3e352b45aca666e7666ae Mon Sep 17 00:00:00 2001 From: Kathryn Baldauf Date: Tue, 6 Jul 2021 16:14:02 -0700 Subject: [PATCH] Add support for reading in device extension files for container create hcs document Signed-off-by: Kathryn Baldauf --- internal/hcs/schema2/container.go | 2 + internal/hcs/schema2/device.go | 8 +-- .../model_container_definition_device.go | 14 +++++ internal/hcs/schema2/model_device_category.go | 15 +++++ .../hcs/schema2/model_device_extension.go | 15 +++++ internal/hcs/schema2/model_device_instance.go | 17 ++++++ .../hcs/schema2/model_device_namespace.go | 16 +++++ internal/hcs/schema2/model_interface_class.go | 16 +++++ internal/hcs/schema2/model_namespace.go | 15 +++++ .../hcs/schema2/model_object_directory.go | 18 ++++++ .../hcs/schema2/model_object_namespace.go | 16 +++++ internal/hcs/schema2/model_object_symlink.go | 18 ++++++ internal/hcsoci/devices.go | 61 ++++++++++++------- internal/hcsoci/hcsdoc_wcow.go | 9 ++- internal/hcsoci/resources_wcow.go | 16 +++++ internal/oci/annotations.go | 4 ++ internal/oci/uvm.go | 11 ++++ 17 files changed, 245 insertions(+), 26 deletions(-) create mode 100644 internal/hcs/schema2/model_container_definition_device.go create mode 100644 internal/hcs/schema2/model_device_category.go create mode 100644 internal/hcs/schema2/model_device_extension.go create mode 100644 internal/hcs/schema2/model_device_instance.go create mode 100644 internal/hcs/schema2/model_device_namespace.go create mode 100644 internal/hcs/schema2/model_interface_class.go create mode 100644 internal/hcs/schema2/model_namespace.go create mode 100644 internal/hcs/schema2/model_object_directory.go create mode 100644 internal/hcs/schema2/model_object_namespace.go create mode 100644 internal/hcs/schema2/model_object_symlink.go diff --git a/internal/hcs/schema2/container.go b/internal/hcs/schema2/container.go index 4fb2310768..39a54432c0 100644 --- a/internal/hcs/schema2/container.go +++ b/internal/hcs/schema2/container.go @@ -31,4 +31,6 @@ type Container struct { RegistryChanges *RegistryChanges `json:"RegistryChanges,omitempty"` AssignedDevices []Device `json:"AssignedDevices,omitempty"` + + AdditionalDeviceNamespace *ContainerDefinitionDevice `json:"AdditionalDeviceNamespace,omitempty"` } diff --git a/internal/hcs/schema2/device.go b/internal/hcs/schema2/device.go index 107caddada..31c4538aff 100644 --- a/internal/hcs/schema2/device.go +++ b/internal/hcs/schema2/device.go @@ -12,9 +12,9 @@ package hcsschema type DeviceType string const ( - ClassGUID DeviceType = "ClassGuid" - DeviceInstance DeviceType = "DeviceInstance" - GPUMirror DeviceType = "GpuMirror" + ClassGUID DeviceType = "ClassGuid" + DeviceInstanceID DeviceType = "DeviceInstance" + GPUMirror DeviceType = "GpuMirror" ) type Device struct { @@ -22,6 +22,6 @@ type Device struct { Type DeviceType `json:"Type,omitempty"` // The interface class guid of the device interfaces to assign to the container. Only used when Type is ClassGuid. InterfaceClassGuid string `json:"InterfaceClassGuid,omitempty"` - // The location path of the device to assign to the container. Only used when Type is DeviceInstance. + // The location path of the device to assign to the container. Only used when Type is DeviceInstanceID. LocationPath string `json:"LocationPath,omitempty"` } diff --git a/internal/hcs/schema2/model_container_definition_device.go b/internal/hcs/schema2/model_container_definition_device.go new file mode 100644 index 0000000000..8dbe40b3be --- /dev/null +++ b/internal/hcs/schema2/model_container_definition_device.go @@ -0,0 +1,14 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type ContainerDefinitionDevice struct { + DeviceExtension []DeviceExtension `json:"device_extension,omitempty"` +} diff --git a/internal/hcs/schema2/model_device_category.go b/internal/hcs/schema2/model_device_category.go new file mode 100644 index 0000000000..8fe89f9274 --- /dev/null +++ b/internal/hcs/schema2/model_device_category.go @@ -0,0 +1,15 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type DeviceCategory struct { + Name string `json:"name,omitempty"` + InterfaceClass []InterfaceClass `json:"interface_class,omitempty"` +} diff --git a/internal/hcs/schema2/model_device_extension.go b/internal/hcs/schema2/model_device_extension.go new file mode 100644 index 0000000000..a62568d892 --- /dev/null +++ b/internal/hcs/schema2/model_device_extension.go @@ -0,0 +1,15 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type DeviceExtension struct { + DeviceCategory *DeviceCategory `json:"device_category,omitempty"` + Namespace *DeviceExtensionNamespace `json:"namespace,omitempty"` +} diff --git a/internal/hcs/schema2/model_device_instance.go b/internal/hcs/schema2/model_device_instance.go new file mode 100644 index 0000000000..a7410febd6 --- /dev/null +++ b/internal/hcs/schema2/model_device_instance.go @@ -0,0 +1,17 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type DeviceInstance struct { + Id string `json:"id,omitempty"` + LocationPath string `json:"location_path,omitempty"` + PortName string `json:"port_name,omitempty"` + InterfaceClass []InterfaceClass `json:"interface_class,omitempty"` +} diff --git a/internal/hcs/schema2/model_device_namespace.go b/internal/hcs/schema2/model_device_namespace.go new file mode 100644 index 0000000000..3553640647 --- /dev/null +++ b/internal/hcs/schema2/model_device_namespace.go @@ -0,0 +1,16 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type DeviceNamespace struct { + RequiresDriverstore bool `json:"requires_driverstore,omitempty"` + DeviceCategory []DeviceCategory `json:"device_category,omitempty"` + DeviceInstance []DeviceInstance `json:"device_instance,omitempty"` +} diff --git a/internal/hcs/schema2/model_interface_class.go b/internal/hcs/schema2/model_interface_class.go new file mode 100644 index 0000000000..7be98b5410 --- /dev/null +++ b/internal/hcs/schema2/model_interface_class.go @@ -0,0 +1,16 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type InterfaceClass struct { + Type_ string `json:"type,omitempty"` + Identifier string `json:"identifier,omitempty"` + Recurse bool `json:"recurse,omitempty"` +} diff --git a/internal/hcs/schema2/model_namespace.go b/internal/hcs/schema2/model_namespace.go new file mode 100644 index 0000000000..3ab9cf1ecf --- /dev/null +++ b/internal/hcs/schema2/model_namespace.go @@ -0,0 +1,15 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type DeviceExtensionNamespace struct { + Ob *ObjectNamespace `json:"ob,omitempty"` + Device *DeviceNamespace `json:"device,omitempty"` +} diff --git a/internal/hcs/schema2/model_object_directory.go b/internal/hcs/schema2/model_object_directory.go new file mode 100644 index 0000000000..d2f51b3b53 --- /dev/null +++ b/internal/hcs/schema2/model_object_directory.go @@ -0,0 +1,18 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type ObjectDirectory struct { + Name string `json:"name,omitempty"` + Clonesd string `json:"clonesd,omitempty"` + Shadow string `json:"shadow,omitempty"` + Symlink []ObjectSymlink `json:"symlink,omitempty"` + Objdir []ObjectDirectory `json:"objdir,omitempty"` +} diff --git a/internal/hcs/schema2/model_object_namespace.go b/internal/hcs/schema2/model_object_namespace.go new file mode 100644 index 0000000000..47dfb55bfa --- /dev/null +++ b/internal/hcs/schema2/model_object_namespace.go @@ -0,0 +1,16 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type ObjectNamespace struct { + Shadow string `json:"shadow,omitempty"` + Symlink []ObjectSymlink `json:"symlink,omitempty"` + Objdir []ObjectDirectory `json:"objdir,omitempty"` +} diff --git a/internal/hcs/schema2/model_object_symlink.go b/internal/hcs/schema2/model_object_symlink.go new file mode 100644 index 0000000000..8867ebe5f0 --- /dev/null +++ b/internal/hcs/schema2/model_object_symlink.go @@ -0,0 +1,18 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type ObjectSymlink struct { + Name string `json:"name,omitempty"` + Path string `json:"path,omitempty"` + Scope string `json:"scope,omitempty"` + Pathtoclone string `json:"pathtoclone,omitempty"` + AccessMask int32 `json:"access_mask,omitempty"` +} diff --git a/internal/hcsoci/devices.go b/internal/hcsoci/devices.go index 1c5e6f9553..ede997bb7e 100644 --- a/internal/hcsoci/devices.go +++ b/internal/hcsoci/devices.go @@ -2,12 +2,14 @@ package hcsoci import ( "context" + "encoding/json" "fmt" + "io/ioutil" "os" "path/filepath" - "strings" "github.com/Microsoft/hcsshim/internal/devices" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/resources" @@ -20,14 +22,8 @@ const deviceUtilExeName = "device-util.exe" // getAssignedDeviceKernelDrivers gets any device drivers specified on the spec. // Drivers are optional, therefore do not return an error if none are on the spec. -// -// See comment on oci.AnnotationAssignedDeviceKernelDrivers for expected format. func getAssignedDeviceKernelDrivers(annotations map[string]string) ([]string, error) { - csDrivers, ok := annotations[oci.AnnotationAssignedDeviceKernelDrivers] - if !ok || csDrivers == "" { - return nil, nil - } - drivers := strings.Split(csDrivers, ",") + drivers := oci.ParseAnnotationCommaSeparated(oci.AnnotationAssignedDeviceKernelDrivers, annotations) for _, driver := range drivers { if _, err := os.Stat(driver); err != nil { return nil, errors.Wrapf(err, "failed to find path to drivers at %s", driver) @@ -36,11 +32,47 @@ func getAssignedDeviceKernelDrivers(annotations map[string]string) ([]string, er return drivers, nil } +// getDeviceExtensionPaths gets any device extensions paths specified on the spec. +// device extensions are optional, therefore if none are on the spec, do not return an error. +func getDeviceExtensionPaths(annotations map[string]string) ([]string, error) { + extensions := oci.ParseAnnotationCommaSeparated(oci.AnnotationDeviceExtensions, annotations) + for _, ext := range extensions { + if _, err := os.Stat(ext); err != nil { + return nil, errors.Wrapf(err, "failed to find path to driver extensions at %s", ext) + } + } + return extensions, nil +} + // getDeviceUtilHostPath is a simple helper function to find the host path of the device-util tool func getDeviceUtilHostPath() string { return filepath.Join(filepath.Dir(os.Args[0]), deviceUtilExeName) } +// getDeviceExtensions is a helper function to read the files at `extensionPaths` and unmarshal the contents +// into a `hcsshema.DeviceExtension` to be added to a container's hcs create document. +func getDeviceExtensions(annotations map[string]string) (*hcsschema.ContainerDefinitionDevice, error) { + extensionPaths, err := getDeviceExtensionPaths(annotations) + if err != nil { + return nil, err + } + results := &hcsschema.ContainerDefinitionDevice{ + DeviceExtension: []hcsschema.DeviceExtension{}, + } + for _, extensionPath := range extensionPaths { + data, err := ioutil.ReadFile(extensionPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to read extension file at %s", extensionPath) + } + extension := hcsschema.DeviceExtension{} + if err := json.Unmarshal(data, &extension); err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal extension file at %s", extensionPath) + } + results.DeviceExtension = append(results.DeviceExtension, extension) + } + return results, nil +} + // handleAssignedDevicesWindows does all of the work to setup the hosting UVM, assign in devices // specified on the spec, and install any necessary, specified kernel drivers into the UVM. // @@ -91,18 +123,5 @@ func handleAssignedDevicesWindows(ctx context.Context, vm *uvm.UtilityVM, annota } } - // get the spec specified kernel drivers and install them on the UVM - drivers, err := getAssignedDeviceKernelDrivers(annotations) - if err != nil { - return nil, closers, err - } - for _, d := range drivers { - driverCloser, err := devices.InstallWindowsDriver(ctx, vm, d) - if err != nil { - return nil, closers, err - } - closers = append(closers, driverCloser) - } - return resultDevs, closers, nil } diff --git a/internal/hcsoci/hcsdoc_wcow.go b/internal/hcsoci/hcsdoc_wcow.go index 7423668dda..eb7019698f 100644 --- a/internal/hcsoci/hcsdoc_wcow.go +++ b/internal/hcsoci/hcsdoc_wcow.go @@ -367,6 +367,13 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter return nil, nil, err } + // add any device extensions + extensions, err := getDeviceExtensions(coi.Spec.Annotations) + if err != nil { + return nil, nil, err + } + v2Container.AdditionalDeviceNamespace = extensions + return v1, v2Container, nil } @@ -383,7 +390,7 @@ func parseAssignedDevices(ctx context.Context, coi *createOptionsInternal, v2 *h switch d.IDType { case uvm.VPCILocationPathIDType: v2Dev.LocationPath = d.ID - v2Dev.Type = hcsschema.DeviceInstance + v2Dev.Type = hcsschema.DeviceInstanceID case uvm.VPCIClassGUIDTypeLegacy: v2Dev.InterfaceClassGuid = d.ID case uvm.VPCIClassGUIDType: diff --git a/internal/hcsoci/resources_wcow.go b/internal/hcsoci/resources_wcow.go index 7f3c67140b..83646dd0c1 100644 --- a/internal/hcsoci/resources_wcow.go +++ b/internal/hcsoci/resources_wcow.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/Microsoft/hcsshim/internal/credentials" + "github.com/Microsoft/hcsshim/internal/devices" "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/resources" @@ -102,6 +103,21 @@ func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r coi.Spec.Windows.Devices = windowsDevices } + if coi.HostingSystem != nil { + // get the spec specified kernel drivers and install them on the UVM + drivers, err := getAssignedDeviceKernelDrivers(coi.Spec.Annotations) + if err != nil { + return err + } + for _, d := range drivers { + driverCloser, err := devices.InstallWindowsDriver(ctx, coi.HostingSystem, d) + if err != nil { + return err + } + r.Add(driverCloser) + } + } + return nil } diff --git a/internal/oci/annotations.go b/internal/oci/annotations.go index 9ac11dcbcd..eb011c656a 100644 --- a/internal/oci/annotations.go +++ b/internal/oci/annotations.go @@ -79,6 +79,10 @@ const ( // .inf, .cer, and/or other files used during standard installation with pnputil. AnnotationAssignedDeviceKernelDrivers = "io.microsoft.assigneddevice.kerneldrivers" + // AnnotationDeviceExtensions contains a comma separated list of full paths to device extension files. + // The content of these are added to a container's hcs create document. + AnnotationDeviceExtensions = "io.microsoft.container.wcow.deviceextensions" + // AnnotationHostProcessInheritUser indicates whether to ignore the username passed in to run a host process // container as and instead inherit the user token from the executable that is launching the container process. AnnotationHostProcessInheritUser = "microsoft.com/hostprocess-inherit-user" diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index 5b8c5e75c1..06715099f0 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -36,6 +36,17 @@ func parseAnnotationsBool(ctx context.Context, a map[string]string, key string, return def } +// ParseAnnotationCommaSeparated searches `annotations` for `annotation` corresponding to a +// list of comma separated strings +func ParseAnnotationCommaSeparated(annotation string, annotations map[string]string) []string { + cs, ok := annotations[annotation] + if !ok || cs == "" { + return nil + } + results := strings.Split(cs, ",") + return results +} + // ParseAnnotationsCPUCount searches `s.Annotations` for the CPU annotation. If // not found searches `s` for the Windows CPU section. If neither are found // returns `def`.