virtcontainers: vfio: add support for hot plugging VFIO devices

virtcontainers/device.go

@@ -132,7 +132,13 @@ func (device *VFIODevice) attach(h hypervisor, c *Container) error {
 
 		device.BDF = deviceBDF
 
-		if err := h.addDevice(*device, vfioDev); err != nil {
+		randBytes, err := generateRandomBytes(8)
+		if err != nil {
+			return err
+		}
+		device.DeviceInfo.ID = hex.EncodeToString(randBytes)
+
+		if err := h.hotplugAddDevice(*device, vfioDev); err != nil {
 			deviceLogger().WithError(err).Error("Failed to add device")
 			return err
 		}

virtcontainers/pod.go

@@ -596,11 +596,6 @@ func createPod(podConfig PodConfig) (*Pod, error) {
 		return nil, err
 	}
 
-	// Passthrough devices
-	if err := p.attachDevices(); err != nil {
-		return nil, err
-	}
-
 	// Set pod state
 	if err := p.setPodState(StateReady); err != nil {
 		return nil, err
@@ -1070,23 +1065,3 @@ func togglePausePod(podID string, pause bool) (*Pod, error) {
 
 	return p, nil
 }
-
-func (p *Pod) attachDevices() error {
-	for _, container := range p.containers {
-		if err := container.attachDevices(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (p *Pod) detachDevices() error {
-	for _, container := range p.containers {
-		if err := container.detachDevices(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

virtcontainers/pod_test.go

@@ -1171,10 +1171,10 @@ func TestPodAttachDevicesVFIO(t *testing.T) {
 
 	containers[0].pod = &pod
 
-	err = pod.attachDevices()
+	err = containers[0].attachDevices()
 	assert.Nil(t, err, "Error while attaching devices %s", err)
 
-	err = pod.detachDevices()
+	err = containers[0].detachDevices()
 	assert.Nil(t, err, "Error while detaching devices %s", err)
 }
 

virtcontainers/qemu.go

@@ -346,12 +346,12 @@ func (q *qemu) createPod(podConfig PodConfig) error {
 		return err
 	}
 
-	if q.config.BlockDeviceDriver == VirtioBlock {
-		devices = q.arch.appendBridges(devices, q.state.Bridges)
-		if err != nil {
-			return err
-		}
-	} else {
+	devices = q.arch.appendBridges(devices, q.state.Bridges)
+	if err != nil {
+		return err
+	}
+
+	if q.config.BlockDeviceDriver == VirtioSCSI {
 		devices = q.arch.appendSCSIController(devices)
 	}
 
@@ -638,6 +638,44 @@ func (q *qemu) hotplugBlockDevice(drive Drive, op operation) error {
 	return nil
 }
 
+func (q *qemu) hotplugVFIODevice(device VFIODevice, op operation) error {
+	defer func(qemu *qemu) {
+		if q.qmpMonitorCh.qmp != nil {
+			q.qmpMonitorCh.qmp.Shutdown()
+		}
+	}(q)
+
+	qmp, err := q.qmpSetup()
+	if err != nil {
+		return err
+	}
+
+	q.qmpMonitorCh.qmp = qmp
+
+	devID := "vfio-" + device.DeviceInfo.ID
+
+	if op == addDevice {
+		addr, bus, err := q.addDeviceToBridge(devID)
+		if err != nil {
+			return err
+		}
+
+		if err := q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bus); err != nil {
+			return err
+		}
+	} else {
+		if err := q.removeDeviceFromBridge(devID); err != nil {
+			return err
+		}
+
+		if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operation) error {
 	switch devType {
 	case blockDev:
@@ -646,6 +684,9 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operati
 	case cpuDev:
 		vcpus := devInfo.(uint32)
 		return q.hotplugCPUs(vcpus, op)
+	case vfioDev:
+		device := devInfo.(VFIODevice)
+		return q.hotplugVFIODevice(device, op)
 	default:
 		return fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType)
 	}