Skip to content
This repository was archived by the owner on May 12, 2021. It is now read-only.

Firecracker: virtio mmio support #1045

Merged
sboeuf merged 1 commit into from
Dec 21, 2018
Merged

Firecracker: virtio mmio support #1045

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions pkg/katautils/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (

"github.com/BurntSushi/toml"
vc "github.com/kata-containers/runtime/virtcontainers"
"github.com/kata-containers/runtime/virtcontainers/device/config"
"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
"github.com/kata-containers/runtime/virtcontainers/utils"
"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -292,15 +293,19 @@ func (h hypervisor) defaultBridges() uint32 {
}

func (h hypervisor) blockDeviceDriver() (string, error) {
supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio}

if h.BlockDeviceDriver == "" {
return defaultBlockDeviceDriver, nil
}

if h.BlockDeviceDriver != vc.VirtioSCSI && h.BlockDeviceDriver != vc.VirtioBlock {
return "", fmt.Errorf("Invalid value %s provided for hypervisor block storage driver, can be either %s or %s", h.BlockDeviceDriver, vc.VirtioSCSI, vc.VirtioBlock)
for _, b := range supportedBlockDrivers {
if b == h.BlockDeviceDriver {
return h.BlockDeviceDriver, nil
}
}

return h.BlockDeviceDriver, nil
return "", fmt.Errorf("Invalid hypervisor block storage driver %v specified (supported drivers: %v)", h.BlockDeviceDriver, supportedBlockDrivers)
}

func (h hypervisor) msize9p() uint32 {
Expand Down
14 changes: 14 additions & 0 deletions virtcontainers/device/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,17 @@ const (
VhostUserBlk = "vhost-user-blk-pci"
)

const (
// VirtioMmio means use virtio-mmio for mmio based drives
VirtioMmio = "virtio-mmio"

// VirtioBlock means use virtio-blk for hotplugging drives
VirtioBlock = "virtio-blk"

// VirtioSCSI means use virtio-scsi for hotplugging drives
VirtioSCSI = "virtio-scsi"
)

// Defining these as a variable instead of a const, to allow
// overriding this in the tests.

Expand Down Expand Up @@ -98,6 +109,9 @@ type BlockDrive struct {
// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
Index int

// MmioAddr is used to identify the slot at which the drive is attached (order?).
MmioAddr string

// PCIAddr is the PCI address used to identify the slot at which the drive is attached.
PCIAddr string

Expand Down
34 changes: 26 additions & 8 deletions virtcontainers/device/drivers/block.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,21 +67,39 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
Index: index,
}

driveName, err := utils.GetVirtDriveName(index)
if err != nil {
return err
}

customOptions := device.DeviceInfo.DriverOptions
if customOptions != nil && customOptions["block-driver"] == "virtio-blk" {
drive.VirtPath = filepath.Join("/dev", driveName)
} else {
if customOptions == nil ||
customOptions["block-driver"] == "virtio-scsi" {
// User has not chosen a specific block device type
// Default to SCSI
scsiAddr, err := utils.GetSCSIAddress(index)
if err != nil {
return err
}

drive.SCSIAddr = scsiAddr
} else {
var globalIdx int

switch customOptions["block-driver"] {
case "virtio-blk":
globalIdx = index
case "virtio-mmio":
//With firecracker the rootfs for the VM itself
//sits at /dev/vda and consumes the first index.
//Longer term block based VM rootfs should be added
//as a regular block device which eliminates the
//offset.
//https://github.com/kata-containers/runtime/issues/1061
globalIdx = index + 1
}

driveName, err := utils.GetVirtDriveName(globalIdx)
Copy link
Copy Markdown
Member

@bergwolf bergwolf Dec 20, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@amshinde I'm wondering if we still rely on GetVirtDriveName for virtio-blk device identification? I thought we now use pci address for them. Then at least GetVirtDriveName is not needed for virtio-blk. We can still keep it for virtio-mmio. In future we might want to replace it with disk UUID though.

Copy link
Copy Markdown
Member

@WeiZhang555 WeiZhang555 Dec 21, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agree with @bergwolf

Overall it LGTM

if err != nil {
return err
}
Comment thread
egernst marked this conversation as resolved.
Outdated

drive.VirtPath = filepath.Join("/dev", driveName)
}

deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device")
Expand Down
6 changes: 5 additions & 1 deletion virtcontainers/device/manager/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import (
)

const (
// VirtioMmio indicates block driver is virtio-mmio based
VirtioMmio string = "virtio-mmio"
// VirtioBlock indicates block driver is virtio-blk based
VirtioBlock string = "virtio-blk"
// VirtioSCSI indicates block driver is virtio-scsi based
Expand Down Expand Up @@ -55,7 +57,9 @@ func NewDeviceManager(blockDriver string, devices []api.Device) api.DeviceManage
dm := &deviceManager{
devices: make(map[string]api.Device),
}
if blockDriver == VirtioBlock {
if blockDriver == VirtioMmio {
dm.blockDriver = VirtioMmio
} else if blockDriver == VirtioBlock {
dm.blockDriver = VirtioBlock
} else {
dm.blockDriver = VirtioSCSI
Expand Down
3 changes: 2 additions & 1 deletion virtcontainers/hyperstart_agent.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -522,7 +522,8 @@ func (h *hyper) startOneContainer(sandbox *Sandbox, c *Container) error {

if c.state.Fstype != "" {
// Pass a drive name only in case of block driver
if sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock ||
sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio {
driveName, err := utils.GetVirtDriveName(c.state.BlockIndex)
if err != nil {
return err
Expand Down
4 changes: 3 additions & 1 deletion virtcontainers/hypervisor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import (
"runtime"
"strconv"
"strings"

Comment thread
egernst marked this conversation as resolved.
"github.com/kata-containers/runtime/virtcontainers/device/config"
)

// HypervisorType describes an hypervisor type.
Expand All @@ -38,7 +40,7 @@ const (

defaultBridges = 1

defaultBlockDriver = VirtioSCSI
defaultBlockDriver = config.VirtioSCSI
)

// In some architectures the maximum number of vCPUs depends on the number of physical cores.
Expand Down
20 changes: 16 additions & 4 deletions virtcontainers/kata_agent.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ var (
// CAP_NET_BIND_SERVICE capability may bind to these port numbers.
vSockPort = 1024
kata9pDevType = "9p"
kataMmioBlkDevType = "mmioblk"
kataBlkDevType = "blk"
kataSCSIDevType = "scsi"
sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L,cache=mmap", "nodev"}
Expand Down Expand Up @@ -831,10 +832,15 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr
ContainerPath: dev.ContainerPath,
}

if d.SCSIAddr == "" {
switch c.sandbox.config.HypervisorConfig.BlockDeviceDriver {
case config.VirtioMmio:
kataDevice.Type = kataMmioBlkDevType
kataDevice.Id = d.VirtPath
kataDevice.VmPath = d.VirtPath
case config.VirtioBlock:
kataDevice.Type = kataBlkDevType
kataDevice.Id = d.PCIAddr
} else {
case config.VirtioSCSI:
kataDevice.Type = kataSCSIDevType
kataDevice.Id = d.SCSIAddr
}
Expand Down Expand Up @@ -883,7 +889,10 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat
return nil, fmt.Errorf("malformed block drive")
}

if sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio {
rootfs.Driver = kataMmioBlkDevType
rootfs.Source = blockDrive.VirtPath
} else if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock {
rootfs.Driver = kataBlkDevType
rootfs.Source = blockDrive.PCIAddr
} else {
Expand Down Expand Up @@ -1086,9 +1095,12 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
k.Logger().Error("malformed block drive")
continue
}
if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock {
vol.Driver = kataBlkDevType
vol.Source = blockDrive.PCIAddr
} else if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio {
vol.Driver = kataMmioBlkDevType
vol.Source = blockDrive.VirtPath
} else {
vol.Driver = kataSCSIDevType
vol.Source = blockDrive.SCSIAddr
Expand Down
9 changes: 8 additions & 1 deletion virtcontainers/kata_agent_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,9 +402,16 @@ func TestAppendDevices(t *testing.T) {
},
}

sandboxConfig := &SandboxConfig{
HypervisorConfig: HypervisorConfig{
BlockDeviceDriver: config.VirtioBlock,
},
}

c := &Container{
sandbox: &Sandbox{
devManager: manager.NewDeviceManager("virtio-scsi", ctrDevices),
devManager: manager.NewDeviceManager("virtio-blk", ctrDevices),
config: sandboxConfig,
},
}
c.devices = append(c.devices, ContainerDevice{
Expand Down
6 changes: 3 additions & 3 deletions virtcontainers/qemu.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,7 +372,7 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
}

var ioThread *govmmQemu.IOThread
if q.config.BlockDeviceDriver == VirtioSCSI {
if q.config.BlockDeviceDriver == config.VirtioSCSI {
devices, ioThread = q.arch.appendSCSIController(devices, q.config.EnableIOThreads)
}

Expand Down Expand Up @@ -746,7 +746,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
return err
}

if q.config.BlockDeviceDriver == VirtioBlock {
if q.config.BlockDeviceDriver == config.VirtioBlock {
driver := "virtio-blk-pci"
addr, bridge, err := q.addDeviceToBridge(drive.ID)
if err != nil {
Expand Down Expand Up @@ -776,7 +776,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
}
}
} else {
if q.config.BlockDeviceDriver == VirtioBlock {
if q.config.BlockDeviceDriver == config.VirtioBlock {
if err := q.removeDeviceFromBridge(drive.ID); err != nil {
return err
}
Expand Down
8 changes: 0 additions & 8 deletions virtcontainers/qemu_arch_base.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,14 +128,6 @@ const (
// 0 is reserved.
const bridgePCIStartAddr = 2

const (
// VirtioBlock means use virtio-blk for hotplugging drives
VirtioBlock = "virtio-blk"

// VirtioSCSI means use virtio-scsi for hotplugging drives
VirtioSCSI = "virtio-scsi"
)

const (
// QemuPCLite is the QEMU pc-lite machine type for amd64
QemuPCLite = "pc-lite"
Expand Down
10 changes: 5 additions & 5 deletions virtcontainers/sandbox_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1553,7 +1553,7 @@ func TestAttachBlockDevice(t *testing.T) {
hypervisor := &mockHypervisor{}

hConfig := HypervisorConfig{
BlockDeviceDriver: VirtioBlock,
BlockDeviceDriver: config.VirtioBlock,
}

sconfig := &SandboxConfig{
Expand Down Expand Up @@ -1598,7 +1598,7 @@ func TestAttachBlockDevice(t *testing.T) {
DevType: "b",
}

dm := manager.NewDeviceManager(VirtioBlock, nil)
dm := manager.NewDeviceManager(config.VirtioBlock, nil)
device, err := dm.NewDevice(deviceInfo)
assert.Nil(t, err)
_, ok := device.(*drivers.BlockDevice)
Expand All @@ -1618,7 +1618,7 @@ func TestAttachBlockDevice(t *testing.T) {
err = device.Detach(sandbox)
assert.Nil(t, err)

container.sandbox.config.HypervisorConfig.BlockDeviceDriver = VirtioSCSI
container.sandbox.config.HypervisorConfig.BlockDeviceDriver = config.VirtioSCSI
err = device.Attach(sandbox)
assert.Nil(t, err)

Expand All @@ -1638,14 +1638,14 @@ func TestPreAddDevice(t *testing.T) {
hypervisor := &mockHypervisor{}

hConfig := HypervisorConfig{
BlockDeviceDriver: VirtioBlock,
BlockDeviceDriver: config.VirtioBlock,
}

sconfig := &SandboxConfig{
HypervisorConfig: hConfig,
}

dm := manager.NewDeviceManager(VirtioBlock, nil)
dm := manager.NewDeviceManager(config.VirtioBlock, nil)
// create a sandbox first
sandbox := &Sandbox{
id: testSandboxID,
Expand Down