Skip to content

IoT/ARM64 support: allow cloudstack-agent on Raspberry Pi 4 (armv8) to use kvm acceleration #3644

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yadvr merged 2 commits into from
Nov 11, 2019
Merged

IoT/ARM64 support: allow cloudstack-agent on Raspberry Pi 4 (armv8) to use kvm acceleration #3644

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
01b679e
kvm: add support for aarch64/arm64 platform such as Raspberry Pi 4
yadvr Oct 17, 2019
8f33a88
kvm: fix unit test failure
yadvr Oct 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions agent/conf/agent.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,12 @@ hypervisor.type=kvm
# on,run virsh capabilities for more details.
# guest.cpu.model=
#
# This param will set the CPU architecture for the domain override what
# the management server would send
# In case of arm64 (aarch64), this will change the machine type to 'virt' and
# adds a SCSI and a USB controller in the domain xml.
# guest.cpu.arch=x86_64|aarch64
#
# This param will require CPU features on the <cpu> section
# guest.cpu.features=vmx vme
#
Expand Down
3 changes: 2 additions & 1 deletion .../hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtCapXMLParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ public void startElement(String uri, String localName, String qName, Attributes
}
} else if (qName.equalsIgnoreCase("arch")) {
for (int i = 0; i < attributes.getLength(); i++) {
if (attributes.getQName(i).equalsIgnoreCase("name") && attributes.getValue(i).equalsIgnoreCase("x86_64")) {
if (attributes.getQName(i).equalsIgnoreCase("name") &&
(attributes.getValue(i).equalsIgnoreCase("x86_64") || attributes.getValue(i).equalsIgnoreCase("aarch64"))) {
_archTypex8664 = true;
}
}
Expand Down
35 changes: 31 additions & 4 deletions ...rvisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,6 @@
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import com.cloud.hypervisor.kvm.dpdk.DpdkHelper;
import com.cloud.resource.RequestWrapper;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
Expand Down Expand Up @@ -112,6 +110,7 @@
import com.cloud.exception.InternalErrorException;
import com.cloud.host.Host.Type;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.hypervisor.kvm.dpdk.DpdkHelper;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ChannelDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ClockDef;
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.ConsoleDef;
Expand Down Expand Up @@ -148,6 +147,7 @@
import com.cloud.network.Networks.BroadcastDomainType;
import com.cloud.network.Networks.RouterPrivateIpStrategy;
import com.cloud.network.Networks.TrafficType;
import com.cloud.resource.RequestWrapper;
import com.cloud.resource.ServerResource;
import com.cloud.resource.ServerResourceBase;
import com.cloud.storage.JavaStorageLayer;
Expand Down Expand Up @@ -260,6 +260,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
protected String _localStoragePath;
protected String _localStorageUUID;
protected boolean _noMemBalloon = false;
protected String _guestCpuArch;
protected String _guestCpuMode;
protected String _guestCpuModel;
protected boolean _noKvmClock;
Expand Down Expand Up @@ -955,6 +956,12 @@ public boolean configure(final String name, final Map<String, Object> params) th
s_logger.trace("Ignoring libvirt error.", e);
}

final String cpuArchOverride = (String)params.get("guest.cpu.arch");
if (!Strings.isNullOrEmpty(cpuArchOverride)) {
_guestCpuArch = cpuArchOverride;
s_logger.info("Using guest CPU architecture: " + _guestCpuArch);
}

_guestCpuMode = (String)params.get("guest.cpu.mode");
if (_guestCpuMode != null) {
_guestCpuModel = (String)params.get("guest.cpu.model");
Expand Down Expand Up @@ -2086,8 +2093,8 @@ public LibvirtVMDef createVMFromSpec(final VirtualMachineTO vmTO) {
vm.setLibvirtVersion(_hypervisorLibvirtVersion);
vm.setQemuVersion(_hypervisorQemuVersion);
}
guest.setGuestArch(vmTO.getArch());
guest.setMachineType("pc");
guest.setGuestArch(_guestCpuArch != null ? _guestCpuArch : vmTO.getArch());
guest.setMachineType(_guestCpuArch != null && _guestCpuArch.equals("aarch64") ? "virt" : "pc");
guest.setUuid(uuid);
guest.setBootOrder(GuestDef.BootOrder.CDROM);
guest.setBootOrder(GuestDef.BootOrder.HARDISK);
Expand Down Expand Up @@ -2208,6 +2215,12 @@ So if getMinSpeed() returns null we fall back to getSpeed().
final InputDef input = new InputDef("tablet", "usb");
devices.addDevice(input);

// Add an explicit USB devices for ARM64
if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) {
devices.addDevice(new InputDef("keyboard", "usb"));
devices.addDevice(new InputDef("mouse", "usb"));
devices.addDevice(new LibvirtVMDef.USBDef((short)0, 0, 5, 0, 0));
}

DiskDef.DiskBus busT = getDiskModelFromVMDetail(vmTO);

Expand Down Expand Up @@ -2353,6 +2366,9 @@ public int compare(final DiskTO arg0, final DiskTO arg1) {
} else {
disk.defISODisk(volPath, devId);
}
if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) {
disk.setBusType(DiskDef.DiskBus.SCSI);
}
} else {
if (diskBusType == DiskDef.DiskBus.SCSI ) {
disk.setQemuDriver(true);
Expand Down Expand Up @@ -2411,6 +2427,9 @@ public int compare(final DiskTO arg0, final DiskTO arg1) {
if (_sysvmISOPath != null) {
final DiskDef iso = new DiskDef();
iso.defISODisk(_sysvmISOPath);
if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) {
iso.setBusType(DiskDef.DiskBus.SCSI);
}
vm.getDevices().addDevice(iso);
}
}
Expand Down Expand Up @@ -3183,6 +3202,10 @@ public DiskDef.DiskBus getDiskModelFromVMDetail(final VirtualMachineTO vmTO) {
return null;
}

if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) {
return DiskDef.DiskBus.SCSI;
}

final String rootDiskController = details.get(VmDetailConstants.ROOT_DISK_CONTROLLER);
if (StringUtils.isNotBlank(rootDiskController)) {
s_logger.debug("Passed custom disk bus " + rootDiskController);
Expand All @@ -3197,6 +3220,10 @@ public DiskDef.DiskBus getDiskModelFromVMDetail(final VirtualMachineTO vmTO) {
}

private DiskDef.DiskBus getGuestDiskModel(final String platformEmulator) {
if (_guestCpuArch != null && _guestCpuArch.equals("aarch64")) {
return DiskDef.DiskBus.SCSI;
}

if (platformEmulator == null) {
return DiskDef.DiskBus.IDE;
} else if (platformEmulator.startsWith("Other PV Virtio-SCSI")) {
Expand Down
42 changes: 41 additions & 1 deletion plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,12 +125,17 @@ public String toString() {
guestDef.append(" machine='" + _machine + "'");
}
guestDef.append(">hvm</type>\n");
if (_arch != null && _arch.equals("aarch64")) {
guestDef.append("<loader readonly='yes' type='pflash'>/usr/share/AAVMF/AAVMF_CODE.fd</loader>\n");
}
if (!_bootdevs.isEmpty()) {
for (BootOrder bo : _bootdevs) {
guestDef.append("<boot dev='" + bo + "'/>\n");
}
}
guestDef.append("<smbios mode='sysinfo'/>\n");
if (_arch == null || !_arch.equals("aarch64")) {
guestDef.append("<smbios mode='sysinfo'/>\n");
}
guestDef.append("</os>\n");
return guestDef.toString();
} else if (_type == GuestType.LXC) {
Expand Down Expand Up @@ -782,6 +787,10 @@ public DiskBus getBusType() {
return _bus;
}

public void setBusType(DiskBus busType) {
_bus = busType;
}

public DiskFmtType getDiskFormatType() {
return _diskFmtType;
}
Expand Down Expand Up @@ -1624,6 +1633,37 @@ public String toString() {
}
}

public static class USBDef {
private short index = 0;
private int domain = 0;
private int bus = 0;
private int slot = 9;
private int function = 0;

public USBDef(short index, int domain, int bus, int slot, int function) {
this.index = index;
this.domain = domain;
this.bus = bus;
this.slot = slot;
this.function = function;
}

public USBDef() {
}

@Override
public String toString() {
StringBuilder scsiBuilder = new StringBuilder();

scsiBuilder.append(String.format("<controller type='usb' index='%d' model='qemu-xhci'>\n", this.index));
scsiBuilder.append("<alias name='usb'/>");
scsiBuilder.append(String.format("<address type='pci' domain='0x%04X' bus='0x%02X' slot='0x%02X' function='0x%01X'/>\n",
this.domain, this.bus, this.slot, this.function ) );
scsiBuilder.append("</controller>\n");
return scsiBuilder.toString();
}
}

public static class InputDef {
private final String _type; /* tablet, mouse */
private final String _bus; /* ps2, usb, xen */
Expand Down
3 changes: 3 additions & 0 deletions ...ors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ public void testCreateVMFromSpecLegacy() {

final VirtualMachineTO to = new VirtualMachineTO(id, name, VirtualMachine.Type.User, cpus, speed, minRam, maxRam, BootloaderType.HVM, os, false, false, vncPassword);
to.setVncAddr(vncAddr);
to.setArch("x86_64");
to.setUuid("b0f0a72d-7efb-3cad-a8ff-70ebf30b3af9");

final LibvirtVMDef vm = lcr.createVMFromSpec(to);
Expand Down Expand Up @@ -275,6 +276,7 @@ public void testCreateVMFromSpecWithTopology6() {

final VirtualMachineTO to = new VirtualMachineTO(id, name, VirtualMachine.Type.User, cpus, minSpeed, maxSpeed, minRam, maxRam, BootloaderType.HVM, os, false, false, vncPassword);
to.setVncAddr(vncAddr);
to.setArch("x86_64");
to.setUuid("b0f0a72d-7efb-3cad-a8ff-70ebf30b3af9");

final LibvirtVMDef vm = lcr.createVMFromSpec(to);
Expand Down Expand Up @@ -344,6 +346,7 @@ public void testCreateVMFromSpec() {
final VirtualMachineTO to =
new VirtualMachineTO(id, name, VirtualMachine.Type.User, cpus, minSpeed, maxSpeed, minRam, maxRam, BootloaderType.HVM, os, false, false, vncPassword);
to.setVncAddr(vncAddr);
to.setArch("x86_64");
to.setUuid("b0f0a72d-7efb-3cad-a8ff-70ebf30b3af9");

final LibvirtVMDef vm = lcr.createVMFromSpec(to);
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ private void setupAgentSecurity(final Connection sshConnection, final String age
throw new DiscoveredWithErrorException("Authentication error");
}

if (!SSHCmdHelper.sshExecuteCmd(sshConnection, "lsmod|grep kvm")) {
if (!SSHCmdHelper.sshExecuteCmd(sshConnection, "ls /dev/kvm")) {
s_logger.debug("It's not a KVM enabled machine");
return null;
}
Expand Down