target: Implement QEMU Linux target

Add support for launching Linux targets on QEMU.

This requires having buildroot and QEMU packages installed on host
machine.

Newly introduced QEMULinuxTarget class is a simple wrapper around
LinuxTarget:
- Perform sanity checks to ensure QEMU executable, rootfs and kernel
  images exist.
- Spin QEMU process to launch a Linux guest and connect it to over SSH.

Also added a test case (tests/spin_targets.py) to ensure devlib can
launch a QEMU target and run some basic commands on it.

Signed-off-by: Metin Kaya <metin.kaya@arm.com>

Author: metin-arm

devlib/__init__.py

@@ -19,7 +19,7 @@
 
 from devlib.target import (
         Target, LinuxTarget, AndroidTarget, LocalLinuxTarget,
-        ChromeOsTarget,
+        ChromeOsTarget, QEMULinuxTarget,
 )
 
 from devlib.host import (

devlib/target.py

@@ -14,16 +14,20 @@
 #
 
 import asyncio
+import atexit
 import io
 import base64
 import functools
 import gzip
 import glob
 import os
 import re
+import sys
 import time
 import logging
 import posixpath
+import select
+import signal
 import subprocess
 import tarfile
 import tempfile
@@ -38,6 +42,7 @@
 from past.types import basestring
 from numbers import Number
 from shlex import quote
+from platform import machine
 try:
     from collections.abc import Mapping
 except ImportError:
@@ -56,7 +61,7 @@
 from devlib.utils.ssh import SshConnection
 from devlib.utils.android import AdbConnection, AndroidProperties, LogcatMonitor, adb_command, INTENT_FLAGS
 from devlib.utils.misc import memoized, isiterable, convert_new_lines, groupby_value
-from devlib.utils.misc import commonprefix, merge_lists
+from devlib.utils.misc import check_output, get_subprocess, commonprefix, merge_lists, which
 from devlib.utils.misc import ABI_MAP, get_cpu_name, ranges_to_list
 from devlib.utils.misc import batch_contextmanager, tls_property, _BoundTLSProperty, nullcontext
 from devlib.utils.misc import safe_extract
@@ -2892,6 +2897,95 @@ def _resolve_paths(self):
         if self.executables_directory is None:
             self.executables_directory = '/tmp/devlib-target/bin'
 
+class QEMULinuxTarget(LinuxTarget):
+    '''
+    Class for launching Linux target on QEMU.
+    '''
+
+    # pylint: disable=too-many-locals,too-many-arguments
+    def __init__(self,
+                 kernel_image,
+                 arch='aarch64',
+                 cpu_types='-cpu cortex-a72',
+                 initrd_image=str(),
+                 mem_size=512,
+                 num_cores=2,
+                 num_threads=2,
+                 cmdline='console=ttyAMA0',
+                 enable_kvm=True,
+                 boot_timeout=60,
+                 banner='Welcome to Buildroot',
+                 connect=True,
+                 connection_settings=None,
+                 **kwargs):
+        super().__init__(connect=False,
+                         conn_cls=SshConnection,
+                         connection_settings=connection_settings,
+                         **kwargs)
+
+        qemu_executable = f'qemu-system-{arch}'
+        qemu_path = which(qemu_executable)
+        if qemu_path is None:
+            raise FileNotFoundError(f'Cannot find {qemu_executable} executable!')
+
+        if not os.path.exists(kernel_image):
+            raise FileNotFoundError(f'{kernel_image} does not exist!')
+
+        qemu_cmd = f'{qemu_path} -kernel {quote(kernel_image)} -append "{quote(cmdline)}" ' \
+                   f'-smp cores={num_cores},threads={num_threads} -m {mem_size} ' \
+                   f'-netdev user,id=net0,hostfwd=tcp::{connection_settings["port"]}-:22 ' \
+                   '-device virtio-net-pci,netdev=net0 --nographic'
+
+        if initrd_image:
+            if not os.path.exists(initrd_image):
+                raise FileNotFoundError(f'{initrd_image} does not exist!')
+            qemu_cmd = f'{qemu_cmd} -initrd {quote(initrd_image)}'
+
+        if arch == machine():
+            qemu_cmd = f'{qemu_cmd} {"--enable-kvm" if enable_kvm else ""}'
+        else:
+            qemu_cmd = f'{qemu_cmd} -machine virt {cpu_types}'
+
+        self.logger.info('qemu_cmd: %s', qemu_cmd)
+
+        self.banner = banner
+
+        try:
+            self.qemu_process = get_subprocess(qemu_cmd, shell=True)
+        except Exception as ex:
+            raise HostError(f'Error while running "{qemu_cmd}": {ex}') from ex
+
+        atexit.register(self.terminate_qemu)
+
+        self.wait_boot_complete(timeout=boot_timeout)
+
+        if connect:
+            super().connect(check_boot_completed=False)
+
+    def wait_boot_complete(self, timeout=10):
+        start_time = time.time()
+        boot_completed = False
+        poll_obj = select.poll()
+        poll_obj.register(self.qemu_process.stdout, select.POLLIN)
+
+        while not boot_completed and timeout >= (time.time() - start_time):
+            poll_result = poll_obj.poll(0)
+            if poll_result:
+                line = self.qemu_process.stdout.readline().rstrip(b'\r\n')
+                line = line.decode(sys.stdout.encoding or 'utf-8', "replace") if line else ''
+                self.logger.debug(line)
+                if self.banner in line:
+                    self.logger.info('Target is ready.')
+                    boot_completed = True
+                    break
+
+        if not boot_completed:
+            raise TargetStableError(f'Target could not finish boot up in {timeout} seconds!')
+
+    def terminate_qemu(self):
+        self.logger.debug('Terminating QEMU...')
+        os.killpg(self.qemu_process.pid, signal.SIGTERM)
+
 
 def _get_model_name(section):
     name_string = section['model name']

doc/overview.rst

@@ -9,6 +9,8 @@ There are currently four target interfaces:
 - :class:`~devlib.target.ChromeOsTarget`: for interacting with ChromeOS devices
   over SSH, and their Android containers over adb.
 - :class:`~devlib.target.LocalLinuxTarget`: for interacting with the local Linux host.
+- :class:`~devlib.target.QEMULinuxTarget`: for interacting with an emulated Linux
+  target on QEMU.
 
 They all work in more-or-less the same way, with the major difference being in
 how connection settings are specified; though there may also be a few APIs

doc/target.rst

@@ -780,6 +780,50 @@ Android Target
    support the majority of newer devices.
 
 
+QEMU Linux Target
+---------------
+
+.. class:: QEMULinuxTarget(kernel_image, arch='aarch64', cpu_types='-cpu cortex-a72', initrd_image=str(), mem_size=512, num_cores=2, num_threads=2, cmdline='console=ttyAMA0', enable_kvm=True, boot_timeout=60, banner='Welcome to Buildroot', connect=True, connection_settings=None)
+
+    :class:`QEMULinuxTarget` is a subclass of :class:`LinuxTarget` with
+    additional features necessary for launching an emulated Linux guest on QEMU.
+    It implements :meth:`wait_boot_complete` to ensure spinned guest can show
+    login prompt in :param:`boot_timeout` seconds.
+
+    :param kernel_image: This is the location of kernel image
+        (e.g., ``bzImage``) which will be used as target's kernel.
+
+    :param arch: Architecture type. Defaults to ``aarch64``.
+
+    :param cpu_types: List of CPU ids for QEMU. Defaults to ``-cpu cortex-a72``.
+        Valid only for Arm architectures.
+
+    :param initrd_image: This is an optional parameter which points to the
+        location of initrd image (e.g., ``rootfs.cpio.xz``) which will be used
+        as target's root filesystem if kernel does not include one already.
+
+    :param mem_size: Size of guest memory in MiB.
+
+    :param num_cores: Number of CPU cores. Guest will have ``2`` cores in
+        default settings.
+
+    :param num_threads: Number of CPU threads. Set to ``2`` by defaults.
+
+    :param cmdline: Kernel command line parameter. It only specifies console
+        device in default (i.e., ``console=ttyAMA0``) which is valid for Arm
+        architectures. It should be changed to ``ttyS0`` for x86 platforms.
+
+    :param enable_kvm: Specifies if KVM will be used as accelerator in QEMU
+        or not. Valid only for x86 architectures. Enabled by default for
+        improving QEMU performance.
+
+    :param boot_timeout: Timeout for login prompt of guest in seconds.
+        It's set to ``60`` seconds by default.
+
+    :param banner: This is the system banner displayed at login which is set to
+        ``Welcome to Buildroot`` by default.
+
+
 ChromeOS Target
 ---------------
 

tests/spin_targets.py

@@ -22,7 +22,7 @@
 
 import os
 from unittest import TestCase
-from devlib import AndroidTarget, LinuxTarget, LocalLinuxTarget
+from devlib import AndroidTarget, LinuxTarget, LocalLinuxTarget, QEMULinuxTarget
 from devlib.utils.misc import get_random_string
 from devlib.exception import TargetStableError
 
@@ -87,6 +87,15 @@ def create_targets():
                                          working_directory='/tmp/devlib-target')
             targets.append(ll_target)
 
+            q_target = QEMULinuxTarget(kernel_image='/home/user/devlib/tools/buildroot/buildroot-v2023.11.1/output/images/Image',
+                                       connection_settings={'host': '127.0.0.1',
+                                                            'port': 8022,
+                                                            'username': 'root',
+                                                            'password': 'root',
+                                                            'strict_host_check': False},
+                                       working_directory='/tmp/devlib-target')
+            targets.append(q_target)
+
             return targets
 
         method_name = self.__class__.test_read_multiline_values.__qualname__