tools/docker: Add Docker image support for devlib

devlib/target.py

@@ -2939,7 +2939,7 @@ def _get_part_name(section):
     variant = section.get('CPU variant', '0x0')
     name = get_cpu_name(*list(map(integer, [implementer, part, variant])))
     if name is None:
-        name = '{}/{}/{}'.format(implementer, part, variant)
+        name = f'{implementer}/{part}/{variant}'
     return name
 
 
@@ -2973,10 +2973,13 @@ def process_node(node, path, value):
 
 
 class ChromeOsTarget(LinuxTarget):
+    """
+    Class for interacting with ChromeOS targets.
+    """
 
     os = 'chromeos'
 
-    # pylint: disable=too-many-locals
+    # pylint: disable=too-many-locals,too-many-arguments
     def __init__(self,
                  connection_settings=None,
                  platform=None,
@@ -3009,17 +3012,17 @@ def __init__(self,
             if key in ssh_conn_params
         )
 
-        super(ChromeOsTarget, self).__init__(connection_settings=self.ssh_connection_settings,
-                                             platform=platform,
-                                             working_directory=working_directory,
-                                             executables_directory=executables_directory,
-                                             connect=False,
-                                             modules=modules,
-                                             load_default_modules=load_default_modules,
-                                             shell_prompt=shell_prompt,
-                                             conn_cls=SshConnection,
-                                             is_container=is_container,
-                                             max_async=max_async)
+        super().__init__(connection_settings=self.ssh_connection_settings,
+            platform=platform,
+            working_directory=working_directory,
+            executables_directory=executables_directory,
+            connect=False,
+            modules=modules,
+            load_default_modules=load_default_modules,
+            shell_prompt=shell_prompt,
+            conn_cls=SshConnection,
+            is_container=is_container,
+            max_async=max_async)
 
         # We can't determine if the target supports android until connected to the linux host so
         # create unconditionally.
@@ -3037,16 +3040,15 @@ def __init__(self,
             self.android_connection_settings['device'] = connection_settings.get('host', None)
 
         self.android_container = AndroidTarget(connection_settings=self.android_connection_settings,
-                                               platform=platform,
-                                               working_directory=android_working_directory,
-                                               executables_directory=android_executables_directory,
-                                               connect=False,
-                                               modules=[], # Only use modules with linux target
-                                               load_default_modules=False,
-                                               shell_prompt=shell_prompt,
-                                               conn_cls=AdbConnection,
-                                               package_data_directory=package_data_directory,
-                                               is_container=True)
+            platform=platform,
+            working_directory=android_working_directory,
+            executables_directory=android_executables_directory,
+            connect=False,
+            load_default_modules=False,
+            shell_prompt=shell_prompt,
+            conn_cls=AdbConnection,
+            package_data_directory=package_data_directory,
+            is_container=True)
         if connect:
             self.connect()
 
@@ -3056,15 +3058,15 @@ def __getattr__(self, attr):
         if not present, use android implementation if available.
         """
         try:
-            return super(ChromeOsTarget, self).__getattribute__(attr)
+            return super().__getattribute__(attr)
         except AttributeError:
             if hasattr(self.android_container, attr):
                 return getattr(self.android_container, attr)
-            else:
-                raise
+            raise
 
-    def connect(self, timeout=30, check_boot_completed=True, max_async=None):
-        super(ChromeOsTarget, self).connect(
+    @asyn.asyncf
+    async def connect(self, timeout=30, check_boot_completed=True, max_async=None):
+        super().connect(
             timeout=timeout,
             check_boot_completed=check_boot_completed,
             max_async=max_async,

doc/tools.rst

@@ -83,3 +83,27 @@ system, you may want to run commands similar to these:
 
 See https://buildroot.org/downloads/manual/manual.html for details.
 
+Docker support
+--------------
+
+A Docker image for devlib can be created via ``tools/docker/Dockerfile``.
+
+Once the Docker image is run, ``tools/docker/run_tests.sh`` script can execute
+tests for Android, Linux, LocalLinux, and QEMU targets.
+
+The Dockerfile forks from ``Ubuntu-22.04``, installs required system packages,
+checks out ``master`` branch of devlib, installs devlib, creates Android
+virtual devices via ``tools/android/install_base.sh``, and QEMU images for
+aarch64 and x86_84 architectures.
+
+Version Android command line tools (``CMDLINE_VERSION``), buildroot
+(``BUILDROOT_VERSION``) and devlib (``DEVLIB_REF``) branches can be customized
+for the Docker image via aforementioned environment variables.
+
+.. code:: shell
+
+   cd tools/docker
+   docker build -t devlib .
+   docker run -it --privileged devlib
+   /devlib/tools/docker/run_tests.sh
+

tests/target_configs.yaml.example

@@ -1,8 +1,17 @@
 AndroidTarget:
   entry-0:
+    timeout: 60
     connection_settings:
       device: 'emulator-5554'
 
+ChromeOsTarget:
+  entry-0:
+    connection_settings:
+      device: 'emulator-5556'
+      host: 'example.com'
+      username: 'username'
+      password: 'password'
+
 LinuxTarget:
   entry-0:
     connection_settings:

tests/test_target.py

@@ -20,7 +20,7 @@
 from pprint import pp
 import pytest
 
-from devlib import AndroidTarget, LinuxTarget, LocalLinuxTarget, QEMUTargetRunner
+from devlib import AndroidTarget, ChromeOsTarget, LinuxTarget, LocalLinuxTarget, QEMUTargetRunner
 from devlib.utils.android import AdbConnection
 from devlib.utils.misc import load_struct_from_yaml
 
@@ -41,9 +41,11 @@ def build_targets():
         for entry in target_configs['AndroidTarget'].values():
             pp(entry)
             a_target = AndroidTarget(
+                connect=False,
                 connection_settings=entry['connection_settings'],
                 conn_cls=lambda **kwargs: AdbConnection(adb_as_root=True, **kwargs),
             )
+            a_target.connect(timeout=entry.get('timeout', 60))
             targets.append((a_target, None))
 
     if target_configs.get('LinuxTarget') is not None:
@@ -53,6 +55,16 @@ def build_targets():
             l_target = LinuxTarget(connection_settings=entry['connection_settings'])
             targets.append((l_target, None))
 
+    if target_configs.get('ChromeOsTarget') is not None:
+        print('> ChromeOS targets:')
+        for entry in target_configs['ChromeOsTarget'].values():
+            pp(entry)
+            c_target = ChromeOsTarget(
+                connection_settings=entry['connection_settings'],
+                working_directory='/tmp/devlib-target',
+            )
+            targets.append((c_target, None))
+
     if target_configs.get('LocalLinuxTarget') is not None:
         print('> LocalLinux targets:')
         for entry in target_configs['LocalLinuxTarget'].values():
@@ -72,7 +84,24 @@ def build_targets():
                 qemu_settings=qemu_settings,
                 connection_settings=connection_settings,
             )
-            targets.append((qemu_runner.target, qemu_runner))
+
+            if entry.get('ChromeOsTarget') is None:
+                targets.append((qemu_runner.target, qemu_runner))
+                continue
+
+            # Leave termination of QEMU runner to ChromeOS target.
+            targets.append((qemu_runner.target, None))
+
+            print('> ChromeOS targets:')
+            pp(entry['ChromeOsTarget'])
+            c_target = ChromeOsTarget(
+                connection_settings={
+                    **entry['ChromeOsTarget']['connection_settings'],
+                    **qemu_runner.target.connection_settings,
+                },
+                working_directory='/tmp/devlib-target',
+            )
+            targets.append((c_target, qemu_runner))
 
     return targets
 

tools/docker/Dockerfile

@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright (C) 2024, ARM Limited and contributors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This Dockerfile creates an image to run devlib CI tests.
+#
+# Running ``docker build -t devlib .`` command in ``tools/docker`` directory
+# creates the docker image.
+#
+# The image can be runned via ``docker run -it --privileged devlib`` command.
+#
+
+FROM ubuntu:22.04
+
+ENV DEBIAN_FRONTEND noninteractive
+
+ENV DEVLIB_REF master
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    aapt \
+    bc \
+    bison \
+    build-essential \
+    cmake \
+    cpio \
+    file \
+    flex \
+    git \
+    libelf-dev \
+    libncurses5-dev \
+    libssl-dev \
+    locales \
+    python3-pip \
+    qemu-system-arm \
+    qemu-system-x86 \
+    rsync \
+    sudo \
+    unzip \
+    wget \
+    vim \
+    xz-utils
+
+RUN apt-get -y autoremove && \
+    apt-get -y autoclean && \
+    apt-get clean && \
+    rm -rf /var/cache/apt
+
+RUN git clone -b ${DEVLIB_REF} -v https://github.com/ARM-software/devlib.git /devlib
+RUN cd /devlib && \
+    pip install --upgrade pip setuptools wheel && \
+    pip install .[full]
+
+# Set CMDLINE_VERSION environment variable if you want to use a specific
+# version of Android command line tools rather than default which is
+# ``11076708`` as of writing this comment.
+RUN cd /devlib/tools/android && ./install_base.sh
+
+# Set BUILDROOT_VERSION environment variable if you want to use a specific
+# branch of buildroot rather than default which is ``2023.11.1`` as of
+# writing this comment.
+RUN cd /devlib/tools/buildroot && \
+    ./generate-kernel-initrd.sh && \
+    ./generate-kernel-initrd.sh -a x86_64
+

tools/docker/run_tests.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright (C) 2024, ARM Limited and contributors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Prepare the groundwork and run tests/test_target.py on the Docker image.
+#
+
+set -eu
+
+ANDROID_HOME="/devlib/tools/android/android-sdk-linux"
+export ANDROID_HOME
+export ANDROID_USER_HOME="${ANDROID_HOME}/.android"
+export ANDROID_EMULATOR_HOME="${ANDROID_HOME}/.android"
+export PATH=${ANDROID_HOME}/platform-tools/:${PATH}
+
+EMULATOR="${ANDROID_HOME}/emulator/emulator"
+EMULATOR_ARGS="-no-window -no-snapshot -memory 2048"
+${EMULATOR} -avd devlib-p6-12 ${EMULATOR_ARGS} &
+${EMULATOR} -avd devlib-p6-14 ${EMULATOR_ARGS} &
+${EMULATOR} -avd devlib-chromeos ${EMULATOR_ARGS} &
+
+echo "Waiting 30 seconds for Android virtual devices to finish boot up..."
+sleep 30
+
+cd /devlib
+cp -f tools/docker/target_configs.yaml tests/
+python3 -m pytest --log-cli-level DEBUG ./tests/test_target.py

tools/docker/target_configs.yaml

@@ -0,0 +1,43 @@
+AndroidTarget:
+  # Android-12, Pixel-6
+  entry-0:
+    timeout: 60
+    connection_settings:
+      device: 'emulator-5554'
+
+  # Android-14, Pixel-6
+  entry-1:
+    connection_settings:
+      device: 'emulator-5556'
+
+  # Android-13, Pixel tablet
+  entry-2:
+    connection_settings:
+      device: 'emulator-5558'
+
+LocalLinuxTarget:
+  entry-0:
+    connection_settings:
+      unrooted: True
+
+QEMUTargetRunner:
+  entry-0:
+    qemu_settings:
+      kernel_image: '/devlib/tools/buildroot/buildroot-v2023.11.1-aarch64/output/images/Image'
+
+    ChromeOsTarget:
+      connection_settings:
+        device: 'emulator-5558'
+
+  entry-1:
+    connection_settings:
+      port: 8023
+
+    qemu_settings:
+      kernel_image: '/devlib/tools/buildroot/buildroot-v2023.11.1-x86_64/output/images/bzImage'
+      arch: 'x86_64'
+      cmdline: 'console=ttyS0'
+
+    ChromeOsTarget:
+      connection_settings:
+        device: 'emulator-5558'