From 24a084e64c3c9c982d972af8b3174af2baa79f0b Mon Sep 17 00:00:00 2001 From: Metin Kaya Date: Fri, 9 Feb 2024 16:03:52 +0000 Subject: [PATCH 1/4] target: Address pylint issues in ChromeOsTarget class Also clean a mutable default value (``modules=[]`` in ``ChromeOsTarget`` class). Signed-off-by: Metin Kaya --- devlib/target.py | 58 +++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/devlib/target.py b/devlib/target.py index 2bd4d85ac..0f957201e 100644 --- a/devlib/target.py +++ b/devlib/target.py @@ -2939,7 +2939,7 @@ def _get_part_name(section): variant = section.get('CPU variant', '0x0') name = get_cpu_name(*list(map(integer, [implementer, part, variant]))) if name is None: - name = '{}/{}/{}'.format(implementer, part, variant) + name = f'{implementer}/{part}/{variant}' return name @@ -2973,10 +2973,13 @@ def process_node(node, path, value): class ChromeOsTarget(LinuxTarget): + """ + Class for interacting with ChromeOS targets. + """ os = 'chromeos' - # pylint: disable=too-many-locals + # pylint: disable=too-many-locals,too-many-arguments def __init__(self, connection_settings=None, platform=None, @@ -3009,17 +3012,17 @@ def __init__(self, if key in ssh_conn_params ) - super(ChromeOsTarget, self).__init__(connection_settings=self.ssh_connection_settings, - platform=platform, - working_directory=working_directory, - executables_directory=executables_directory, - connect=False, - modules=modules, - load_default_modules=load_default_modules, - shell_prompt=shell_prompt, - conn_cls=SshConnection, - is_container=is_container, - max_async=max_async) + super().__init__(connection_settings=self.ssh_connection_settings, + platform=platform, + working_directory=working_directory, + executables_directory=executables_directory, + connect=False, + modules=modules, + load_default_modules=load_default_modules, + shell_prompt=shell_prompt, + conn_cls=SshConnection, + is_container=is_container, + max_async=max_async) # We can't determine if the target supports android until connected to the linux host so # create unconditionally. @@ -3037,16 +3040,15 @@ def __init__(self, self.android_connection_settings['device'] = connection_settings.get('host', None) self.android_container = AndroidTarget(connection_settings=self.android_connection_settings, - platform=platform, - working_directory=android_working_directory, - executables_directory=android_executables_directory, - connect=False, - modules=[], # Only use modules with linux target - load_default_modules=False, - shell_prompt=shell_prompt, - conn_cls=AdbConnection, - package_data_directory=package_data_directory, - is_container=True) + platform=platform, + working_directory=android_working_directory, + executables_directory=android_executables_directory, + connect=False, + load_default_modules=False, + shell_prompt=shell_prompt, + conn_cls=AdbConnection, + package_data_directory=package_data_directory, + is_container=True) if connect: self.connect() @@ -3056,15 +3058,15 @@ def __getattr__(self, attr): if not present, use android implementation if available. """ try: - return super(ChromeOsTarget, self).__getattribute__(attr) + return super().__getattribute__(attr) except AttributeError: if hasattr(self.android_container, attr): return getattr(self.android_container, attr) - else: - raise + raise - def connect(self, timeout=30, check_boot_completed=True, max_async=None): - super(ChromeOsTarget, self).connect( + @asyn.asyncf + async def connect(self, timeout=30, check_boot_completed=True, max_async=None): + super().connect( timeout=timeout, check_boot_completed=check_boot_completed, max_async=max_async, From 1bd22c4abb37498eefe4e173dcc73c811cf0b669 Mon Sep 17 00:00:00 2001 From: Metin Kaya Date: Fri, 9 Feb 2024 15:48:33 +0000 Subject: [PATCH 2/4] target: tests: Add support for testing ChromeOS targets We can mimic ChromeOS target by combining a QEMU guest (for Linux bindings of ``ChromeOsTarget`` class) with a Android virtual desktop (for Android bits of ``ChromeOsTarget``). Note that Android bindings of ``ChromeOsTarget`` class also requires existence of ``/opt/google/containers/android`` folder on the Linux guest. Signed-off-by: Metin Kaya --- tests/target_configs.yaml.example | 8 ++++++++ tests/test_target.py | 31 +++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/tests/target_configs.yaml.example b/tests/target_configs.yaml.example index 6ad859a8c..a47acea82 100644 --- a/tests/target_configs.yaml.example +++ b/tests/target_configs.yaml.example @@ -3,6 +3,14 @@ AndroidTarget: connection_settings: device: 'emulator-5554' +ChromeOsTarget: + entry-0: + connection_settings: + device: 'emulator-5556' + host: 'example.com' + username: 'username' + password: 'password' + LinuxTarget: entry-0: connection_settings: diff --git a/tests/test_target.py b/tests/test_target.py index 63f806f82..05f56851c 100644 --- a/tests/test_target.py +++ b/tests/test_target.py @@ -20,7 +20,7 @@ from pprint import pp import pytest -from devlib import AndroidTarget, LinuxTarget, LocalLinuxTarget, QEMUTargetRunner +from devlib import AndroidTarget, ChromeOsTarget, LinuxTarget, LocalLinuxTarget, QEMUTargetRunner from devlib.utils.android import AdbConnection from devlib.utils.misc import load_struct_from_yaml @@ -53,6 +53,16 @@ def build_targets(): l_target = LinuxTarget(connection_settings=entry['connection_settings']) targets.append((l_target, None)) + if target_configs.get('ChromeOsTarget') is not None: + print('> ChromeOS targets:') + for entry in target_configs['ChromeOsTarget'].values(): + pp(entry) + c_target = ChromeOsTarget( + connection_settings=entry['connection_settings'], + working_directory='/tmp/devlib-target', + ) + targets.append((c_target, None)) + if target_configs.get('LocalLinuxTarget') is not None: print('> LocalLinux targets:') for entry in target_configs['LocalLinuxTarget'].values(): @@ -72,7 +82,24 @@ def build_targets(): qemu_settings=qemu_settings, connection_settings=connection_settings, ) - targets.append((qemu_runner.target, qemu_runner)) + + if entry.get('ChromeOsTarget') is None: + targets.append((qemu_runner.target, qemu_runner)) + continue + + # Leave termination of QEMU runner to ChromeOS target. + targets.append((qemu_runner.target, None)) + + print('> ChromeOS targets:') + pp(entry['ChromeOsTarget']) + c_target = ChromeOsTarget( + connection_settings={ + **entry['ChromeOsTarget']['connection_settings'], + **qemu_runner.target.connection_settings, + }, + working_directory='/tmp/devlib-target', + ) + targets.append((c_target, qemu_runner)) return targets From c659c2b95a408e305e5b816f610afed85b978ef2 Mon Sep 17 00:00:00 2001 From: Metin Kaya Date: Thu, 21 Mar 2024 11:32:32 +0000 Subject: [PATCH 3/4] test_target.py: Allow specifying connection timeout for Android targets Default connection timeout (30 secs) may be insufficient for some test setups or in some conditions. Thus, support specifying timeout parameter in target configuration file. Signed-off-by: Metin Kaya --- tests/target_configs.yaml.example | 1 + tests/test_target.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/target_configs.yaml.example b/tests/target_configs.yaml.example index a47acea82..1154ea8b9 100644 --- a/tests/target_configs.yaml.example +++ b/tests/target_configs.yaml.example @@ -1,5 +1,6 @@ AndroidTarget: entry-0: + timeout: 60 connection_settings: device: 'emulator-5554' diff --git a/tests/test_target.py b/tests/test_target.py index 05f56851c..2d59a2374 100644 --- a/tests/test_target.py +++ b/tests/test_target.py @@ -41,9 +41,11 @@ def build_targets(): for entry in target_configs['AndroidTarget'].values(): pp(entry) a_target = AndroidTarget( + connect=False, connection_settings=entry['connection_settings'], conn_cls=lambda **kwargs: AdbConnection(adb_as_root=True, **kwargs), ) + a_target.connect(timeout=entry.get('timeout', 60)) targets.append((a_target, None)) if target_configs.get('LinuxTarget') is not None: From 3e708bb11430d7791a04a1d923ccb4c8d4904e82 Mon Sep 17 00:00:00 2001 From: Metin Kaya Date: Tue, 6 Feb 2024 15:46:53 +0000 Subject: [PATCH 4/4] tools/docker: Add Docker image support for devlib Introduce a Dockerfile in order to create Docker image for devlib and ``run_tests.sh`` script to test Android, Linux, LocalLinux, and QEMU targets on the Docker image. The Dockerfile forks from ``Ubuntu-22.04``, installs required system packages, checks out ``master`` branch of devlib, installs devlib, creates Android virtual devices via ``tools/android/install_base.sh``, and QEMU images for aarch64 and x86_84 architectures. Note that Android command line tools version, buildroot and devlib branches can be customized via environment variables. Signed-off-by: Metin Kaya --- doc/tools.rst | 24 ++++++++++ tools/docker/Dockerfile | 77 ++++++++++++++++++++++++++++++++ tools/docker/run_tests.sh | 41 +++++++++++++++++ tools/docker/target_configs.yaml | 43 ++++++++++++++++++ 4 files changed, 185 insertions(+) create mode 100644 tools/docker/Dockerfile create mode 100755 tools/docker/run_tests.sh create mode 100644 tools/docker/target_configs.yaml diff --git a/doc/tools.rst b/doc/tools.rst index c2134142c..d007301bf 100644 --- a/doc/tools.rst +++ b/doc/tools.rst @@ -83,3 +83,27 @@ system, you may want to run commands similar to these: See https://buildroot.org/downloads/manual/manual.html for details. +Docker support +-------------- + +A Docker image for devlib can be created via ``tools/docker/Dockerfile``. + +Once the Docker image is run, ``tools/docker/run_tests.sh`` script can execute +tests for Android, Linux, LocalLinux, and QEMU targets. + +The Dockerfile forks from ``Ubuntu-22.04``, installs required system packages, +checks out ``master`` branch of devlib, installs devlib, creates Android +virtual devices via ``tools/android/install_base.sh``, and QEMU images for +aarch64 and x86_84 architectures. + +Version Android command line tools (``CMDLINE_VERSION``), buildroot +(``BUILDROOT_VERSION``) and devlib (``DEVLIB_REF``) branches can be customized +for the Docker image via aforementioned environment variables. + +.. code:: shell + + cd tools/docker + docker build -t devlib . + docker run -it --privileged devlib + /devlib/tools/docker/run_tests.sh + diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile new file mode 100644 index 000000000..e52a7d613 --- /dev/null +++ b/tools/docker/Dockerfile @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2024, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This Dockerfile creates an image to run devlib CI tests. +# +# Running ``docker build -t devlib .`` command in ``tools/docker`` directory +# creates the docker image. +# +# The image can be runned via ``docker run -it --privileged devlib`` command. +# + +FROM ubuntu:22.04 + +ENV DEBIAN_FRONTEND noninteractive + +ENV DEVLIB_REF master + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + aapt \ + bc \ + bison \ + build-essential \ + cmake \ + cpio \ + file \ + flex \ + git \ + libelf-dev \ + libncurses5-dev \ + libssl-dev \ + locales \ + python3-pip \ + qemu-system-arm \ + qemu-system-x86 \ + rsync \ + sudo \ + unzip \ + wget \ + vim \ + xz-utils + +RUN apt-get -y autoremove && \ + apt-get -y autoclean && \ + apt-get clean && \ + rm -rf /var/cache/apt + +RUN git clone -b ${DEVLIB_REF} -v https://github.com/ARM-software/devlib.git /devlib +RUN cd /devlib && \ + pip install --upgrade pip setuptools wheel && \ + pip install .[full] + +# Set CMDLINE_VERSION environment variable if you want to use a specific +# version of Android command line tools rather than default which is +# ``11076708`` as of writing this comment. +RUN cd /devlib/tools/android && ./install_base.sh + +# Set BUILDROOT_VERSION environment variable if you want to use a specific +# branch of buildroot rather than default which is ``2023.11.1`` as of +# writing this comment. +RUN cd /devlib/tools/buildroot && \ + ./generate-kernel-initrd.sh && \ + ./generate-kernel-initrd.sh -a x86_64 + diff --git a/tools/docker/run_tests.sh b/tools/docker/run_tests.sh new file mode 100755 index 000000000..e6e1a8af3 --- /dev/null +++ b/tools/docker/run_tests.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (C) 2024, ARM Limited and contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Prepare the groundwork and run tests/test_target.py on the Docker image. +# + +set -eu + +ANDROID_HOME="/devlib/tools/android/android-sdk-linux" +export ANDROID_HOME +export ANDROID_USER_HOME="${ANDROID_HOME}/.android" +export ANDROID_EMULATOR_HOME="${ANDROID_HOME}/.android" +export PATH=${ANDROID_HOME}/platform-tools/:${PATH} + +EMULATOR="${ANDROID_HOME}/emulator/emulator" +EMULATOR_ARGS="-no-window -no-snapshot -memory 2048" +${EMULATOR} -avd devlib-p6-12 ${EMULATOR_ARGS} & +${EMULATOR} -avd devlib-p6-14 ${EMULATOR_ARGS} & +${EMULATOR} -avd devlib-chromeos ${EMULATOR_ARGS} & + +echo "Waiting 30 seconds for Android virtual devices to finish boot up..." +sleep 30 + +cd /devlib +cp -f tools/docker/target_configs.yaml tests/ +python3 -m pytest --log-cli-level DEBUG ./tests/test_target.py diff --git a/tools/docker/target_configs.yaml b/tools/docker/target_configs.yaml new file mode 100644 index 000000000..d616822c3 --- /dev/null +++ b/tools/docker/target_configs.yaml @@ -0,0 +1,43 @@ +AndroidTarget: + # Android-12, Pixel-6 + entry-0: + timeout: 60 + connection_settings: + device: 'emulator-5554' + + # Android-14, Pixel-6 + entry-1: + connection_settings: + device: 'emulator-5556' + + # Android-13, Pixel tablet + entry-2: + connection_settings: + device: 'emulator-5558' + +LocalLinuxTarget: + entry-0: + connection_settings: + unrooted: True + +QEMUTargetRunner: + entry-0: + qemu_settings: + kernel_image: '/devlib/tools/buildroot/buildroot-v2023.11.1-aarch64/output/images/Image' + + ChromeOsTarget: + connection_settings: + device: 'emulator-5558' + + entry-1: + connection_settings: + port: 8023 + + qemu_settings: + kernel_image: '/devlib/tools/buildroot/buildroot-v2023.11.1-x86_64/output/images/bzImage' + arch: 'x86_64' + cmdline: 'console=ttyS0' + + ChromeOsTarget: + connection_settings: + device: 'emulator-5558'