From 03eda20e25610c344264ec8f5714d8ce32e70ff5 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 1 Dec 2020 16:28:10 -0500 Subject: [PATCH 1/4] integration_tests: introduce IntegrationImage abstraction In order to skip tests based on OS or OS release, we need a way of gathering and storing that information. `IntegrationImage` is a class to contain that information and is now used to deconstruct the more complex OS_IMAGE values we now support. --- tests/integration_tests/clouds.py | 48 +++++++++++++++++-- .../integration_tests/integration_settings.py | 6 ++- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index 4d5c2c2a795..d1f17bd567a 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -25,6 +25,46 @@ log = logging.getLogger('integration_testing') +class ImageSpecification: + """A specification of an image to launch for testing. + + If either of ``os`` and ``release`` are not specified, an attempt will be + made to infer the correct values for these on instantiation. + + :param image_id: + The image identifier used by the rest of the codebase to launch this + image. + :param os: + An optional string describing the operating system this image is for + (e.g. "ubuntu", "rhel", "freebsd"). + :param release: + A optional string describing the operating system release (e.g. + "focal", "8"; the exact values here will depend on the OS). + """ + + def __init__( + self, + image_id: str, + os: "Optional[str]" = None, + release: "Optional[str]" = None, + ): + self.image_id = image_id + self.os = os + self.release = release + log.info( + "Detected image: image_id=%s os=%s release=%s", + self.image_id, + self.os, + self.release, + ) + + @classmethod + def from_os_image(cls): + """Return an ImageSpecification for integration_settings.OS_IMAGE.""" + parts = integration_settings.OS_IMAGE.split("::", 2) + return cls(*parts) + + class IntegrationCloud(ABC): datasource = None # type: Optional[str] integration_instance_cls = IntegrationInstance @@ -57,13 +97,11 @@ def _get_cloud_instance(self): raise NotImplementedError def _get_initial_image(self): - _released_image_id = self.settings.OS_IMAGE + image = ImageSpecification.from_os_image() try: - _released_image_id = self.cloud_instance.released_image( - self.settings.OS_IMAGE) + return self.cloud_instance.released_image(image.image_id) except (ValueError, IndexError): - pass - return _released_image_id + return image.image_id def _perform_launch(self, launch_kwargs): pycloudlib_instance = self.cloud_instance.launch(**launch_kwargs) diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py index 94d54f74c1b..f5ceb4176c0 100644 --- a/tests/integration_tests/integration_settings.py +++ b/tests/integration_tests/integration_settings.py @@ -22,8 +22,10 @@ INSTANCE_TYPE = None # Determines the base image to use or generate new images from. -# Can be the name of the OS if running a stock image, -# otherwise the id of the image being used if using a custom image +# +# This can be the name of an Ubuntu release, or in the format +# [::[::]]. If given, os and release should describe +# the image specified by image_id. OS_IMAGE = 'focal' # Populate if you want to use a pre-launched instance instead of From e3cfc558221f705376e40bee2862f7555e1b6671 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 1 Dec 2020 17:11:48 -0500 Subject: [PATCH 2/4] integration_tests: infer correct OS/release for Ubuntu releases --- tests/integration_tests/clouds.py | 21 ++++++++++++++++++- .../integration_tests/integration_settings.py | 3 ++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index d1f17bd567a..8cdb3fccae4 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -6,7 +6,7 @@ from pycloudlib.lxd.instance import LXDInstance import cloudinit -from cloudinit.subp import subp +from cloudinit.subp import subp, ProcessExecutionError from tests.integration_tests import integration_settings from tests.integration_tests.instances import ( IntegrationEc2Instance, @@ -25,6 +25,19 @@ log = logging.getLogger('integration_testing') +def _get_ubuntu_series() -> list: + """Use distro-info-data's ubuntu.csv to get a list of Ubuntu series""" + out = "" + try: + out, _err = subp(["ubuntu-distro-info", "-a"]) + except ProcessExecutionError: + log.info( + "ubuntu-distro-info (from the distro-info package) must be" + " installed to guess Ubuntu os/release" + ) + return out.splitlines() + + class ImageSpecification: """A specification of an image to launch for testing. @@ -48,6 +61,12 @@ def __init__( os: "Optional[str]" = None, release: "Optional[str]" = None, ): + if image_id in _get_ubuntu_series(): + if os is None: + os = "ubuntu" + if release is None: + release = image_id + self.image_id = image_id self.os = os self.release = release diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py index f5ceb4176c0..07a6d54119e 100644 --- a/tests/integration_tests/integration_settings.py +++ b/tests/integration_tests/integration_settings.py @@ -25,7 +25,8 @@ # # This can be the name of an Ubuntu release, or in the format # [::[::]]. If given, os and release should describe -# the image specified by image_id. +# the image specified by image_id. (Ubuntu releases are converted to this +# format internally; in this case, to "focal::ubuntu::focal".) OS_IMAGE = 'focal' # Populate if you want to use a pre-launched instance instead of From 15f76dac9695e8e7ce82c20e961bdb1b1a8974a0 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 1 Dec 2020 10:39:46 -0500 Subject: [PATCH 3/4] doc: add Image Selection section to integration_tests.rst --- doc/rtd/topics/integration_tests.rst | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/doc/rtd/topics/integration_tests.rst b/doc/rtd/topics/integration_tests.rst index aeda326cab4..3cfca31e504 100644 --- a/doc/rtd/topics/integration_tests.rst +++ b/doc/rtd/topics/integration_tests.rst @@ -14,6 +14,36 @@ laid out in :ref:`unit_testing` should be followed for integration tests. Setup is accomplished via a set of fixtures located in ``tests/integration_tests/conftest.py``. +Image Selection +=============== + +Each integration testing run uses a single image as its basis. This +image is configured using the ``OS_IMAGE`` variable; see +:ref:`Configuration` for details of how configuration works. + +``OS_IMAGE`` can take two types of value: an Ubuntu series name (e.g. +"focal"), or an image specification. If an Ubuntu series name is +given, then the most recent image for that series on the target cloud +will be used. For other use cases, an image specification is used. + +In its simplest form, an image specification can simply be a cloud's +image ID (e.g. "ami-deadbeef", "ubuntu:focal"). In this case, the +image so-identified will be used as the basis for this testing run. + +This has a drawback, however: as we do not know what OS or release is +within the image, the integration testing framework will run *all* +tests against the image in question. If it's a RHEL8 image, then we +would expect Ubuntu-specific tests to fail (and vice versa). + +To address this, a full image specification can be given. This is of +the form: ``[::[:: Date: Wed, 2 Dec 2020 16:44:22 -0500 Subject: [PATCH 4/4] integration_tests: introduce generic OS marking/skipping And mark all current tests that are Ubuntu-only with the `ubuntu` mark. --- tests/integration_tests/conftest.py | 12 ++++++++++-- .../modules/test_apt_configure_sources_list.py | 1 + .../modules/test_package_update_upgrade_install.py | 1 + tests/integration_tests/modules/test_snap.py | 1 + .../integration_tests/modules/test_ssh_import_id.py | 5 +++++ tests/integration_tests/modules/test_users_groups.py | 5 +++++ tox.ini | 1 + 7 files changed, 24 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index d7e0fca2c1c..cc545b0fb06 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -10,12 +10,13 @@ from tests.integration_tests import integration_settings from tests.integration_tests.clouds import ( + AzureCloud, Ec2Cloud, GceCloud, - AzureCloud, - OciCloud, + ImageSpecification, LxdContainerCloud, LxdVmCloud, + OciCloud, ) from tests.integration_tests.instances import IntegrationInstance @@ -32,6 +33,7 @@ 'lxd_container': LxdContainerCloud, 'lxd_vm': LxdVmCloud, } +os_list = ["ubuntu"] session_start_time = datetime.datetime.now().strftime('%y%m%d%H%M%S') @@ -60,6 +62,12 @@ def pytest_runtest_setup(item): if supported_platforms and current_platform not in supported_platforms: pytest.skip(unsupported_message) + image = ImageSpecification.from_os_image() + current_os = image.os + supported_os_set = set(os_list).intersection(test_marks) + if current_os and supported_os_set and current_os not in supported_os_set: + pytest.skip("Cannot run on OS {}".format(current_os)) + # disable_subp_usage is defined at a higher level, but we don't # want it applied here diff --git a/tests/integration_tests/modules/test_apt_configure_sources_list.py b/tests/integration_tests/modules/test_apt_configure_sources_list.py index d2bcc61a7fe..28cbe19ff05 100644 --- a/tests/integration_tests/modules/test_apt_configure_sources_list.py +++ b/tests/integration_tests/modules/test_apt_configure_sources_list.py @@ -40,6 +40,7 @@ @pytest.mark.ci +@pytest.mark.ubuntu class TestAptConfigureSourcesList: @pytest.mark.user_data(USER_DATA) diff --git a/tests/integration_tests/modules/test_package_update_upgrade_install.py b/tests/integration_tests/modules/test_package_update_upgrade_install.py index 8a38ad8417b..28d741bc9e7 100644 --- a/tests/integration_tests/modules/test_package_update_upgrade_install.py +++ b/tests/integration_tests/modules/test_package_update_upgrade_install.py @@ -26,6 +26,7 @@ """ +@pytest.mark.ubuntu @pytest.mark.user_data(USER_DATA) class TestPackageUpdateUpgradeInstall: diff --git a/tests/integration_tests/modules/test_snap.py b/tests/integration_tests/modules/test_snap.py index b626f6b03ad..481edbaaa85 100644 --- a/tests/integration_tests/modules/test_snap.py +++ b/tests/integration_tests/modules/test_snap.py @@ -20,6 +20,7 @@ @pytest.mark.ci +@pytest.mark.ubuntu class TestSnap: @pytest.mark.user_data(USER_DATA) diff --git a/tests/integration_tests/modules/test_ssh_import_id.py b/tests/integration_tests/modules/test_ssh_import_id.py index 45d37d6ca33..3db573b548d 100644 --- a/tests/integration_tests/modules/test_ssh_import_id.py +++ b/tests/integration_tests/modules/test_ssh_import_id.py @@ -3,6 +3,10 @@ This test specifies ssh keys to be imported by the ``ssh_import_id`` module and then checks that if the ssh keys were successfully imported. +TODO: +* This test assumes that SSH keys will be imported into the /home/ubuntu; this + will need modification to run on other OSes. + (This is ported from ``tests/cloud_tests/testcases/modules/ssh_import_id.yaml``.)""" @@ -18,6 +22,7 @@ @pytest.mark.ci +@pytest.mark.ubuntu class TestSshImportId: @pytest.mark.user_data(USER_DATA) diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py index 6a51f5a633a..ee08d87be16 100644 --- a/tests/integration_tests/modules/test_users_groups.py +++ b/tests/integration_tests/modules/test_users_groups.py @@ -2,6 +2,10 @@ This test specifies a number of users and groups via user-data, and confirms that they have been configured correctly in the system under test. + +TODO: +* This test assumes that the "ubuntu" user will be created when "default" is + specified; this will need modification to run on other OSes. """ import re @@ -41,6 +45,7 @@ @pytest.mark.ci @pytest.mark.user_data(USER_DATA) class TestUsersGroups: + @pytest.mark.ubuntu @pytest.mark.parametrize( "getent_args,regex", [ diff --git a/tox.ini b/tox.ini index 022b918d6c5..df1deb6ff1e 100644 --- a/tox.ini +++ b/tox.ini @@ -179,3 +179,4 @@ markers = user_data: the user data to be passed to the test instance instance_name: the name to be used for the test instance sru_2020_11: test is part of the 2020/11 SRU verification + ubuntu: this test should run on Ubuntu