integration_tests: introduce skipping of tests by OS

doc/rtd/topics/integration_tests.rst

@@ -14,6 +14,36 @@ laid out in :ref:`unit_testing` should be followed for integration tests.
 Setup is accomplished via a set of fixtures located in
 ``tests/integration_tests/conftest.py``.
 
+Image Selection
+===============
+
+Each integration testing run uses a single image as its basis.  This
+image is configured using the ``OS_IMAGE`` variable; see
+:ref:`Configuration` for details of how configuration works.
+
+``OS_IMAGE`` can take two types of value: an Ubuntu series name (e.g.
+"focal"), or an image specification.  If an Ubuntu series name is
+given, then the most recent image for that series on the target cloud
+will be used.  For other use cases, an image specification is used.
+
+In its simplest form, an image specification can simply be a cloud's
+image ID (e.g. "ami-deadbeef", "ubuntu:focal").  In this case, the
+image so-identified will be used as the basis for this testing run.
+
+This has a drawback, however: as we do not know what OS or release is
+within the image, the integration testing framework will run *all*
+tests against the image in question.  If it's a RHEL8 image, then we
+would expect Ubuntu-specific tests to fail (and vice versa).
+
+To address this, a full image specification can be given.  This is of
+the form: ``<image_id>[::<os>[::<release]]`` where ``image_id`` is a
+cloud's image ID, ``os`` is the OS name, and ``release`` is the OS
+release name.  So, for example, Ubuntu 18.04 (Bionic Beaver) on LXD is
+``ubuntu:bionic::ubuntu::bionic`` or RHEL 8 on Amazon is
+``ami-justanexample::rhel::8``.  When a full specification is given,
+only tests which are intended for use on that OS and release will be
+executed.
+
 Image Setup
 ===========
 

tests/integration_tests/clouds.py

@@ -6,7 +6,7 @@
 from pycloudlib.lxd.instance import LXDInstance
 
 import cloudinit
-from cloudinit.subp import subp
+from cloudinit.subp import subp, ProcessExecutionError
 from tests.integration_tests import integration_settings
 from tests.integration_tests.instances import (
     IntegrationEc2Instance,
@@ -25,6 +25,65 @@
 log = logging.getLogger('integration_testing')
 
 
+def _get_ubuntu_series() -> list:
+    """Use distro-info-data's ubuntu.csv to get a list of Ubuntu series"""
+    out = ""
+    try:
+        out, _err = subp(["ubuntu-distro-info", "-a"])
+    except ProcessExecutionError:
+        log.info(
+            "ubuntu-distro-info (from the distro-info package) must be"
+            " installed to guess Ubuntu os/release"
+        )
+    return out.splitlines()
+
+
+class ImageSpecification:
+    """A specification of an image to launch for testing.
+
+    If either of ``os`` and ``release`` are not specified, an attempt will be
+    made to infer the correct values for these on instantiation.
+
+    :param image_id:
+        The image identifier used by the rest of the codebase to launch this
+        image.
+    :param os:
+        An optional string describing the operating system this image is for
+        (e.g.  "ubuntu", "rhel", "freebsd").
+    :param release:
+        A optional string describing the operating system release (e.g.
+        "focal", "8"; the exact values here will depend on the OS).
+    """
+
+    def __init__(
+        self,
+        image_id: str,
+        os: "Optional[str]" = None,
+        release: "Optional[str]" = None,
+    ):
+        if image_id in _get_ubuntu_series():
+            if os is None:
+                os = "ubuntu"
+            if release is None:
+                release = image_id
+
+        self.image_id = image_id
+        self.os = os
+        self.release = release
+        log.info(
+            "Detected image: image_id=%s os=%s release=%s",
+            self.image_id,
+            self.os,
+            self.release,
+        )
+
+    @classmethod
+    def from_os_image(cls):
+        """Return an ImageSpecification for integration_settings.OS_IMAGE."""
+        parts = integration_settings.OS_IMAGE.split("::", 2)
+        return cls(*parts)
+
+
 class IntegrationCloud(ABC):
     datasource = None  # type: Optional[str]
     integration_instance_cls = IntegrationInstance
@@ -57,13 +116,11 @@ def _get_cloud_instance(self):
         raise NotImplementedError
 
     def _get_initial_image(self):
-        _released_image_id = self.settings.OS_IMAGE
+        image = ImageSpecification.from_os_image()
         try:
-            _released_image_id = self.cloud_instance.released_image(
-                self.settings.OS_IMAGE)
+            return self.cloud_instance.released_image(image.image_id)
         except (ValueError, IndexError):
-            pass
-        return _released_image_id
+            return image.image_id
 
     def _perform_launch(self, launch_kwargs):
         pycloudlib_instance = self.cloud_instance.launch(**launch_kwargs)

tests/integration_tests/conftest.py

@@ -10,12 +10,13 @@
 
 from tests.integration_tests import integration_settings
 from tests.integration_tests.clouds import (
+    AzureCloud,
     Ec2Cloud,
     GceCloud,
-    AzureCloud,
-    OciCloud,
+    ImageSpecification,
     LxdContainerCloud,
     LxdVmCloud,
+    OciCloud,
 )
 from tests.integration_tests.instances import IntegrationInstance
 
@@ -32,6 +33,7 @@
     'lxd_container': LxdContainerCloud,
     'lxd_vm': LxdVmCloud,
 }
+os_list = ["ubuntu"]
 
 session_start_time = datetime.datetime.now().strftime('%y%m%d%H%M%S')
 
@@ -60,6 +62,12 @@ def pytest_runtest_setup(item):
     if supported_platforms and current_platform not in supported_platforms:
         pytest.skip(unsupported_message)
 
+    image = ImageSpecification.from_os_image()
+    current_os = image.os
+    supported_os_set = set(os_list).intersection(test_marks)
+    if current_os and supported_os_set and current_os not in supported_os_set:
+        pytest.skip("Cannot run on OS {}".format(current_os))
+
 
 # disable_subp_usage is defined at a higher level, but we don't
 # want it applied here

tests/integration_tests/integration_settings.py

@@ -22,8 +22,11 @@
 INSTANCE_TYPE = None
 
 # Determines the base image to use or generate new images from.
-# Can be the name of the OS if running a stock image,
-# otherwise the id of the image being used if using a custom image
+#
+# This can be the name of an Ubuntu release, or in the format
+# <image_id>[::<os>[::<release>]].  If given, os and release should describe
+# the image specified by image_id.  (Ubuntu releases are converted to this
+# format internally; in this case, to "focal::ubuntu::focal".)
 OS_IMAGE = 'focal'
 
 # Populate if you want to use a pre-launched instance instead of

tests/integration_tests/modules/test_apt_configure_sources_list.py

@@ -40,6 +40,7 @@
 
 
 @pytest.mark.ci
+@pytest.mark.ubuntu
 class TestAptConfigureSourcesList:
 
     @pytest.mark.user_data(USER_DATA)

tests/integration_tests/modules/test_package_update_upgrade_install.py

@@ -26,6 +26,7 @@
 """
 
 
+@pytest.mark.ubuntu
 @pytest.mark.user_data(USER_DATA)
 class TestPackageUpdateUpgradeInstall:
 

tests/integration_tests/modules/test_snap.py

@@ -20,6 +20,7 @@
 
 
 @pytest.mark.ci
+@pytest.mark.ubuntu
 class TestSnap:
 
     @pytest.mark.user_data(USER_DATA)

tests/integration_tests/modules/test_ssh_import_id.py

@@ -3,6 +3,10 @@
 This test specifies ssh keys to be imported by the ``ssh_import_id`` module
 and then checks that if the ssh keys were successfully imported.
 
+TODO:
+* This test assumes that SSH keys will be imported into the /home/ubuntu; this
+  will need modification to run on other OSes.
+
 (This is ported from
 ``tests/cloud_tests/testcases/modules/ssh_import_id.yaml``.)"""
 
@@ -18,6 +22,7 @@
 
 
 @pytest.mark.ci
+@pytest.mark.ubuntu
 class TestSshImportId:
 
     @pytest.mark.user_data(USER_DATA)

tests/integration_tests/modules/test_users_groups.py

@@ -2,6 +2,10 @@
 
 This test specifies a number of users and groups via user-data, and confirms
 that they have been configured correctly in the system under test.
+
+TODO:
+* This test assumes that the "ubuntu" user will be created when "default" is
+  specified; this will need modification to run on other OSes.
 """
 import re
 
@@ -41,6 +45,7 @@
 @pytest.mark.ci
 @pytest.mark.user_data(USER_DATA)
 class TestUsersGroups:
+    @pytest.mark.ubuntu
     @pytest.mark.parametrize(
         "getent_args,regex",
         [

tox.ini

@@ -179,3 +179,4 @@ markers =
     user_data: the user data to be passed to the test instance
     instance_name: the name to be used for the test instance
     sru_2020_11: test is part of the 2020/11 SRU verification
+    ubuntu: this test should run on Ubuntu