testing: integration_test updates (SC-362)

tests/integration_tests/bugs/test_gh632.py

@@ -3,16 +3,15 @@
 Verify that if cloud-init is using DataSourceRbxCloud, there is
 no traceback if the metadata disk cannot be found.
 """
-
 import pytest
 
 from tests.integration_tests.instances import IntegrationInstance
+from tests.integration_tests.util import verify_clean_log
 
 
 # With some datasource hacking, we can run this on a NoCloud instance
 @pytest.mark.lxd_container
 @pytest.mark.lxd_vm
-@pytest.mark.sru_2020_11
 def test_datasource_rbx_no_stacktrace(client: IntegrationInstance):
     client.write_to_file(
         '/etc/cloud/cloud.cfg.d/90_dpkg.cfg',
@@ -26,8 +25,7 @@ def test_datasource_rbx_no_stacktrace(client: IntegrationInstance):
     client.restart()
 
     log = client.read_from_file('/var/log/cloud-init.log')
-    assert 'WARNING' not in log
-    assert 'Traceback' not in log
+    verify_clean_log(log)
     assert 'Failed to load metadata and userdata' not in log
     assert ("Getting data from <class 'cloudinit.sources.DataSourceRbxCloud."
             "DataSourceRbxCloud'> failed") not in log

tests/integration_tests/bugs/test_gh868.py

@@ -1,6 +1,8 @@
 """Ensure no Traceback when 'chef_license' is set"""
 import pytest
+
 from tests.integration_tests.instances import IntegrationInstance
+from tests.integration_tests.util import verify_clean_log
 
 
 USERDATA = """\
@@ -17,4 +19,4 @@
 @pytest.mark.user_data(USERDATA)
 def test_chef_license(client: IntegrationInstance):
     log = client.read_from_file('/var/log/cloud-init.log')
-    assert 'Traceback' not in log
+    verify_clean_log(log)

tests/integration_tests/bugs/test_lp1813396.py

@@ -29,6 +29,5 @@ def test_gpg_no_tty(client: IntegrationInstance):
         "'--keyserver=keyserver.ubuntu.com', '--recv-keys', 'E4D304DF'] "
         "with allowed return codes [0] (shell=False, capture=True)",
         "Imported key 'E4D304DF' from keyserver 'keyserver.ubuntu.com'",
-        "finish: modules-config/config-apt-configure: SUCCESS",
     ]
     verify_ordered_items_in_text(to_verify, log)

tests/integration_tests/bugs/test_lp1835584.py

@@ -59,6 +59,9 @@ def _check_iid_insensitive_across_kernel_upgrade(
     result = instance.execute("apt-get install linux-azure --assume-yes")
     if not result.ok:
         pytest.fail("Unable to install linux-azure kernel: {}".format(result))
+    # Remove ubuntu-azure-fips metapkg which mandates FIPS-flavour kernel
+    result = instance.execute("ua disable fips --assume-yes")
+    assert result.ok, "Unable to disable fips: {}".format(result)
     instance.restart()
     new_kernel = instance.execute("uname -r").strip()
     assert orig_kernel != new_kernel

tests/integration_tests/bugs/test_lp1886531.py

@@ -11,6 +11,8 @@
 """
 import pytest
 
+from tests.integration_tests.util import verify_clean_log
+
 
 USER_DATA = """\
 #cloud-config
@@ -24,4 +26,4 @@ class TestLp1886531:
     @pytest.mark.user_data(USER_DATA)
     def test_lp1886531(self, client):
         log_content = client.read_from_file("/var/log/cloud-init.log")
-        assert "WARNING" not in log_content
+        verify_clean_log(log_content)

tests/integration_tests/bugs/test_lp1898997.py

@@ -10,7 +10,9 @@
 default gateway.
 """
 import pytest
+
 from tests.integration_tests import random_mac_address
+from tests.integration_tests.util import verify_clean_log
 
 MAC_ADDRESS = random_mac_address()
 
@@ -59,7 +61,7 @@ def test_ovs_member_interfaces_not_excluded(self, client):
         cloudinit_output = client.read_from_file("/var/log/cloud-init.log")
 
         # Confirm that the network configuration was applied successfully
-        assert "WARN" not in cloudinit_output
+        verify_clean_log(cloudinit_output)
         # Confirm that the applied network config created the OVS bridge
         assert "ovs-br" in client.execute("ip addr")
 

tests/integration_tests/instances.py

@@ -9,6 +9,7 @@
 from pycloudlib.result import Result
 
 from tests.integration_tests import integration_settings
+from tests.integration_tests.util import retry
 
 try:
     from typing import TYPE_CHECKING
@@ -142,26 +143,31 @@ def install_new_cloud_init(
             snapshot_id = self.snapshot()
             self.cloud.snapshot_id = snapshot_id
 
+    # assert with retry because we can compete with apt already running in the
+    # background and get: E: Could not get lock /var/lib/apt/lists/lock - open
+    # (11: Resource temporarily unavailable)
+
+    @retry(tries=30, delay=1)
     def install_proposed_image(self):
         log.info('Installing proposed image')
-        remote_script = (
+        assert self.execute(
             'echo deb "http://archive.ubuntu.com/ubuntu '
-            '$(lsb_release -sc)-proposed main" | '
-            'tee /etc/apt/sources.list.d/proposed.list\n'
-            'apt-get update -q\n'
-            'apt-get install -qy cloud-init'
-        )
-        self.execute(remote_script)
+            '$(lsb_release -sc)-proposed main" >> '
+            '/etc/apt/sources.list.d/proposed.list'
+        ).ok
+        assert self.execute('apt-get update -q').ok
+        assert self.execute('apt-get install -qy cloud-init').ok
 
+    @retry(tries=30, delay=1)
     def install_ppa(self):
         log.info('Installing PPA')
-        remote_script = (
-            'add-apt-repository {repo} -y && '
-            'apt-get update -q && '
-            'apt-get install -qy cloud-init'
-        ).format(repo=self.settings.CLOUD_INIT_SOURCE)
-        self.execute(remote_script)
+        assert self.execute('add-apt-repository {} -y'.format(
+            self.settings.CLOUD_INIT_SOURCE)
+        ).ok
+        assert self.execute('apt-get update -q').ok
+        assert self.execute('apt-get install -qy cloud-init').ok
 
+    @retry(tries=30, delay=1)
     def install_deb(self):
         log.info('Installing deb package')
         deb_path = integration_settings.CLOUD_INIT_SOURCE
@@ -170,13 +176,13 @@ def install_deb(self):
         self.push_file(
             local_path=integration_settings.CLOUD_INIT_SOURCE,
             remote_path=remote_path)
-        remote_script = 'dpkg -i {path}'.format(path=remote_path)
-        self.execute(remote_script)
+        assert self.execute('dpkg -i {path}'.format(path=remote_path)).ok
 
+    @retry(tries=30, delay=1)
     def upgrade_cloud_init(self):
         log.info('Upgrading cloud-init to latest version in archive')
-        self.execute("apt-get update -q")
-        self.execute("apt-get install -qy cloud-init")
+        assert self.execute("apt-get update -q").ok
+        assert self.execute("apt-get install -qy cloud-init").ok
 
     def __enter__(self):
         return self

tests/integration_tests/modules/test_combined.py

@@ -8,14 +8,15 @@
 import json
 import pytest
 import re
-from datetime import date
 
 from tests.integration_tests.clouds import ImageSpecification
 from tests.integration_tests.instances import IntegrationInstance
-from tests.integration_tests.util import verify_ordered_items_in_text
+from tests.integration_tests.util import (
+    verify_clean_log,
+    verify_ordered_items_in_text,
+)
 
 USER_DATA = """\
-## template: jinja
 #cloud-config
 apt:
   primary:
@@ -33,8 +34,7 @@
 ntp:
   servers: ['ntp.ubuntu.com']
 runcmd:
-  - echo {{ds.meta_data.local_hostname}} > /var/tmp/runcmd_output
-  - echo {{merged_cfg.def_log_file}} >> /var/tmp/runcmd_output
+  - echo 'hello world' > /var/tmp/runcmd_output
 """
 
 
@@ -44,11 +44,17 @@ class TestCombined:
     def test_final_message(self, class_client: IntegrationInstance):
         """Test that final_message module works as expected.
 
-        Also tests LP 1511485: final_message is silent
+        Also tests LP 1511485: final_message is silent.
+
+        It's possible that if this test is run within a minute or so of
+        midnight that we'll see a failure because the day in the logs
+        is different from the day specified in the test definition.
         """
         client = class_client
         log = client.read_from_file('/var/log/cloud-init.log')
-        today = date.today().strftime('%a, %d %b %Y')
+        # Get date on host rather than locally as our host could be in a
+        # wildly different timezone (or more likely recording UTC)
+        today = client.execute('date "+%a, %d %b %Y"')
         expected = (
             'This is my final message!\n'
             r'\d+\.\d+.*\n'
@@ -96,21 +102,10 @@ def test_configured_locale(self, class_client: IntegrationInstance):
             'en_US.UTF-8'
         ], locale_gen)
 
-    def test_runcmd_with_variable_substitution(
-        self, class_client: IntegrationInstance
-    ):
-        """Test runcmd, while including jinja substitution.
-
-        Ensure we can also substitue variables from instance-data-sensitive
-        LP: #1931392
-        """
+    def test_runcmd(self, class_client: IntegrationInstance):
+        """Test runcmd works as expected"""
         client = class_client
-        expected = [
-            client.execute('hostname').stdout.strip(),
-            '/var/log/cloud-init.log',
-        ]
-        output = client.read_from_file('/var/tmp/runcmd_output')
-        verify_ordered_items_in_text(expected, output)
+        assert 'hello world' == client.read_from_file('/var/tmp/runcmd_output')
 
     def test_no_problems(self, class_client: IntegrationInstance):
         """Test no errors, warnings, or tracebacks"""
@@ -124,8 +119,7 @@ def test_no_problems(self, class_client: IntegrationInstance):
         assert result_json['errors'] == []
 
         log = client.read_from_file('/var/log/cloud-init.log')
-        assert 'WARN' not in log
-        assert 'Traceback' not in log
+        verify_clean_log(log)
 
     def _check_common_metadata(self, data):
         assert data['base64_encoded_keys'] == []
@@ -171,8 +165,10 @@ def test_instance_json_lxd_vm(self, class_client: IntegrationInstance):
         v1_data = data['v1']
         assert v1_data['cloud_name'] == 'unknown'
         assert v1_data['platform'] == 'lxd'
-        assert v1_data['subplatform'] == (
-            'seed-dir (/var/lib/cloud/seed/nocloud-net)')
+        assert any([
+            '/var/lib/cloud/seed/nocloud-net' in v1_data['subplatform'],
+            '/dev/sr0' in v1_data['subplatform']
+        ])
         assert v1_data['availability_zone'] is None
         assert v1_data['instance_id'] == client.instance.name
         assert v1_data['local_hostname'] == client.instance.name

tests/integration_tests/modules/test_disk_setup.py

@@ -6,6 +6,7 @@
 
 from cloudinit.subp import subp
 from tests.integration_tests.instances import IntegrationInstance
+from tests.integration_tests.util import verify_clean_log
 
 DISK_PATH = '/tmp/test_disk_setup_{}'.format(uuid4())
 
@@ -59,8 +60,7 @@ def test_device_alias(self, create_disk, client: IntegrationInstance):
         ) in log
         assert 'changed my_alias.1 => /dev/sdb1' in log
         assert 'changed my_alias.2 => /dev/sdb2' in log
-        assert 'WARN' not in log
-        assert 'Traceback' not in log
+        verify_clean_log(log)
 
         lsblk = json.loads(client.execute('lsblk --json'))
         sdb = [x for x in lsblk['blockdevices'] if x['name'] == 'sdb'][0]
@@ -120,8 +120,7 @@ class TestPartProbeAvailability:
     """
 
     def _verify_first_disk_setup(self, client, log):
-        assert 'Traceback' not in log
-        assert 'WARN' not in log
+        verify_clean_log(log)
         lsblk = json.loads(client.execute('lsblk --json'))
         sdb = [x for x in lsblk['blockdevices'] if x['name'] == 'sdb'][0]
         assert len(sdb['children']) == 2
@@ -167,8 +166,7 @@ def test_disk_setup_when_mounted(
         client.restart()
 
         # Assert new setup works as expected
-        assert 'Traceback' not in log
-        assert 'WARN' not in log
+        verify_clean_log(log)
 
         lsblk = json.loads(client.execute('lsblk --json'))
         sdb = [x for x in lsblk['blockdevices'] if x['name'] == 'sdb'][0]

tests/integration_tests/modules/test_hotplug.py

@@ -40,6 +40,10 @@ def _get_ip_addr(client):
 
 
 @pytest.mark.openstack
+# On Bionic, we traceback when attempting to detect the hotplugged
+# device in the updated metadata. This is because Bionic is specifically
+# configured not to provide network metadata.
+@pytest.mark.not_bionic
 @pytest.mark.user_data(USER_DATA)
 def test_hotplug_add_remove(client: IntegrationInstance):
     ips_before = _get_ip_addr(client)

tests/integration_tests/modules/test_jinja_templating.py

@@ -0,0 +1,30 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+import pytest
+
+from tests.integration_tests.instances import IntegrationInstance
+from tests.integration_tests.util import verify_ordered_items_in_text
+
+
+USER_DATA = """\
+## template: jinja
+#cloud-config
+runcmd:
+  - echo {{v1.local_hostname}} > /var/tmp/runcmd_output
+  - echo {{merged_cfg._doc}} >> /var/tmp/runcmd_output
+"""
+
+
+@pytest.mark.user_data(USER_DATA)
+def test_runcmd_with_variable_substitution(client: IntegrationInstance):
+    """Test jinja substitution.
+
+    Ensure we can also substitute variables from instance-data-sensitive
+    LP: #1931392
+    """
+    expected = [
+        client.execute('hostname').stdout.strip(),
+        ('Merged cloud-init system config from /etc/cloud/cloud.cfg and '
+            '/etc/cloud/cloud.cfg.d/')
+    ]
+    output = client.read_from_file('/var/tmp/runcmd_output')
+    verify_ordered_items_in_text(expected, output)

tests/integration_tests/modules/test_lxd_bridge.py

@@ -6,6 +6,8 @@
 import pytest
 import yaml
 
+from tests.integration_tests.util import verify_clean_log
+
 
 USER_DATA = """\
 #cloud-config
@@ -38,7 +40,7 @@ def test_binaries_installed(self, class_client, binary_name):
     def test_bridge(self, class_client):
         """Check that the given bridge is configured"""
         cloud_init_log = class_client.read_from_file("/var/log/cloud-init.log")
-        assert "WARN" not in cloud_init_log
+        verify_clean_log(cloud_init_log)
 
         # The bridge should exist
         assert class_client.execute("ip addr show lxdbr0")

tests/integration_tests/modules/test_ntp_servers.py

@@ -78,6 +78,8 @@ def test_ntpq_servers(self, class_client: IntegrationInstance):
 ntp:
   enabled: true
   ntp_client: chrony
+  servers:
+      - 172.16.15.14
 """
 
 
@@ -89,14 +91,16 @@ def test_chrony(client: IntegrationInstance):
     else:
         chrony_conf = '/etc/chrony/chrony.conf'
     contents = client.read_from_file(chrony_conf)
-    assert '.pool.ntp.org' in contents
+    assert 'server 172.16.15.14' in contents
 
 
 TIMESYNCD_DATA = """\
 #cloud-config
 ntp:
   enabled: true
   ntp_client: systemd-timesyncd
+  servers:
+      - 172.16.15.14
 """
 
 
@@ -106,7 +110,7 @@ def test_timesyncd(client: IntegrationInstance):
     contents = client.read_from_file(
         '/etc/systemd/timesyncd.conf.d/cloud-init.conf'
     )
-    assert '.pool.ntp.org' in contents
+    assert 'NTP=172.16.15.14' in contents
 
 
 EMPTY_NTP = """\