Ubuntu/xenial add quilt patches to disable SRU_BLOCKERS

.travis.yml

@@ -79,8 +79,6 @@ matrix:
               PYTEST_ADDOPTS=-v  # List all tests run by pytest
           dist: xenial
         - python: 3.6
-          env: TOXENV=pycodestyle
-        - python: 3.6
-          env: TOXENV=pyflakes
+          env: TOXENV=flake8
         - python: 3.6
           env: TOXENV=pylint

cloudinit/cmd/tests/test_query.py

@@ -261,7 +261,7 @@ def test_handle_args_list_keys_errors_when_varname_is_not_a_dict(self):
         args = self.args(
             debug=False, dump_all=False, format=None,
             instance_data=self.instance_data, list_keys=True, user_data='ud',
-            vendor_data='vd',  varname='top')
+            vendor_data='vd', varname='top')
         with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
             with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
                 with mock.patch('os.getuid') as m_getuid:

cloudinit/config/cc_apt_configure.py

@@ -297,7 +297,7 @@
                 },
                 'conf': {
                     'type': 'string',
-                    'description':  dedent("""\
+                    'description': dedent("""\
                         Specify configuration for apt, such as proxy
                         configuration. This configuration is specified as a
                         string. For multiline apt configuration, make sure

cloudinit/config/cc_chef.py

@@ -50,6 +50,7 @@
                            written to /etc/chef/client.rb)
 
     chef:
+      chef_license:
       client_key:
       encrypted_data_bag_secret:
       environment:
@@ -125,6 +126,7 @@
     'file_cache_path',
     'pid_file',
     'encrypted_data_bag_secret',
+    'chef_license',
 ])
 CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys())
 CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS)

cloudinit/config/cc_grub_dpkg.py

@@ -1,8 +1,9 @@
-# Copyright (C) 2009-2010 Canonical Ltd.
+# Copyright (C) 2009-2010, 2020 Canonical Ltd.
 # Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
 #
 # Author: Scott Moser <scott.moser@canonical.com>
 # Author: Juerg Haefliger <juerg.haefliger@hp.com>
+# Author: Matthew Ruffell <matthew.ruffell@canonical.com>
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
@@ -15,15 +16,15 @@
 should work correctly by default without any user configuration. It can be
 enabled/disabled using the ``enabled`` config key in the ``grub_dpkg`` config
 dict. The global config key ``grub-dpkg`` is an alias for ``grub_dpkg``. If no
-installation device is specified this module will look for the first existing
-device in:
+installation device is specified this module will execute grub-probe to
+determine which disk the /boot directory is associated with.
 
-    - ``/dev/sda``
-    - ``/dev/vda``
-    - ``/dev/xvda``
-    - ``/dev/sda1``
-    - ``/dev/vda1``
-    - ``/dev/xvda1``
+The value which is placed into the debconf database is in the format which the
+grub postinstall script expects. Normally, this is a /dev/disk/by-id/ value,
+but we do fallback to the plain disk name if a by-id name is not present.
+
+If this module is executed inside a container, then the debconf database is
+seeded with empty values, and install_devices_empty is set to true.
 
 **Internal name:** ``cc_grub_dpkg``
 
@@ -43,10 +44,66 @@
 import os
 
 from cloudinit import util
+from cloudinit.util import ProcessExecutionError
 
 distros = ['ubuntu', 'debian']
 
 
+def fetch_idevs(log):
+    """
+    Fetches the /dev/disk/by-id device grub is installed to.
+    Falls back to plain disk name if no by-id entry is present.
+    """
+    disk = ""
+    devices = []
+
+    try:
+        # get the root disk where the /boot directory resides.
+        disk = util.subp(['grub-probe', '-t', 'disk', '/boot'],
+                         capture=True)[0].strip()
+    except ProcessExecutionError as e:
+        # grub-common may not be installed, especially on containers
+        # FileNotFoundError is a nested exception of ProcessExecutionError
+        if isinstance(e.reason, FileNotFoundError):
+            log.debug("'grub-probe' not found in $PATH")
+        # disks from the container host are present in /proc and /sys
+        # which is where grub-probe determines where /boot is.
+        # it then checks for existence in /dev, which fails as host disks
+        # are not exposed to the container.
+        elif "failed to get canonical path" in e.stderr:
+            log.debug("grub-probe 'failed to get canonical path'")
+        else:
+            # something bad has happened, continue to log the error
+            raise
+    except Exception:
+        util.logexc(log, "grub-probe failed to execute for grub-dpkg")
+
+    if not disk or not os.path.exists(disk):
+        # If we failed to detect a disk, we can return early
+        return ''
+
+    try:
+        # check if disk exists and use udevadm to fetch symlinks
+        devices = util.subp(
+            ['udevadm', 'info', '--root', '--query=symlink', disk],
+            capture=True
+        )[0].strip().split()
+    except Exception:
+        util.logexc(
+            log, "udevadm DEVLINKS symlink query failed for disk='%s'", disk
+        )
+
+    log.debug('considering these device symlinks: %s', ','.join(devices))
+    # filter symlinks for /dev/disk/by-id entries
+    devices = [dev for dev in devices if 'disk/by-id' in dev]
+    log.debug('filtered to these disk/by-id symlinks: %s', ','.join(devices))
+    # select first device if there is one, else fall back to plain name
+    idevs = sorted(devices)[0] if devices else disk
+    log.debug('selected %s', idevs)
+
+    return idevs
+
+
 def handle(name, cfg, _cloud, log, _args):
 
     mycfg = cfg.get("grub_dpkg", cfg.get("grub-dpkg", {}))
@@ -62,22 +119,10 @@ def handle(name, cfg, _cloud, log, _args):
     idevs_empty = util.get_cfg_option_str(
         mycfg, "grub-pc/install_devices_empty", None)
 
-    if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or
-       (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))):
-        if idevs is None:
-            idevs = ""
-        if idevs_empty is None:
-            idevs_empty = "true"
-    else:
-        if idevs_empty is None:
-            idevs_empty = "false"
-        if idevs is None:
-            idevs = "/dev/sda"
-            for dev in ("/dev/sda", "/dev/vda", "/dev/xvda",
-                        "/dev/sda1", "/dev/vda1", "/dev/xvda1"):
-                if os.path.exists(dev):
-                    idevs = dev
-                    break
+    if idevs is None:
+        idevs = fetch_idevs(log)
+    if idevs_empty is None:
+        idevs_empty = "false" if idevs else "true"
 
     # now idevs and idevs_empty are set to determined values
     # or, those set by user

cloudinit/config/tests/test_grub_dpkg.py

@@ -0,0 +1,176 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import pytest
+
+from unittest import mock
+from logging import Logger
+from cloudinit.util import ProcessExecutionError
+from cloudinit.config.cc_grub_dpkg import fetch_idevs, handle
+
+
+class TestFetchIdevs:
+    """Tests cc_grub_dpkg.fetch_idevs()"""
+
+    # Note: udevadm info returns devices in a large single line string
+    @pytest.mark.parametrize(
+        "grub_output,path_exists,expected_log_call,udevadm_output"
+        ",expected_idevs",
+        [
+            # Inside a container, grub not installed
+            (
+                ProcessExecutionError(reason=FileNotFoundError()),
+                False,
+                mock.call("'grub-probe' not found in $PATH"),
+                '',
+                '',
+            ),
+            # Inside a container, grub installed
+            (
+                ProcessExecutionError(stderr="failed to get canonical path"),
+                False,
+                mock.call("grub-probe 'failed to get canonical path'"),
+                '',
+                '',
+            ),
+            # KVM Instance
+            (
+                ['/dev/vda'],
+                True,
+                None,
+                (
+                    '/dev/disk/by-path/pci-0000:00:00.0 ',
+                    '/dev/disk/by-path/virtio-pci-0000:00:00.0 '
+                ),
+                '/dev/vda',
+            ),
+            # Xen Instance
+            (
+                ['/dev/xvda'],
+                True,
+                None,
+                '',
+                '/dev/xvda',
+            ),
+            # NVMe Hardware Instance
+            (
+                ['/dev/nvme1n1'],
+                True,
+                None,
+                (
+                    '/dev/disk/by-id/nvme-Company_hash000 ',
+                    '/dev/disk/by-id/nvme-nvme.000-000-000-000-000 ',
+                    '/dev/disk/by-path/pci-0000:00:00.0-nvme-0 '
+                ),
+                '/dev/disk/by-id/nvme-Company_hash000',
+            ),
+            # SCSI Hardware Instance
+            (
+                ['/dev/sda'],
+                True,
+                None,
+                (
+                    '/dev/disk/by-id/company-user-1 ',
+                    '/dev/disk/by-id/scsi-0Company_user-1 ',
+                    '/dev/disk/by-path/pci-0000:00:00.0-scsi-0:0:0:0 '
+                ),
+                '/dev/disk/by-id/company-user-1',
+            ),
+        ],
+    )
+    @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.os.path.exists")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp")
+    def test_fetch_idevs(self, m_subp, m_exists, m_logexc, grub_output,
+                         path_exists, expected_log_call, udevadm_output,
+                         expected_idevs):
+        """Tests outputs from grub-probe and udevadm info against grub-dpkg"""
+        m_subp.side_effect = [
+            grub_output,
+            ["".join(udevadm_output)]
+        ]
+        m_exists.return_value = path_exists
+        log = mock.Mock(spec=Logger)
+        idevs = fetch_idevs(log)
+        assert expected_idevs == idevs
+        if expected_log_call is not None:
+            assert expected_log_call in log.debug.call_args_list
+
+
+class TestHandle:
+    """Tests cc_grub_dpkg.handle()"""
+
+    @pytest.mark.parametrize(
+        "cfg_idevs,cfg_idevs_empty,fetch_idevs_output,expected_log_output",
+        [
+            (
+                # No configuration
+                None,
+                None,
+                '/dev/disk/by-id/nvme-Company_hash000',
+                (
+                    "Setting grub debconf-set-selections with ",
+                    "'/dev/disk/by-id/nvme-Company_hash000','false'"
+                ),
+            ),
+            (
+                # idevs set, idevs_empty unset
+                '/dev/sda',
+                None,
+                '/dev/sda',
+                (
+                    "Setting grub debconf-set-selections with ",
+                    "'/dev/sda','false'"
+                ),
+            ),
+            (
+                # idevs unset, idevs_empty set
+                None,
+                'true',
+                '/dev/xvda',
+                (
+                    "Setting grub debconf-set-selections with ",
+                    "'/dev/xvda','true'"
+                ),
+            ),
+            (
+                # idevs set, idevs_empty set
+                '/dev/vda',
+                'false',
+                '/dev/disk/by-id/company-user-1',
+                (
+                    "Setting grub debconf-set-selections with ",
+                    "'/dev/vda','false'"
+                ),
+            ),
+            (
+                # idevs set, idevs_empty set
+                # Respect what the user defines, even if its logically wrong
+                '/dev/nvme0n1',
+                'true',
+                '',
+                (
+                    "Setting grub debconf-set-selections with ",
+                    "'/dev/nvme0n1','true'"
+                ),
+            )
+        ],
+    )
+    @mock.patch("cloudinit.config.cc_grub_dpkg.fetch_idevs")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.util.get_cfg_option_str")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp")
+    def test_handle(self, m_subp, m_logexc, m_get_cfg_str, m_fetch_idevs,
+                    cfg_idevs, cfg_idevs_empty, fetch_idevs_output,
+                    expected_log_output):
+        """Test setting of correct debconf database entries"""
+        m_get_cfg_str.side_effect = [
+            cfg_idevs,
+            cfg_idevs_empty
+        ]
+        m_fetch_idevs.return_value = fetch_idevs_output
+        log = mock.Mock(spec=Logger)
+        handle(mock.Mock(), mock.Mock(), mock.Mock(), log, mock.Mock())
+        log.debug.assert_called_with("".join(expected_log_output))
+
+
+# vi: ts=4 expandtab

cloudinit/distros/__init__.py

@@ -582,7 +582,7 @@ def lock_passwd(self, name):
         # passwd must use short '-l' due to SLES11 lacking long form '--lock'
         lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name])
         try:
-            cmd = next(l for l in lock_tools if util.which(l[0]))
+            cmd = next(tool for tool in lock_tools if util.which(tool[0]))
         except StopIteration:
             raise RuntimeError((
                 "Unable to lock user account '%s'. No tools available. "

cloudinit/net/__init__.py

@@ -824,13 +824,13 @@ def get_interfaces_by_mac_on_freebsd():
     # flatten each interface block in a single line
     def flatten(out):
         curr_block = ''
-        for l in out.split('\n'):
-            if l.startswith('\t'):
-                curr_block += l
+        for line in out.split('\n'):
+            if line.startswith('\t'):
+                curr_block += line
             else:
                 if curr_block:
                     yield curr_block
-                curr_block = l
+                curr_block = line
         yield curr_block
 
     # looks for interface and mac in a list of flatten block

cloudinit/net/tests/test_dhcp.py

@@ -211,7 +211,7 @@ def test_parse_static_routes_logs_error_truncated(self):
             "class_b": "16,172,16,10",
             "class_a": "8,10,10",
             "gateway": "0,0",
-            "netlen":  "33,0",
+            "netlen": "33,0",
         }
         for rfc3442 in bad_rfc3442.values():
             self.assertEqual([], parse_static_routes(rfc3442))

cloudinit/sources/DataSourceGCE.py

@@ -116,7 +116,7 @@ def _write_host_key_to_guest_attributes(key_type, key_value):
     resp = url_helper.readurl(url=url, data=key_value, headers=HEADERS,
                               request_method='PUT', check_status=False)
     if resp.ok():
-        LOG.debug('Wrote %s host key to guest attributes.',  key_type)
+        LOG.debug('Wrote %s host key to guest attributes.', key_type)
     else:
         LOG.debug('Unable to write %s host key to guest attributes.', key_type)