Add "install hotplug" module (SC-476)

cloudinit/cmd/devel/hotplug_hook.py

@@ -8,6 +8,7 @@
 
 from cloudinit import log
 from cloudinit import reporting
+from cloudinit import stages
 from cloudinit.event import EventScope, EventType
 from cloudinit.net import activators, read_sys_net_safe
 from cloudinit.net.network_state import parse_net_config_data
@@ -164,7 +165,9 @@ def is_enabled(hotplug_init, subsystem):
                 subsystem)
         ) from e
 
-    return hotplug_init.update_event_enabled(
+    return stages.update_event_enabled(
+        datasource=hotplug_init.datasource,
+        cfg=hotplug_init.cfg,
         event_source_type=EventType.HOTPLUG,
         scope=scope
     )

cloudinit/config/cc_install_hotplug.py

@@ -0,0 +1,136 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+"""Install hotplug udev rules if supported and enabled"""
+import os
+from textwrap import dedent
+
+from cloudinit import util
+from cloudinit import subp
+from cloudinit import stages
+from cloudinit.config.schema import get_schema_doc, validate_cloudconfig_schema
+from cloudinit.distros import ALL_DISTROS
+from cloudinit.event import EventType, EventScope
+from cloudinit.settings import PER_INSTANCE
+
+
+frequency = PER_INSTANCE
+distros = [ALL_DISTROS]
+
+schema = {
+    "id": "cc_install_hotplug",
+    "name": "Install Hotplug",
+    "title": "Install hotplug if supported and enabled",
+    "description": dedent("""\
+        This module will install the udev rules to enable hotplug if
+        supported by the datasource and enabled in the userdata. The udev
+        rules will be installed as
+        ``/etc/udev/rules.d/10-cloud-init-hook-hotplug.rules``.
+
+        When hotplug is enabled, newly added network devices will be added
+        to the system by cloud-init. After udev detects the event,
+        cloud-init will referesh the instance metadata from the datasource,
+        detect the device in the updated metadata, then apply the updated
+        network configuration.
+
+        Currently supported datasources: Openstack, EC2
+    """),
+    "distros": distros,
+    "examples": [
+        dedent("""\
+            # Enable hotplug of network devices
+            updates:
+              network:
+                when: ["hotplug"]
+        """),
+        dedent("""\
+            # Enable network hotplug alongside boot event
+            updates:
+              network:
+                when: ["boot", "hotplug"]
+        """),
+    ],
+    "frequency": frequency,
+    "type": "object",
+    "properties": {
+        "updates": {
+            "type": "object",
+            "additionalProperties": False,
+            "properties": {
+                "network": {
+                    "type": "object",
+                    "required": ["when"],
+                    "additionalProperties": False,
+                    "properties": {
+                        "when": {
+                            "type": "array",
+                            "additionalProperties": False,
+                            "items": {
+                                "type": "string",
+                                "additionalProperties": False,
+                                "enum": [
+                                    "boot-new-instance",
+                                    "boot-legacy",
+                                    "boot",
+                                    "hotplug",
+                                ]
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+__doc__ = get_schema_doc(schema)
+
+
+HOTPLUG_UDEV_PATH = "/etc/udev/rules.d/10-cloud-init-hook-hotplug.rules"
+HOTPLUG_UDEV_RULES = """\
+# Installed by cloud-init due to network hotplug userdata
+ACTION!="add|remove", GOTO="cloudinit_end"
+LABEL="cloudinit_hook"
+SUBSYSTEM=="net", RUN+="/usr/lib/cloud-init/hook-hotplug"
+LABEL="cloudinit_end"
+"""
+
+
+def handle(_name, cfg, cloud, log, _args):
+    validate_cloudconfig_schema(cfg, schema)
+    network_hotplug_enabled = (
+        'updates' in cfg and
+        'network' in cfg['updates'] and
+        'when' in cfg['updates']['network'] and
+        'hotplug' in cfg['updates']['network']['when']
+    )
+    hotplug_supported = EventType.HOTPLUG in (
+        cloud.datasource.get_supported_events(
+            [EventType.HOTPLUG]).get(EventScope.NETWORK, set())
+    )
+    hotplug_enabled = stages.update_event_enabled(
+        datasource=cloud.datasource,
+        cfg=cfg,
+        event_source_type=EventType.HOTPLUG,
+        scope=EventScope.NETWORK,
+    )
+    if not (hotplug_supported and hotplug_enabled):
+        if os.path.exists(HOTPLUG_UDEV_PATH):
+            log.debug("Uninstalling hotplug, not enabled")
+            util.del_file(HOTPLUG_UDEV_PATH)
+            subp.subp(["udevadm", "control", "--reload-rules"])
+        elif network_hotplug_enabled:
+            log.warning(
+                "Hotplug is unsupported by current datasource. "
+                "Udev rules will NOT be installed."
+            )
+        else:
+            log.debug("Skipping hotplug install, not enabled")
+        return
+    if not subp.which("udevadm"):
+        log.debug("Skipping hotplug install, udevadm not found")
+        return
+
+    util.write_file(
+        filename=HOTPLUG_UDEV_PATH,
+        content=HOTPLUG_UDEV_RULES,
+    )
+    subp.subp(["udevadm", "control", "--reload-rules"])

cloudinit/stages.py

@@ -49,6 +49,54 @@
 NO_PREVIOUS_INSTANCE_ID = "NO_PREVIOUS_INSTANCE_ID"
 
 
+def update_event_enabled(
+    datasource: sources.DataSource,
+    cfg: dict,
+    event_source_type: EventType,
+    scope: EventScope = None
+) -> bool:
+    """Determine if a particular EventType is enabled.
+
+    For the `event_source_type` passed in, check whether this EventType
+    is enabled in the `updates` section of the userdata. If `updates`
+    is not enabled in userdata, check if defined as one of the
+    `default_events` on the datasource. `scope` may be used to
+    narrow the check to a particular `EventScope`.
+
+    Note that on first boot, userdata may NOT be available yet. In this
+    case, we only have the data source's `default_update_events`,
+    so an event that should be enabled in userdata may be denied.
+    """
+    default_events = datasource.default_update_events  # type: Dict[EventScope, Set[EventType]]    # noqa: E501
+    user_events = userdata_to_events(cfg.get('updates', {}))  # type: Dict[EventScope, Set[EventType]]  # noqa: E501
+    # A value in the first will override a value in the second
+    allowed = util.mergemanydict([
+        copy.deepcopy(user_events),
+        copy.deepcopy(default_events),
+    ])
+    LOG.debug('Allowed events: %s', allowed)
+
+    if not scope:
+        scopes = allowed.keys()
+    else:
+        scopes = [scope]
+    scope_values = [s.value for s in scopes]
+
+    for evt_scope in scopes:
+        if event_source_type in allowed.get(evt_scope, []):
+            LOG.debug(
+                'Event Allowed: scope=%s EventType=%s',
+                evt_scope.value, event_source_type
+            )
+            return True
+
+    LOG.debug(
+        'Event Denied: scopes=%s EventType=%s',
+        scope_values, event_source_type
+    )
+    return False
+
+
 class Init(object):
     def __init__(self, ds_deps=None, reporter=None):
         if ds_deps is not None:
@@ -715,46 +763,6 @@ def _find_networking_config(self):
         return (self.distro.generate_fallback_config(),
                 NetworkConfigSource.fallback)
 
-    def update_event_enabled(
-        self, event_source_type: EventType, scope: EventScope = None
-    ) -> bool:
-        """Determine if a particular EventType is enabled.
-
-        For the `event_source_type` passed in, check whether this EventType
-        is enabled in the `updates` section of the userdata. If `updates`
-        is not enabled in userdata, check if defined as one of the
-        `default_events` on the datasource. `scope` may be used to
-        narrow the check to a particular `EventScope`.
-
-        Note that on first boot, userdata may NOT be available yet. In this
-        case, we only have the data source's `default_update_events`,
-        so an event that should be enabled in userdata may be denied.
-        """
-        default_events = self.datasource.default_update_events  # type: Dict[EventScope, Set[EventType]]    # noqa: E501
-        user_events = userdata_to_events(self.cfg.get('updates', {}))  # type: Dict[EventScope, Set[EventType]]  # noqa: E501
-        # A value in the first will override a value in the second
-        allowed = util.mergemanydict([
-            copy.deepcopy(user_events),
-            copy.deepcopy(default_events),
-        ])
-        LOG.debug('Allowed events: %s', allowed)
-
-        if not scope:
-            scopes = allowed.keys()
-        else:
-            scopes = [scope]
-        scope_values = [s.value for s in scopes]
-
-        for evt_scope in scopes:
-            if event_source_type in allowed.get(evt_scope, []):
-                LOG.debug('Event Allowed: scope=%s EventType=%s',
-                          evt_scope.value, event_source_type)
-                return True
-
-        LOG.debug('Event Denied: scopes=%s EventType=%s',
-                  scope_values, event_source_type)
-        return False
-
     def _apply_netcfg_names(self, netcfg):
         try:
             LOG.debug("applying net config names for %s", netcfg)
@@ -784,8 +792,11 @@ def apply_network_config(self, bring_up):
             return
 
         def event_enabled_and_metadata_updated(event_type):
-            return self.update_event_enabled(
-                event_type, scope=EventScope.NETWORK
+            return update_event_enabled(
+                datasource=self.datasource,
+                cfg=self.cfg,
+                event_source_type=event_type,
+                scope=EventScope.NETWORK
             ) and self.datasource.update_metadata_if_supported([event_type])
 
         def should_run_on_boot_event():

config/cloud.cfg.tmpl

@@ -165,6 +165,7 @@ cloud_final_modules:
  - scripts-user
  - ssh-authkey-fingerprints
  - keys-to-console
+ - install-hotplug
  - phone-home
  - final-message
  - power-state-change

doc/rtd/topics/modules.rst

@@ -22,6 +22,7 @@ Modules
 .. automodule:: cloudinit.config.cc_foo
 .. automodule:: cloudinit.config.cc_growpart
 .. automodule:: cloudinit.config.cc_grub_dpkg
+.. automodule:: cloudinit.config.cc_install_hotplug
 .. automodule:: cloudinit.config.cc_keys_to_console
 .. automodule:: cloudinit.config.cc_landscape
 .. automodule:: cloudinit.config.cc_locale

requirements.txt

@@ -31,7 +31,7 @@ requests
 jsonpatch
 
 # For validating cloud-config sections per schema definitions
-jsonschema==3.2.0
+jsonschema
 
 # Used by DataSourceVMware to inspect the host's network configuration during
 # the "setup()" function.

test-requirements.txt

@@ -5,4 +5,4 @@ pytest-cov
 
 # Only really needed on older versions of python
 setuptools
-jsonschema==3.2.0
+jsonschema

tests/integration_tests/modules/test_hotplug.py

@@ -49,10 +49,13 @@ def test_hotplug_add_remove(client: IntegrationInstance):
     ips_before = _get_ip_addr(client)
     log = client.read_from_file('/var/log/cloud-init.log')
     assert 'Exiting hotplug handler' not in log
+    assert client.execute(
+        'test -f /etc/udev/rules.d/10-cloud-init-hook-hotplug.rules'
+    ).ok
 
     # Add new NIC
     added_ip = client.instance.add_network_interface()
-    _wait_till_hotplug_complete(client, expected_runs=2)
+    _wait_till_hotplug_complete(client, expected_runs=1)
     ips_after_add = _get_ip_addr(client)
     new_addition = [ip for ip in ips_after_add if ip.ip4 == added_ip][0]
 
@@ -67,7 +70,7 @@ def test_hotplug_add_remove(client: IntegrationInstance):
 
     # Remove new NIC
     client.instance.remove_network_interface(added_ip)
-    _wait_till_hotplug_complete(client, expected_runs=4)
+    _wait_till_hotplug_complete(client, expected_runs=2)
     ips_after_remove = _get_ip_addr(client)
     assert len(ips_after_remove) == len(ips_before)
     assert added_ip not in [ip.ip4 for ip in ips_after_remove]
@@ -86,12 +89,14 @@ def test_no_hotplug_in_userdata(client: IntegrationInstance):
     ips_before = _get_ip_addr(client)
     log = client.read_from_file('/var/log/cloud-init.log')
     assert 'Exiting hotplug handler' not in log
+    assert client.execute(
+        'test -f /etc/udev/rules.d/10-cloud-init-hook-hotplug.rules'
+    ).failed
 
     # Add new NIC
     client.instance.add_network_interface()
-    _wait_till_hotplug_complete(client)
     log = client.read_from_file('/var/log/cloud-init.log')
-    assert "Event Denied: scopes=['network'] EventType=hotplug" in log
+    assert 'hotplug-hook' not in log
 
     ips_after_add = _get_ip_addr(client)
     if len(ips_after_add) == len(ips_before) + 1:

tests/unittests/cmd/devel/test_hotplug_hook.py

@@ -30,6 +30,11 @@ def mocks():
         return_value=FAKE_MAC
     )
 
+    update_event_enabled = mock.patch(
+        'cloudinit.stages.update_event_enabled',
+        return_value=True,
+    )
+
     m_network_state = mock.MagicMock(spec=NetworkState)
     parse_net = mock.patch(
         'cloudinit.cmd.devel.hotplug_hook.parse_net_config_data',
@@ -45,6 +50,7 @@ def mocks():
     sleep = mock.patch('time.sleep')
 
     read_sys_net.start()
+    update_event_enabled.start()
     parse_net.start()
     select_activator.start()
     m_sleep = sleep.start()
@@ -57,6 +63,7 @@ def mocks():
     )
 
     read_sys_net.stop()
+    update_event_enabled.stop()
     parse_net.stop()
     select_activator.stop()
     sleep.stop()
@@ -122,13 +129,16 @@ def test_successful_remove(self, mocks):
 
     def test_update_event_disabled(self, mocks, caplog):
         init = mocks.m_init
-        init.update_event_enabled.return_value = False
-        handle_hotplug(
-            hotplug_init=init,
-            devpath='/dev/fake',
-            udevaction='remove',
-            subsystem='net'
-        )
+        with mock.patch(
+            'cloudinit.stages.update_event_enabled',
+            return_value=False
+        ):
+            handle_hotplug(
+                hotplug_init=init,
+                devpath='/dev/fake',
+                udevaction='remove',
+                subsystem='net'
+            )
         assert 'hotplug not enabled for event of type' in caplog.text
         init.datasource.update_metadata_if_supported.assert_not_called()
         mocks.m_activator.bring_up_interface.assert_not_called()