openstack: read the dynamic metadata group vendor_data2.json

cloudinit/cmd/tests/test_main.py

@@ -127,7 +127,8 @@ def set_hostname(name, cfg, cloud, log, args):
                  'syslog_fix_perms': [
                      'syslog:adm', 'root:adm', 'root:wheel', 'root:root'
                  ],
-                 'vendor_data': {'enabled': True, 'prefix': []}})
+                 'vendor_data': {'enabled': True, 'prefix': []},
+                 'vendor_data2': {'enabled': True, 'prefix': []}})
             updated_cfg.pop('system_info')
 
             self.assertEqual(updated_cfg, cfg)

cloudinit/helpers.py

@@ -230,6 +230,10 @@ def _get_instance_configs(self):
 
         cc_paths = ['cloud_config']
         if self._include_vendor:
+            # the order is important here: we want vendor2
+            #  (dynamic vendor data from OpenStack)
+            #  to override vendor (static data from OpenStack)
+            cc_paths.append('vendor2_cloud_config')
             cc_paths.append('vendor_cloud_config')
 
         for cc_p in cc_paths:
@@ -337,9 +341,12 @@ def __init__(self, path_cfgs, ds=None):
             "obj_pkl": "obj.pkl",
             "cloud_config": "cloud-config.txt",
             "vendor_cloud_config": "vendor-cloud-config.txt",
+            "vendor2_cloud_config": "vendor2-cloud-config.txt",
             "data": "data",
             "vendordata_raw": "vendor-data.txt",
+            "vendordata2_raw": "vendor-data2.txt",
             "vendordata": "vendor-data.txt.i",
+            "vendordata2": "vendor-data2.txt.i",
             "instance_id": ".instance-id",
             "manual_clean_marker": "manual-clean",
             "warnings": "warnings",

cloudinit/settings.py

@@ -56,6 +56,7 @@
         'network': {'renderers': None},
     },
     'vendor_data': {'enabled': True, 'prefix': []},
+    'vendor_data2': {'enabled': True, 'prefix': []},
 }
 
 # Valid frequencies of handlers/modules

cloudinit/sources/DataSourceOpenStack.py

@@ -167,6 +167,14 @@ def _get_data(self):
             LOG.warning("Invalid content in vendor-data: %s", e)
             self.vendordata_raw = None
 
+        vd2 = results.get('vendordata2')
+        self.vendordata2_pure = vd2
+        try:
+            self.vendordata2_raw = sources.convert_vendordata(vd2)
+        except ValueError as e:
+            LOG.warning("Invalid content in vendor-data2: %s", e)
+            self.vendordata2_raw = None
+
         return True
 
     def _crawl_metadata(self):

cloudinit/sources/__init__.py

@@ -187,7 +187,8 @@ class DataSource(metaclass=abc.ABCMeta):
     cached_attr_defaults = (
         ('ec2_metadata', UNSET), ('network_json', UNSET),
         ('metadata', {}), ('userdata', None), ('userdata_raw', None),
-        ('vendordata', None), ('vendordata_raw', None))
+        ('vendordata', None), ('vendordata_raw', None),
+        ('vendordata2', None), ('vendordata2_raw', None))
 
     _dirty_cache = False
 
@@ -203,7 +204,9 @@ def __init__(self, sys_cfg, distro, paths, ud_proc=None):
         self.metadata = {}
         self.userdata_raw = None
         self.vendordata = None
+        self.vendordata2 = None
         self.vendordata_raw = None
+        self.vendordata2_raw = None
 
         self.ds_cfg = util.get_cfg_by_path(
             self.sys_cfg, ("datasource", self.dsname), {})
@@ -392,6 +395,11 @@ def get_vendordata(self):
             self.vendordata = self.ud_proc.process(self.get_vendordata_raw())
         return self.vendordata
 
+    def get_vendordata2(self):
+        if self.vendordata2 is None:
+            self.vendordata2 = self.ud_proc.process(self.get_vendordata2_raw())
+        return self.vendordata2
+
     @property
     def fallback_interface(self):
         """Determine the network interface used during local network config."""
@@ -494,6 +502,9 @@ def get_userdata_raw(self):
     def get_vendordata_raw(self):
         return self.vendordata_raw
 
+    def get_vendordata2_raw(self):
+        return self.vendordata2_raw
+
     # the data sources' config_obj is a cloud-config formated
     # object that came to it from ways other than cloud-config
     # because cloud-config content would be handled elsewhere

cloudinit/sources/helpers/openstack.py

@@ -247,6 +247,11 @@ def datafiles(version):
                 False,
                 load_json_anytype,
             )
+            files['vendordata2'] = (
+                self._path_join("openstack", version, 'vendor_data2.json'),
+                False,
+                load_json_anytype,
+            )
             files['networkdata'] = (
                 self._path_join("openstack", version, 'network_data.json'),
                 False,

cloudinit/stages.py

@@ -360,8 +360,18 @@ def cloudify(self):
                            reporter=self.reporter)
 
     def update(self):
-        self._store_userdata()
-        self._store_vendordata()
+        self._store_rawdata(self.datasource.get_userdata_raw(),
+                            'userdata')
+        self._store_processeddata(self.datasource.get_userdata(),
+                                  'userdata')
+        self._store_rawdata(self.datasource.get_vendordata_raw(),
+                            'vendordata')
+        self._store_processeddata(self.datasource.get_vendordata(),
+                                  'vendordata')
+        self._store_rawdata(self.datasource.get_vendordata2_raw(),
+                            'vendordata2')
+        self._store_processeddata(self.datasource.get_vendordata2(),
+                                  'vendordata2')
 
     def setup_datasource(self):
         with events.ReportEventStack("setup-datasource",
@@ -381,28 +391,18 @@ def activate_datasource(self):
                                      is_new_instance=self.is_new_instance())
             self._write_to_cache()
 
-    def _store_userdata(self):
-        raw_ud = self.datasource.get_userdata_raw()
-        if raw_ud is None:
-            raw_ud = b''
-        util.write_file(self._get_ipath('userdata_raw'), raw_ud, 0o600)
-        # processed userdata is a Mime message, so write it as string.
-        processed_ud = self.datasource.get_userdata()
-        if processed_ud is None:
-            raw_ud = ''
-        util.write_file(self._get_ipath('userdata'), str(processed_ud), 0o600)
-
-    def _store_vendordata(self):
-        raw_vd = self.datasource.get_vendordata_raw()
-        if raw_vd is None:
-            raw_vd = b''
-        util.write_file(self._get_ipath('vendordata_raw'), raw_vd, 0o600)
-        # processed vendor data is a Mime message, so write it as string.
-        processed_vd = str(self.datasource.get_vendordata())
-        if processed_vd is None:
-            processed_vd = ''
-        util.write_file(self._get_ipath('vendordata'), str(processed_vd),
-                        0o600)
+    def _store_rawdata(self, data, datasource):
+        # Raw data is bytes, not a string
+        if data is None:
+            data = b''
+        util.write_file(self._get_ipath('%s_raw' % datasource), data, 0o600)
+
+    def _store_processeddata(self, processed_data, datasource):
+        # processed is a Mime message, so write as string.
+        if processed_data is None:
+            processed_data = ''
+        util.write_file(self._get_ipath(datasource),
+                        str(processed_data), 0o600)
 
     def _default_handlers(self, opts=None):
         if opts is None:
@@ -434,6 +434,11 @@ def _default_vendordata_handlers(self):
             opts={'script_path': 'vendor_scripts',
                   'cloud_config_path': 'vendor_cloud_config'})
 
+    def _default_vendordata2_handlers(self):
+        return self._default_handlers(
+            opts={'script_path': 'vendor_scripts',
+                  'cloud_config_path': 'vendor2_cloud_config'})
+
     def _do_handlers(self, data_msg, c_handlers_list, frequency,
                      excluded=None):
         """
@@ -555,7 +560,12 @@ def consume_data(self, frequency=PER_INSTANCE):
         with events.ReportEventStack("consume-vendor-data",
                                      "reading and applying vendor-data",
                                      parent=self.reporter):
-            self._consume_vendordata(frequency)
+            self._consume_vendordata("vendordata", frequency)
+
+        with events.ReportEventStack("consume-vendor-data2",
+                                     "reading and applying vendor-data2",
+                                     parent=self.reporter):
+            self._consume_vendordata("vendordata2", frequency)
 
         # Perform post-consumption adjustments so that
         # modules that run during the init stage reflect
@@ -568,46 +578,62 @@ def consume_data(self, frequency=PER_INSTANCE):
         # objects before the load of the userdata happened,
         # this is expected.
 
-    def _consume_vendordata(self, frequency=PER_INSTANCE):
+    def _consume_vendordata(self, vendor_source, frequency=PER_INSTANCE):
         """
         Consume the vendordata and run the part handlers on it
         """
+
         # User-data should have been consumed first.
         # So we merge the other available cloud-configs (everything except
         # vendor provided), and check whether or not we should consume
         # vendor data at all. That gives user or system a chance to override.
-        if not self.datasource.get_vendordata_raw():
-            LOG.debug("no vendordata from datasource")
-            return
+        if vendor_source == 'vendordata':
+            if not self.datasource.get_vendordata_raw():
+                LOG.debug("no vendordata from datasource")
+                return
+            cfg_name = 'vendor_data'
+        elif vendor_source == 'vendordata2':
+            if not self.datasource.get_vendordata2_raw():
+                LOG.debug("no vendordata2 from datasource")
+                return
+            cfg_name = 'vendor_data2'
+        else:
+            raise RuntimeError("vendor_source arg must be either 'vendordata'"
+                               " or 'vendordata2'")
 
         _cc_merger = helpers.ConfigMerger(paths=self._paths,
                                           datasource=self.datasource,
                                           additional_fns=[],
                                           base_cfg=self.cfg,
                                           include_vendor=False)
-        vdcfg = _cc_merger.cfg.get('vendor_data', {})
+        vdcfg = _cc_merger.cfg.get(cfg_name, {})
 
         if not isinstance(vdcfg, dict):
             vdcfg = {'enabled': False}
-            LOG.warning("invalid 'vendor_data' setting. resetting to: %s",
-                        vdcfg)
+            LOG.warning("invalid %s setting. resetting to: %s",
+                        cfg_name, vdcfg)
 
         enabled = vdcfg.get('enabled')
         no_handlers = vdcfg.get('disabled_handlers', None)
 
         if not util.is_true(enabled):
-            LOG.debug("vendordata consumption is disabled.")
+            LOG.debug("%s consumption is disabled.", vendor_source)
             return
 
-        LOG.debug("vendor data will be consumed. disabled_handlers=%s",
-                  no_handlers)
+        LOG.debug("%s will be consumed. disabled_handlers=%s",
+                  vendor_source, no_handlers)
 
-        # Ensure vendordata source fetched before activation (just incase)
-        vendor_data_msg = self.datasource.get_vendordata()
+        # Ensure vendordata source fetched before activation (just in case.)
 
-        # This keeps track of all the active handlers, while excluding what the
-        # users doesn't want run, i.e. boot_hook, cloud_config, shell_script
-        c_handlers_list = self._default_vendordata_handlers()
+        # c_handlers_list keeps track of all the active handlers, while
+        # excluding what the users doesn't want run, i.e. boot_hook,
+        # cloud_config, shell_script
+        if vendor_source == 'vendordata':
+            vendor_data_msg = self.datasource.get_vendordata()
+            c_handlers_list = self._default_vendordata_handlers()
+        else:
+            vendor_data_msg = self.datasource.get_vendordata2()
+            c_handlers_list = self._default_vendordata2_handlers()
 
         # Run the handlers
         self._do_handlers(vendor_data_msg, c_handlers_list, frequency,

doc/rtd/topics/datasources/openstack.rst

@@ -82,4 +82,12 @@ For more general information about how cloud-init handles vendor data,
 including how it can be disabled by users on instances, see
 :doc:`/topics/vendordata`.
 
+OpenStack can also be configured to provide 'dynamic vendordata'
+which is provided by the DynamicJSON provider and appears under a
+different metadata path, /vendor_data2.json.
+
+Cloud-init will look for a ``cloud-init`` at the vendor_data2 path; if found,
+settings are applied after (and, hence, overriding) the settings from static
+vendor data. Both sets of vendor data can be overridden by user data.
+
 .. vi: textwidth=78

tests/unittests/test_data.py

@@ -33,11 +33,12 @@
 
 class FakeDataSource(sources.DataSource):
 
-    def __init__(self, userdata=None, vendordata=None):
+    def __init__(self, userdata=None, vendordata=None, vendordata2=None):
         sources.DataSource.__init__(self, {}, None, None)
         self.metadata = {'instance-id': INSTANCE_ID}
         self.userdata_raw = userdata
         self.vendordata_raw = vendordata
+        self.vendordata2_raw = vendordata2
 
 
 def count_messages(root):
@@ -105,26 +106,38 @@ def test_simple_jsonp(self):
         self.assertEqual('qux', cc['baz'])
         self.assertEqual('qux2', cc['bar'])
 
-    def test_simple_jsonp_vendor_and_user(self):
+    def test_simple_jsonp_vendor_and_vendor2_and_user(self):
         # test that user-data wins over vendor
         user_blob = '''
 #cloud-config-jsonp
 [
      { "op": "add", "path": "/baz", "value": "qux" },
-     { "op": "add", "path": "/bar", "value": "qux2" }
+     { "op": "add", "path": "/bar", "value": "qux2" },
+     { "op": "add", "path": "/foobar", "value": "qux3" }
 ]
 '''
         vendor_blob = '''
 #cloud-config-jsonp
 [
      { "op": "add", "path": "/baz", "value": "quxA" },
      { "op": "add", "path": "/bar", "value": "quxB" },
-     { "op": "add", "path": "/foo", "value": "quxC" }
+     { "op": "add", "path": "/foo", "value": "quxC" },
+     { "op": "add", "path": "/corge", "value": "quxEE" }
+]
+'''
+        vendor2_blob = '''
+#cloud-config-jsonp
+[
+     { "op": "add", "path": "/corge", "value": "quxD" },
+     { "op": "add", "path": "/grault", "value": "quxFF" },
+     { "op": "add", "path": "/foobar", "value": "quxGG" }
 ]
 '''
         self.reRoot()
         initer = stages.Init()
-        initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob)
+        initer.datasource = FakeDataSource(user_blob,
+                                           vendordata=vendor_blob,
+                                           vendordata2=vendor2_blob)
         initer.read_cfg()
         initer.initialize()
         initer.fetch()
@@ -138,9 +151,15 @@ def test_simple_jsonp_vendor_and_user(self):
         (_which_ran, _failures) = mods.run_section('cloud_init_modules')
         cfg = mods.cfg
         self.assertIn('vendor_data', cfg)
+        self.assertIn('vendor_data2', cfg)
+        # Confirm that vendordata2 overrides vendordata, and that
+        #  userdata overrides both
         self.assertEqual('qux', cfg['baz'])
         self.assertEqual('qux2', cfg['bar'])
+        self.assertEqual('qux3', cfg['foobar'])
         self.assertEqual('quxC', cfg['foo'])
+        self.assertEqual('quxD', cfg['corge'])
+        self.assertEqual('quxFF', cfg['grault'])
 
     def test_simple_jsonp_no_vendor_consumed(self):
         # make sure that vendor data is not consumed
@@ -293,6 +312,10 @@ def test_vendordata_script(self):
         vendor_blob = '''
 #!/bin/bash
 echo "test"
+'''
+        vendor2_blob = '''
+#!/bin/bash
+echo "dynamic test"
 '''
 
         user_blob = '''
@@ -303,7 +326,9 @@ def test_vendordata_script(self):
 '''
         new_root = self.reRoot()
         initer = stages.Init()
-        initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob)
+        initer.datasource = FakeDataSource(user_blob,
+                                           vendordata=vendor_blob,
+                                           vendordata2=vendor2_blob)
         initer.read_cfg()
         initer.initialize()
         initer.fetch()