From 4725bd609a77f6bfdff19c9367dc1bfd2a59bce0 Mon Sep 17 00:00:00 2001 From: xiaofengw-vmware Date: Mon, 16 Nov 2020 02:32:51 +0000 Subject: [PATCH 1/8] VMware: support raw cloud-init data in vm customization --- cloudinit/sources/DataSourceOVF.py | 164 ++++++++- .../sources/helpers/vmware/imc/config.py | 12 + tests/unittests/test_datasource/test_ovf.py | 336 +++++++++++++++++- tests/unittests/test_vmware_config_file.py | 16 + 4 files changed, 512 insertions(+), 16 deletions(-) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 741c140ad86..aed5e082bf9 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -9,6 +9,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import base64 +import json import os import re import time @@ -16,6 +17,7 @@ from cloudinit import dmi from cloudinit import log as logging +from cloudinit import safeyaml from cloudinit import sources from cloudinit import subp from cloudinit import util @@ -76,7 +78,7 @@ def _get_data(self): ud = "" vd = "" vmwareImcConfigFilePath = None - nicspath = None + nicsPath = None defaults = { "instance-id": "iid-dsovf", @@ -124,31 +126,75 @@ def _get_data(self): vmwareImcConfigFilePath = util.log_time( logfunc=LOG.debug, msg="waiting for configuration file", - func=wait_for_imc_cfg_file, + func=wait_for_imc_file, args=("cust.cfg", max_wait)) + + if vmwareImcConfigFilePath: + imcdirpath = os.path.dirname(vmwareImcConfigFilePath) + cf = ConfigFile(vmwareImcConfigFilePath) + self._vmware_cust_conf = Config(cf) + else: LOG.debug("Did not find the customization plugin.") if vmwareImcConfigFilePath: LOG.debug("Found VMware Customization Config File at %s", vmwareImcConfigFilePath) - nicspath = wait_for_imc_cfg_file( - filename="nics.txt", maxwait=10, naplen=5) + try: + (metaPath, userPath, nicsPath) = collect_imc_files( + self._vmware_cust_conf) + except Exception as e: + _raise_error_status( + "File(s) missing in directory", + e, + GuestCustEvent.GUESTCUST_EVENT_CUSTOMIZE_FAILED, + vmwareImcConfigFilePath, + self._vmware_cust_conf) else: LOG.debug("Did not find VMware Customization Config File") else: LOG.debug("Customization for VMware platform is disabled.") - if vmwareImcConfigFilePath: + if vmwareImcConfigFilePath and metaPath: + try: + set_gc_status(self._vmware_cust_conf, "Started") + + (md, ud, cfg, network) = load_cloudinit_data( + metaPath, userPath) + # TODO, if user data is enabled by default, nothing to do + + if network: + self._network_config = network + else: + fallbackNetwork = self.distro.generate_fallback_config() + self._network_config = fallbackNetwork + + except Exception as e: + _raise_error_status( + "Error parsing the customization Config File", + e, + GuestCustEvent.GUESTCUST_EVENT_CUSTOMIZE_FAILED, + vmwareImcConfigFilePath, + self._vmware_cust_conf) + + self._vmware_cust_found = True + found.append('vmware-tools') + + util.del_dir(os.path.dirname(imcdirpath)) + # xiaofengw, no need to enable_nics + # enable_nics(self._vmware_nics_to_enable) + set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_DONE, + GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) + set_gc_status(self._vmware_cust_conf, "Successful") + + elif vmwareImcConfigFilePath: self._vmware_nics_to_enable = "" try: - cf = ConfigFile(vmwareImcConfigFilePath) - self._vmware_cust_conf = Config(cf) set_gc_status(self._vmware_cust_conf, "Started") (md, ud, cfg) = read_vmware_imc(self._vmware_cust_conf) - self._vmware_nics_to_enable = get_nics_to_enable(nicspath) - imcdirpath = os.path.dirname(vmwareImcConfigFilePath) + self._vmware_nics_to_enable = get_nics_to_enable(nicsPath) product_marker = self._vmware_cust_conf.marker_id hasmarkerfile = check_marker_exists( product_marker, os.path.join(self.paths.cloud_dir, 'data')) @@ -378,15 +424,16 @@ def get_max_wait_from_cfg(cfg): return max_wait -def wait_for_imc_cfg_file(filename, maxwait=180, naplen=5, - dirpath="/var/run/vmware-imc"): +def wait_for_imc_file(filename, maxwait=180, naplen=5, + dirpath="/var/run/vmware-imc"): waited = 0 while waited < maxwait: fileFullPath = os.path.join(dirpath, filename) if os.path.isfile(fileFullPath): + LOG.debug("VMware Customization File '%s' is found", fileFullPath) return fileFullPath - LOG.debug("Waiting for VMware Customization Config File") + LOG.debug("Waiting for VMware Customization File '%s'", fileFullPath) time.sleep(naplen) waited += naplen return None @@ -684,4 +731,97 @@ def _raise_error_status(prefix, error, event, config_file, conf): util.del_dir(os.path.dirname(config_file)) raise error + +def load_cloudinit_data(metaPath, userPath): + """ + Load the cloud-init meta data, user data, cfg and network from the + given files + """ + LOG.debug('load meta data: %s: user data: %s', metaPath, userPath) + md = {} + cfg = {} + ud = {} + network = {} + # How to handle instance-id? + md = load(util.load_file(metaPath).replace("\r", "")) + + if 'network' in md: + network = md['network'] + del md['network'] + + if userPath: + ud = util.load_file(userPath).replace("\r", "") + return md, ud, cfg, network + + +class LoadError(Exception): + pass + + +class FileNotFound(Exception): + pass + + +def load(data): + ''' + first attempts to unmarshal the provided data as JSON, and if + that fails then attempts to unmarshal the data as YAML. If data is + None then a new dictionary is returned. + ''' + if not data: + return {} + try: + return json.loads(data) + except Exception: + try: + return safeyaml.load(data) + except Exception as e: + raise LoadError("Error parsing the meta data") from e + + +def collect_imc_files(cust_conf): + ''' + wait the "VMware Tools" daemon to copy all the customization files to + /var/run/vmware-imc directory. + - meta data: + mandatory if customization specification is raw cloud-init data + - user data: + optional if customization specification is raw cloud-init data + - nics.txt: + mandatory if customization specification is traditional one + ''' + metaPath = None + userPath = None + nicsPath = None + metaDataFile = cust_conf.meta_data_name + if metaDataFile: + metaPath = util.log_time( + logfunc=LOG.debug, + msg="waiting for meta data file", + func=wait_for_imc_file, + args=(metaDataFile, 10)) + if not metaPath: + raise FileNotFound("cloud-init meta data file is not found") + + userDataFile = cust_conf.user_data_name + if userDataFile: + userPath = util.log_time( + logfunc=LOG.debug, + msg="waiting for user data file", + func=wait_for_imc_file, + args=(userDataFile, 10)) + if not userPath: + raise FileNotFound("cloud-init user data file is not found") + else: + nicsPath = util.log_time( + logfunc=LOG.debug, + msg="waiting for nics.txt", + func=wait_for_imc_file, + args=("nics.txt", 10)) + if not nicsPath: + raise FileNotFound("nics.txt is not found") + + return metaPath, userPath, nicsPath + + # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py index 7109aef3815..bdfab5a041a 100644 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -27,6 +27,8 @@ class Config(object): UTC = 'DATETIME|UTC' POST_GC_STATUS = 'MISC|POST-GC-STATUS' DEFAULT_RUN_POST_SCRIPT = 'MISC|DEFAULT-RUN-POST-CUST-SCRIPT' + CLOUDINIT_META_DATA = 'CLOUDINIT|METADATA' + CLOUDINIT_USER_DATA = 'CLOUDINIT|USERDATA' def __init__(self, configFile): self._configFile = configFile @@ -130,4 +132,14 @@ def default_run_post_script(self): raise ValueError('defaultRunPostScript value should be yes/no') return defaultRunPostScript == 'yes' + @property + def meta_data_name(self): + """Return the name of cloud-init meta data.""" + return self._configFile.get(Config.CLOUDINIT_META_DATA, None) + + @property + def user_data_name(self): + """Return the name of cloud-init user data.""" + return self._configFile.get(Config.CLOUDINIT_USER_DATA, None) + # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index 16773de5217..a07a041a6f3 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -17,6 +17,8 @@ from cloudinit.sources import DataSourceOVF as dsovf from cloudinit.sources.helpers.vmware.imc.config_custom_script import ( CustomScriptNotFound) +from cloudinit.sources.DataSourceOVF import ( + FileNotFound, LoadError) MPATH = 'cloudinit.sources.DataSourceOVF.' @@ -177,7 +179,7 @@ def test_get_data_vmware_customization_disabled(self): {'dmi.read_dmi_data': 'vmware', 'util.del_dir': True, 'search_file': self.tdir, - 'wait_for_imc_cfg_file': conf_file, + 'wait_for_imc_file': conf_file, 'get_nics_to_enable': ''}, ds.get_data) customscript = self.tmp_path('test-script', self.tdir) @@ -214,7 +216,7 @@ def test_get_data_cust_script_disabled(self): {'dmi.read_dmi_data': 'vmware', 'util.del_dir': True, 'search_file': self.tdir, - 'wait_for_imc_cfg_file': conf_file, + 'wait_for_imc_file': conf_file, 'get_nics_to_enable': ''}, ds.get_data) self.assertIn('Custom script is disabled by VM Administrator', @@ -249,7 +251,7 @@ def test_get_data_cust_script_enabled(self): {'dmi.read_dmi_data': 'vmware', 'util.del_dir': True, 'search_file': self.tdir, - 'wait_for_imc_cfg_file': conf_file, + 'wait_for_imc_file': conf_file, 'get_nics_to_enable': ''}, ds.get_data) # Verify custom script is trying to be executed @@ -293,7 +295,7 @@ def my_get_tools_config(*args, **kwargs): {'dmi.read_dmi_data': 'vmware', 'util.del_dir': True, 'search_file': self.tdir, - 'wait_for_imc_cfg_file': conf_file, + 'wait_for_imc_file': conf_file, 'get_nics_to_enable': ''}, ds.get_data) # Verify custom script still runs although it is @@ -344,6 +346,332 @@ def test_get_data_vmware_seed_platform_info(self): 'vmware (%s/seed/ovf-env.xml)' % self.tdir, ds.subplatform) + def test_get_data_cloudinit_metadata_json(self): + """Test metadata can be loaded to cloud-init metadata and network. + The metadata format is json. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': False}, distro={}, + paths=paths) + # Prepare the conf file + conf_file = self.tmp_path('cust.cfg', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + """) + util.write_file(conf_file, conf_content) + # Prepare the meta data file + metadata_file = self.tmp_path('test-meta', self.tdir) + metadata_content = dedent("""\ + { + "instance-id": "cloud-vm", + "local-hostname": "my-host.domain.com", + "network": { + "version": 2, + "ethernets": { + "eths": { + "match": { + "name": "ens*" + }, + "dhcp4": true + } + } + } + } + """) + util.write_file(metadata_file, metadata_content) + + # Mock wait_for_imc_file() to return the file path + def my_wait_for_imc_file(*args, **kwargs): + if len(args): + return self.tdir + "/" + args[0] + else: + return self.tdir + "/" + kwargs['filename'] + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + with mock.patch(MPATH + 'wait_for_imc_file', + side_effect=my_wait_for_imc_file): + result = wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'get_nics_to_enable': ''}, + ds.get_data) + + self.assertTrue(result) + self.assertEqual("cloud-vm", ds.metadata['instance-id']) + self.assertEqual("my-host.domain.com", ds.metadata['local-hostname']) + self.assertEqual(2, ds.network_config['version']) + self.assertTrue(ds.network_config['ethernets']['eths']['dhcp4']) + + def test_get_data_cloudinit_metadata_yaml(self): + """Test metadata can be loaded to cloud-init metadata and network. + The metadata format is yaml. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': False}, distro={}, + paths=paths) + # Prepare the conf file + conf_file = self.tmp_path('cust.cfg', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + """) + util.write_file(conf_file, conf_content) + # Prepare the meta data file + metadata_file = self.tmp_path('test-meta', self.tdir) + metadata_content = dedent("""\ + instance-id: cloud-vm + local-hostname: my-host.domain.com + network: + version: 2 + ethernets: + nics: + match: + name: ens* + dhcp4: yes + """) + util.write_file(metadata_file, metadata_content) + + # Mock wait_for_imc_file() to return the file path + def my_wait_for_imc_file(*args, **kwargs): + if len(args): + return self.tdir + "/" + args[0] + else: + return self.tdir + "/" + kwargs['filename'] + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + with mock.patch(MPATH + 'wait_for_imc_file', + side_effect=my_wait_for_imc_file): + result = wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'get_nics_to_enable': ''}, + ds.get_data) + + self.assertTrue(result) + self.assertEqual("cloud-vm", ds.metadata['instance-id']) + self.assertEqual("my-host.domain.com", ds.metadata['local-hostname']) + self.assertEqual(2, ds.network_config['version']) + self.assertTrue(ds.network_config['ethernets']['nics']['dhcp4']) + + def test_get_data_cloudinit_metadata_not_valid(self): + """Test metadata is not JSON or YAML format. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': False}, distro={}, + paths=paths) + + # Prepare the conf file + conf_file = self.tmp_path('cust.cfg', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + """) + util.write_file(conf_file, conf_content) + + # Prepare the meta data file + metadata_file = self.tmp_path('test-meta', self.tdir) + metadata_content = "[This is not json or yaml format]a=b" + util.write_file(metadata_file, metadata_content) + + # Mock wait_for_imc_file() to return the file path + def my_wait_for_imc_file(*args, **kwargs): + if len(args): + return self.tdir + "/" + args[0] + else: + return self.tdir + "/" + kwargs['filename'] + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + with mock.patch(MPATH + 'wait_for_imc_file', + side_effect=my_wait_for_imc_file): + with self.assertRaises(LoadError) as context: + wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'get_nics_to_enable': ''}, + ds.get_data) + + self.assertIn('Error parsing the meta data', + str(context.exception)) + + def test_get_data_cloudinit_metadata_not_found(self): + """Test metadata file can't be found. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': False}, distro={}, + paths=paths) + # Prepare the conf file + conf_file = self.tmp_path('cust.cfg', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + """) + util.write_file(conf_file, conf_content) + # Don't prepare the meta data file + + # Mock wait_for_imc_file() to return the file path + def my_wait_for_imc_file(*args, **kwargs): + if len(args): + filename = args[0] + else: + filename = kwargs['filename'] + # mock test-meta can't be found + if filename == "test-meta": + return None + else: + return self.tdir + "/" + filename + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + with mock.patch(MPATH + 'wait_for_imc_file', + side_effect=my_wait_for_imc_file): + with self.assertRaises(FileNotFound) as context: + wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'get_nics_to_enable': ''}, + ds.get_data) + + self.assertIn('cloud-init meta data file is not found', + str(context.exception)) + + def test_get_data_cloudinit_userdata(self): + """Test user data can be loaded to cloud-init user data. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': False}, distro={}, + paths=paths) + + # Prepare the conf file + conf_file = self.tmp_path('cust.cfg', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + USERDATA = test-user + """) + util.write_file(conf_file, conf_content) + + # Prepare the meta data file + metadata_file = self.tmp_path('test-meta', self.tdir) + metadata_content = dedent("""\ + instance-id: cloud-vm + local-hostname: my-host.domain.com + network: + version: 2 + ethernets: + nics: + match: + name: ens* + dhcp4: yes + """) + util.write_file(metadata_file, metadata_content) + + # Prepare the user data file + userdata_file = self.tmp_path('test-user', self.tdir) + userdata_content = "This is the user data" + util.write_file(userdata_file, userdata_content) + + # Mock wait_for_imc_file() to return the file path + def my_wait_for_imc_file(*args, **kwargs): + if len(args): + return self.tdir + "/" + args[0] + else: + return self.tdir + "/" + kwargs['filename'] + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + with mock.patch(MPATH + 'wait_for_imc_file', + side_effect=my_wait_for_imc_file): + result = wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'get_nics_to_enable': ''}, + ds.get_data) + + self.assertTrue(result) + self.assertEqual("cloud-vm", ds.metadata['instance-id']) + self.assertEqual(userdata_content, ds.userdata_raw) + + def test_get_data_cloudinit_userdata_not_found(self): + """Test userdata file can't be found. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': False}, distro={}, + paths=paths) + + # Prepare the conf file + conf_file = self.tmp_path('cust.cfg', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + USERDATA = test-user + """) + util.write_file(conf_file, conf_content) + + # Prepare the meta data file + metadata_file = self.tmp_path('test-meta', self.tdir) + metadata_content = dedent("""\ + instance-id: cloud-vm + local-hostname: my-host.domain.com + network: + version: 2 + ethernets: + nics: + match: + name: ens* + dhcp4: yes + """) + util.write_file(metadata_file, metadata_content) + + # Don't prepare the user data file + + # Mock wait_for_imc_file() to return the file path + def my_wait_for_imc_file(*args, **kwargs): + if len(args): + filename = args[0] + else: + filename = kwargs['filename'] + # mock test-user can't be found + if filename == "test-user": + return None + else: + return self.tdir + "/" + filename + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + with mock.patch(MPATH + 'wait_for_imc_file', + side_effect=my_wait_for_imc_file): + with self.assertRaises(FileNotFound) as context: + wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'get_nics_to_enable': ''}, + ds.get_data) + + self.assertIn('cloud-init user data file is not found', + str(context.exception)) + class TestTransportIso9660(CiTestCase): diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py index 9c7d25fac57..430cc69f157 100644 --- a/tests/unittests/test_vmware_config_file.py +++ b/tests/unittests/test_vmware_config_file.py @@ -525,5 +525,21 @@ def test_a_primary_nic_with_gateway(self): 'gateway': '10.20.87.253'}]}], nc.generate()) + def test_meta_data(self): + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + conf = Config(cf) + self.assertIsNone(conf.meta_data_name) + cf._insertKey("CLOUDINIT|METADATA", "test-metadata") + conf = Config(cf) + self.assertEqual("test-metadata", conf.meta_data_name) + + def test_user_data(self): + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + conf = Config(cf) + self.assertIsNone(conf.user_data_name) + cf._insertKey("CLOUDINIT|USERDATA", "test-userdata") + conf = Config(cf) + self.assertEqual("test-userdata", conf.user_data_name) + # vi: ts=4 expandtab From 1c19fef7b507d0af8bbd22d0797114105a75824b Mon Sep 17 00:00:00 2001 From: xiaofengw-vmware Date: Wed, 25 Nov 2020 03:25:08 +0000 Subject: [PATCH 2/8] VMware: support raw cloud-init data in vm customization --- cloudinit/sources/DataSourceOVF.py | 49 ++++++++++++------- .../helpers/vmware/imc/guestcust_error.py | 1 + tests/unittests/test_datasource/test_ovf.py | 27 +++++++--- 3 files changed, 51 insertions(+), 26 deletions(-) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index aed5e082bf9..6bbe8f3c4fc 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -101,9 +101,7 @@ def _get_data(self): if not self.vmware_customization_supported: LOG.debug("Skipping the check for " "VMware Customization support") - elif not util.get_cfg_option_bool( - self.sys_cfg, "disable_vmware_customization", True): - + else: search_paths = ( "/usr/lib/vmware-tools", "/usr/lib64/vmware-tools", "/usr/lib/open-vm-tools", "/usr/lib64/open-vm-tools") @@ -128,16 +126,14 @@ def _get_data(self): msg="waiting for configuration file", func=wait_for_imc_file, args=("cust.cfg", max_wait)) - - if vmwareImcConfigFilePath: - imcdirpath = os.path.dirname(vmwareImcConfigFilePath) - cf = ConfigFile(vmwareImcConfigFilePath) - self._vmware_cust_conf = Config(cf) - else: LOG.debug("Did not find the customization plugin.") + metaPath = None if vmwareImcConfigFilePath: + imcdirpath = os.path.dirname(vmwareImcConfigFilePath) + cf = ConfigFile(vmwareImcConfigFilePath) + self._vmware_cust_conf = Config(cf) LOG.debug("Found VMware Customization Config File at %s", vmwareImcConfigFilePath) try: @@ -152,16 +148,25 @@ def _get_data(self): self._vmware_cust_conf) else: LOG.debug("Did not find VMware Customization Config File") - else: - LOG.debug("Customization for VMware platform is disabled.") + + # if meta data is avaiable, ignore disable_vmware_customization + if not metaPath: + if util.get_cfg_option_bool(self.sys_cfg, + "disable_vmware_customization", + True): + LOG.debug( + "Customization for VMware platform is disabled.") + # reset vmwareImcConfigFilePath to None to avoid + # customization for VMware platform + vmwareImcConfigFilePath = None if vmwareImcConfigFilePath and metaPath: try: set_gc_status(self._vmware_cust_conf, "Started") + LOG.debug("Start to load cloud-init meta data and user data") (md, ud, cfg, network) = load_cloudinit_data( metaPath, userPath) - # TODO, if user data is enabled by default, nothing to do if network: self._network_config = network @@ -169,9 +174,16 @@ def _get_data(self): fallbackNetwork = self.distro.generate_fallback_config() self._network_config = fallbackNetwork + except LoadError as e: + _raise_error_status( + "Error parsing the cloud-init meta data", + e, + GuestCustErrorEnum.GUESTCUST_ERROR_WRONG_META_FORMAT, + vmwareImcConfigFilePath, + self._vmware_cust_conf) except Exception as e: _raise_error_status( - "Error parsing the customization Config File", + "Error loading cloud-init configuration", e, GuestCustEvent.GUESTCUST_EVENT_CUSTOMIZE_FAILED, vmwareImcConfigFilePath, @@ -180,9 +192,7 @@ def _get_data(self): self._vmware_cust_found = True found.append('vmware-tools') - util.del_dir(os.path.dirname(imcdirpath)) - # xiaofengw, no need to enable_nics - # enable_nics(self._vmware_nics_to_enable) + util.del_dir(imcdirpath) set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_DONE, GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) @@ -737,11 +747,12 @@ def load_cloudinit_data(metaPath, userPath): Load the cloud-init meta data, user data, cfg and network from the given files """ - LOG.debug('load meta data: %s: user data: %s', metaPath, userPath) + LOG.debug('load meta data from: %s: user data from: %s', + metaPath, userPath) md = {} cfg = {} - ud = {} - network = {} + ud = None + network = None # How to handle instance-id? md = load(util.load_file(metaPath).replace("\r", "")) diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py index 65ae7390258..96d839b8b65 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py @@ -11,5 +11,6 @@ class GuestCustErrorEnum(object): GUESTCUST_ERROR_SUCCESS = 0 GUESTCUST_ERROR_SCRIPT_DISABLED = 6 + GUESTCUST_ERROR_WRONG_META_FORMAT = 9 # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index a07a041a6f3..e5a84ab1a40 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -140,16 +140,29 @@ def test_get_data_false_on_none_dmi_data(self): 'DEBUG: No system-product-name found', self.logs.getvalue()) def test_get_data_no_vmware_customization_disabled(self): - """When vmware customization is disabled via sys_cfg log a message.""" + """When cloud-init workflow for vmware is disabled via sys_cfgi and + no meta data provided, log a message. + """ paths = Paths({'cloud_dir': self.tdir}) ds = self.datasource( sys_cfg={'disable_vmware_customization': True}, distro={}, paths=paths) + conf_file = self.tmp_path('test-cust', self.tdir) + conf_content = dedent("""\ + [CUSTOM-SCRIPT] + SCRIPT-NAME = test-script + [MISC] + MARKER-ID = 12345345 + """) + util.write_file(conf_file, conf_content) retcode = wrap_and_call( 'cloudinit.sources.DataSourceOVF', {'dmi.read_dmi_data': 'vmware', 'transport_iso9660': NOT_FOUND, - 'transport_vmware_guestinfo': NOT_FOUND}, + 'transport_vmware_guestinfo': NOT_FOUND, + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_file': conf_file}, ds.get_data) self.assertFalse(retcode, 'Expected False return from ds.get_data') self.assertIn( @@ -352,7 +365,7 @@ def test_get_data_cloudinit_metadata_json(self): """ paths = Paths({'cloud_dir': self.tdir}) ds = self.datasource( - sys_cfg={'disable_vmware_customization': False}, distro={}, + sys_cfg={'disable_vmware_customization': True}, distro={}, paths=paths) # Prepare the conf file conf_file = self.tmp_path('cust.cfg', self.tdir) @@ -413,7 +426,7 @@ def test_get_data_cloudinit_metadata_yaml(self): """ paths = Paths({'cloud_dir': self.tdir}) ds = self.datasource( - sys_cfg={'disable_vmware_customization': False}, distro={}, + sys_cfg={'disable_vmware_customization': True}, distro={}, paths=paths) # Prepare the conf file conf_file = self.tmp_path('cust.cfg', self.tdir) @@ -467,7 +480,7 @@ def test_get_data_cloudinit_metadata_not_valid(self): """ paths = Paths({'cloud_dir': self.tdir}) ds = self.datasource( - sys_cfg={'disable_vmware_customization': False}, distro={}, + sys_cfg={'disable_vmware_customization': True}, distro={}, paths=paths) # Prepare the conf file @@ -511,7 +524,7 @@ def test_get_data_cloudinit_metadata_not_found(self): """ paths = Paths({'cloud_dir': self.tdir}) ds = self.datasource( - sys_cfg={'disable_vmware_customization': False}, distro={}, + sys_cfg={'disable_vmware_customization': True}, distro={}, paths=paths) # Prepare the conf file conf_file = self.tmp_path('cust.cfg', self.tdir) @@ -615,7 +628,7 @@ def test_get_data_cloudinit_userdata_not_found(self): """ paths = Paths({'cloud_dir': self.tdir}) ds = self.datasource( - sys_cfg={'disable_vmware_customization': False}, distro={}, + sys_cfg={'disable_vmware_customization': True}, distro={}, paths=paths) # Prepare the conf file From 0fb205b03d96891ca535f08747ec3013595d49e0 Mon Sep 17 00:00:00 2001 From: xiaofengw-vmware Date: Wed, 25 Nov 2020 03:31:41 +0000 Subject: [PATCH 3/8] VMware: support raw cloud-init data in vm customization --- cloudinit/sources/DataSourceOVF.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 6bbe8f3c4fc..b08dc21fd52 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -753,7 +753,7 @@ def load_cloudinit_data(metaPath, userPath): cfg = {} ud = None network = None - # How to handle instance-id? + md = load(util.load_file(metaPath).replace("\r", "")) if 'network' in md: From 71230a7df3b4067958865c75c0c7fa20c430b86f Mon Sep 17 00:00:00 2001 From: xiaofengw-vmware Date: Wed, 25 Nov 2020 11:51:54 +0000 Subject: [PATCH 4/8] fix the failure test cases on travis ci --- tests/unittests/test_datasource/test_ovf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index e5a84ab1a40..090796d3ddf 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -412,7 +412,7 @@ def my_wait_for_imc_file(*args, **kwargs): 'util.del_dir': True, 'search_file': self.tdir, 'get_nics_to_enable': ''}, - ds.get_data) + ds._get_data) self.assertTrue(result) self.assertEqual("cloud-vm", ds.metadata['instance-id']) @@ -467,7 +467,7 @@ def my_wait_for_imc_file(*args, **kwargs): 'util.del_dir': True, 'search_file': self.tdir, 'get_nics_to_enable': ''}, - ds.get_data) + ds._get_data) self.assertTrue(result) self.assertEqual("cloud-vm", ds.metadata['instance-id']) @@ -617,7 +617,7 @@ def my_wait_for_imc_file(*args, **kwargs): 'util.del_dir': True, 'search_file': self.tdir, 'get_nics_to_enable': ''}, - ds.get_data) + ds._get_data) self.assertTrue(result) self.assertEqual("cloud-vm", ds.metadata['instance-id']) From 462612e0e90f07fbf7bc7d498af50a2014475180 Mon Sep 17 00:00:00 2001 From: xiaofengw-vmware Date: Mon, 7 Dec 2020 10:57:50 +0000 Subject: [PATCH 5/8] [VMware] Support cloudinit raw data feature:address comments --- cloudinit/sources/DataSourceOVF.py | 82 +++----- tests/unittests/test_datasource/test_ovf.py | 198 +++++++------------- 2 files changed, 99 insertions(+), 181 deletions(-) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index b08dc21fd52..ba19a7d537c 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -9,7 +9,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import base64 -import json import os import re import time @@ -49,6 +48,7 @@ CONFGROUPNAME_GUESTCUSTOMIZATION = "deployPkg" GUESTCUSTOMIZATION_ENABLE_CUST_SCRIPTS = "enable-custom-scripts" +VMWARE_IMC_DIR = "/var/run/vmware-imc" class DataSourceOVF(sources.DataSource): @@ -78,7 +78,7 @@ def _get_data(self): ud = "" vd = "" vmwareImcConfigFilePath = None - nicsPath = None + nicspath = None defaults = { "instance-id": "iid-dsovf", @@ -124,7 +124,7 @@ def _get_data(self): vmwareImcConfigFilePath = util.log_time( logfunc=LOG.debug, msg="waiting for configuration file", - func=wait_for_imc_file, + func=wait_for_imc_cfg_file, args=("cust.cfg", max_wait)) else: LOG.debug("Did not find the customization plugin.") @@ -137,7 +137,7 @@ def _get_data(self): LOG.debug("Found VMware Customization Config File at %s", vmwareImcConfigFilePath) try: - (metaPath, userPath, nicsPath) = collect_imc_files( + (metaPath, userPath, nicspath) = collect_imc_files( self._vmware_cust_conf) except Exception as e: _raise_error_status( @@ -149,7 +149,7 @@ def _get_data(self): else: LOG.debug("Did not find VMware Customization Config File") - # if meta data is avaiable, ignore disable_vmware_customization + # ignore disable_vmware_customization if meta data is available if not metaPath: if util.get_cfg_option_bool(self.sys_cfg, "disable_vmware_customization", @@ -174,7 +174,7 @@ def _get_data(self): fallbackNetwork = self.distro.generate_fallback_config() self._network_config = fallbackNetwork - except LoadError as e: + except safeyaml.YAMLError as e: _raise_error_status( "Error parsing the cloud-init meta data", e, @@ -204,7 +204,7 @@ def _get_data(self): set_gc_status(self._vmware_cust_conf, "Started") (md, ud, cfg) = read_vmware_imc(self._vmware_cust_conf) - self._vmware_nics_to_enable = get_nics_to_enable(nicsPath) + self._vmware_nics_to_enable = get_nics_to_enable(nicspath) product_marker = self._vmware_cust_conf.marker_id hasmarkerfile = check_marker_exists( product_marker, os.path.join(self.paths.cloud_dir, 'data')) @@ -434,16 +434,15 @@ def get_max_wait_from_cfg(cfg): return max_wait -def wait_for_imc_file(filename, maxwait=180, naplen=5, - dirpath="/var/run/vmware-imc"): +def wait_for_imc_cfg_file(filename, maxwait=180, naplen=5, + dirpath="/var/run/vmware-imc"): waited = 0 while waited < maxwait: fileFullPath = os.path.join(dirpath, filename) if os.path.isfile(fileFullPath): - LOG.debug("VMware Customization File '%s' is found", fileFullPath) return fileFullPath - LOG.debug("Waiting for VMware Customization File '%s'", fileFullPath) + LOG.debug("Waiting for VMware Customization Config File") time.sleep(naplen) waited += naplen return None @@ -754,7 +753,7 @@ def load_cloudinit_data(metaPath, userPath): ud = None network = None - md = load(util.load_file(metaPath).replace("\r", "")) + md = load(util.load_file(metaPath)) if 'network' in md: network = md['network'] @@ -765,72 +764,47 @@ def load_cloudinit_data(metaPath, userPath): return md, ud, cfg, network -class LoadError(Exception): - pass - - -class FileNotFound(Exception): - pass - - def load(data): ''' - first attempts to unmarshal the provided data as JSON, and if - that fails then attempts to unmarshal the data as YAML. If data is - None then a new dictionary is returned. + The meta data could be JSON or YAML. Since YAML is a strict superset of + JSON, we will unmarshal the data as YAML. If data is None then a new + dictionary is returned. ''' if not data: return {} - try: - return json.loads(data) - except Exception: - try: - return safeyaml.load(data) - except Exception as e: - raise LoadError("Error parsing the meta data") from e + return safeyaml.load(data) def collect_imc_files(cust_conf): ''' - wait the "VMware Tools" daemon to copy all the customization files to - /var/run/vmware-imc directory. + collect all the other imc files. Since these files are copied + before cust.cfg is copied, check if they exist or not directly. - meta data: mandatory if customization specification is raw cloud-init data - user data: optional if customization specification is raw cloud-init data - nics.txt: - mandatory if customization specification is traditional one + optional if customization specification is traditional one ''' metaPath = None userPath = None nicsPath = None metaDataFile = cust_conf.meta_data_name if metaDataFile: - metaPath = util.log_time( - logfunc=LOG.debug, - msg="waiting for meta data file", - func=wait_for_imc_file, - args=(metaDataFile, 10)) - if not metaPath: - raise FileNotFound("cloud-init meta data file is not found") + metaPath = os.path.join(VMWARE_IMC_DIR, metaDataFile) + if not os.path.exists(metaPath): + raise FileNotFoundError("meta data file is not found") userDataFile = cust_conf.user_data_name if userDataFile: - userPath = util.log_time( - logfunc=LOG.debug, - msg="waiting for user data file", - func=wait_for_imc_file, - args=(userDataFile, 10)) - if not userPath: - raise FileNotFound("cloud-init user data file is not found") + userPath = os.path.join(VMWARE_IMC_DIR, userDataFile) + if not os.path.exists(userPath): + raise FileNotFoundError("user data file is not found") else: - nicsPath = util.log_time( - logfunc=LOG.debug, - msg="waiting for nics.txt", - func=wait_for_imc_file, - args=("nics.txt", 10)) - if not nicsPath: - raise FileNotFound("nics.txt is not found") + nicsPath = os.path.join(VMWARE_IMC_DIR, "nics.txt") + if not os.path.exists(nicsPath): + LOG.debug('nics.txt is not exist.') + nicsPath = None return metaPath, userPath, nicsPath diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index 090796d3ddf..a7192c5e80a 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -17,8 +17,7 @@ from cloudinit.sources import DataSourceOVF as dsovf from cloudinit.sources.helpers.vmware.imc.config_custom_script import ( CustomScriptNotFound) -from cloudinit.sources.DataSourceOVF import ( - FileNotFound, LoadError) +from cloudinit.safeyaml import YAMLError MPATH = 'cloudinit.sources.DataSourceOVF.' @@ -162,7 +161,7 @@ def test_get_data_no_vmware_customization_disabled(self): 'transport_vmware_guestinfo': NOT_FOUND, 'util.del_dir': True, 'search_file': self.tdir, - 'wait_for_imc_file': conf_file}, + 'wait_for_imc_cfg_file': conf_file}, ds.get_data) self.assertFalse(retcode, 'Expected False return from ds.get_data') self.assertIn( @@ -192,7 +191,7 @@ def test_get_data_vmware_customization_disabled(self): {'dmi.read_dmi_data': 'vmware', 'util.del_dir': True, 'search_file': self.tdir, - 'wait_for_imc_file': conf_file, + 'wait_for_imc_cfg_file': conf_file, 'get_nics_to_enable': ''}, ds.get_data) customscript = self.tmp_path('test-script', self.tdir) @@ -229,7 +228,7 @@ def test_get_data_cust_script_disabled(self): {'dmi.read_dmi_data': 'vmware', 'util.del_dir': True, 'search_file': self.tdir, - 'wait_for_imc_file': conf_file, + 'wait_for_imc_cfg_file': conf_file, 'get_nics_to_enable': ''}, ds.get_data) self.assertIn('Custom script is disabled by VM Administrator', @@ -264,7 +263,7 @@ def test_get_data_cust_script_enabled(self): {'dmi.read_dmi_data': 'vmware', 'util.del_dir': True, 'search_file': self.tdir, - 'wait_for_imc_file': conf_file, + 'wait_for_imc_cfg_file': conf_file, 'get_nics_to_enable': ''}, ds.get_data) # Verify custom script is trying to be executed @@ -308,7 +307,7 @@ def my_get_tools_config(*args, **kwargs): {'dmi.read_dmi_data': 'vmware', 'util.del_dir': True, 'search_file': self.tdir, - 'wait_for_imc_file': conf_file, + 'wait_for_imc_cfg_file': conf_file, 'get_nics_to_enable': ''}, ds.get_data) # Verify custom script still runs although it is @@ -368,7 +367,7 @@ def test_get_data_cloudinit_metadata_json(self): sys_cfg={'disable_vmware_customization': True}, distro={}, paths=paths) # Prepare the conf file - conf_file = self.tmp_path('cust.cfg', self.tdir) + conf_file = self.tmp_path('test-cust', self.tdir) conf_content = dedent("""\ [CLOUDINIT] METADATA = test-meta @@ -395,24 +394,17 @@ def test_get_data_cloudinit_metadata_json(self): """) util.write_file(metadata_file, metadata_content) - # Mock wait_for_imc_file() to return the file path - def my_wait_for_imc_file(*args, **kwargs): - if len(args): - return self.tdir + "/" + args[0] - else: - return self.tdir + "/" + kwargs['filename'] - with mock.patch(MPATH + 'set_customization_status', return_value=('msg', b'')): - with mock.patch(MPATH + 'wait_for_imc_file', - side_effect=my_wait_for_imc_file): - result = wrap_and_call( - 'cloudinit.sources.DataSourceOVF', - {'dmi.read_dmi_data': 'vmware', - 'util.del_dir': True, - 'search_file': self.tdir, - 'get_nics_to_enable': ''}, - ds._get_data) + result = wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'collect_imc_files': [self.tdir + '/test-meta', '', ''], + 'get_nics_to_enable': ''}, + ds._get_data) self.assertTrue(result) self.assertEqual("cloud-vm", ds.metadata['instance-id']) @@ -429,7 +421,7 @@ def test_get_data_cloudinit_metadata_yaml(self): sys_cfg={'disable_vmware_customization': True}, distro={}, paths=paths) # Prepare the conf file - conf_file = self.tmp_path('cust.cfg', self.tdir) + conf_file = self.tmp_path('test-cust', self.tdir) conf_content = dedent("""\ [CLOUDINIT] METADATA = test-meta @@ -450,24 +442,17 @@ def test_get_data_cloudinit_metadata_yaml(self): """) util.write_file(metadata_file, metadata_content) - # Mock wait_for_imc_file() to return the file path - def my_wait_for_imc_file(*args, **kwargs): - if len(args): - return self.tdir + "/" + args[0] - else: - return self.tdir + "/" + kwargs['filename'] - with mock.patch(MPATH + 'set_customization_status', return_value=('msg', b'')): - with mock.patch(MPATH + 'wait_for_imc_file', - side_effect=my_wait_for_imc_file): - result = wrap_and_call( - 'cloudinit.sources.DataSourceOVF', - {'dmi.read_dmi_data': 'vmware', - 'util.del_dir': True, - 'search_file': self.tdir, - 'get_nics_to_enable': ''}, - ds._get_data) + result = wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'collect_imc_files': [self.tdir + '/test-meta', '', ''], + 'get_nics_to_enable': ''}, + ds._get_data) self.assertTrue(result) self.assertEqual("cloud-vm", ds.metadata['instance-id']) @@ -484,7 +469,7 @@ def test_get_data_cloudinit_metadata_not_valid(self): paths=paths) # Prepare the conf file - conf_file = self.tmp_path('cust.cfg', self.tdir) + conf_file = self.tmp_path('test-cust', self.tdir) conf_content = dedent("""\ [CLOUDINIT] METADATA = test-meta @@ -496,27 +481,20 @@ def test_get_data_cloudinit_metadata_not_valid(self): metadata_content = "[This is not json or yaml format]a=b" util.write_file(metadata_file, metadata_content) - # Mock wait_for_imc_file() to return the file path - def my_wait_for_imc_file(*args, **kwargs): - if len(args): - return self.tdir + "/" + args[0] - else: - return self.tdir + "/" + kwargs['filename'] - with mock.patch(MPATH + 'set_customization_status', return_value=('msg', b'')): - with mock.patch(MPATH + 'wait_for_imc_file', - side_effect=my_wait_for_imc_file): - with self.assertRaises(LoadError) as context: - wrap_and_call( - 'cloudinit.sources.DataSourceOVF', - {'dmi.read_dmi_data': 'vmware', - 'util.del_dir': True, - 'search_file': self.tdir, - 'get_nics_to_enable': ''}, - ds.get_data) + with self.assertRaises(YAMLError) as context: + wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'collect_imc_files': [self.tdir + '/test-meta', '', ''], + 'get_nics_to_enable': ''}, + ds.get_data) - self.assertIn('Error parsing the meta data', + self.assertIn("expected '', but found ''", str(context.exception)) def test_get_data_cloudinit_metadata_not_found(self): @@ -527,7 +505,7 @@ def test_get_data_cloudinit_metadata_not_found(self): sys_cfg={'disable_vmware_customization': True}, distro={}, paths=paths) # Prepare the conf file - conf_file = self.tmp_path('cust.cfg', self.tdir) + conf_file = self.tmp_path('test-cust', self.tdir) conf_content = dedent("""\ [CLOUDINIT] METADATA = test-meta @@ -535,33 +513,19 @@ def test_get_data_cloudinit_metadata_not_found(self): util.write_file(conf_file, conf_content) # Don't prepare the meta data file - # Mock wait_for_imc_file() to return the file path - def my_wait_for_imc_file(*args, **kwargs): - if len(args): - filename = args[0] - else: - filename = kwargs['filename'] - # mock test-meta can't be found - if filename == "test-meta": - return None - else: - return self.tdir + "/" + filename - with mock.patch(MPATH + 'set_customization_status', return_value=('msg', b'')): - with mock.patch(MPATH + 'wait_for_imc_file', - side_effect=my_wait_for_imc_file): - with self.assertRaises(FileNotFound) as context: - wrap_and_call( - 'cloudinit.sources.DataSourceOVF', - {'dmi.read_dmi_data': 'vmware', - 'util.del_dir': True, - 'search_file': self.tdir, - 'get_nics_to_enable': ''}, - ds.get_data) + with self.assertRaises(FileNotFoundError) as context: + wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'get_nics_to_enable': ''}, + ds.get_data) - self.assertIn('cloud-init meta data file is not found', - str(context.exception)) + self.assertIn('file is not found', str(context.exception)) def test_get_data_cloudinit_userdata(self): """Test user data can be loaded to cloud-init user data. @@ -572,7 +536,7 @@ def test_get_data_cloudinit_userdata(self): paths=paths) # Prepare the conf file - conf_file = self.tmp_path('cust.cfg', self.tdir) + conf_file = self.tmp_path('test-cust', self.tdir) conf_content = dedent("""\ [CLOUDINIT] METADATA = test-meta @@ -600,24 +564,18 @@ def test_get_data_cloudinit_userdata(self): userdata_content = "This is the user data" util.write_file(userdata_file, userdata_content) - # Mock wait_for_imc_file() to return the file path - def my_wait_for_imc_file(*args, **kwargs): - if len(args): - return self.tdir + "/" + args[0] - else: - return self.tdir + "/" + kwargs['filename'] - with mock.patch(MPATH + 'set_customization_status', return_value=('msg', b'')): - with mock.patch(MPATH + 'wait_for_imc_file', - side_effect=my_wait_for_imc_file): - result = wrap_and_call( - 'cloudinit.sources.DataSourceOVF', - {'dmi.read_dmi_data': 'vmware', - 'util.del_dir': True, - 'search_file': self.tdir, - 'get_nics_to_enable': ''}, - ds._get_data) + result = wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'collect_imc_files': [self.tdir + '/test-meta', + self.tdir + '/test-user', ''], + 'get_nics_to_enable': ''}, + ds._get_data) self.assertTrue(result) self.assertEqual("cloud-vm", ds.metadata['instance-id']) @@ -632,7 +590,7 @@ def test_get_data_cloudinit_userdata_not_found(self): paths=paths) # Prepare the conf file - conf_file = self.tmp_path('cust.cfg', self.tdir) + conf_file = self.tmp_path('test-cust', self.tdir) conf_content = dedent("""\ [CLOUDINIT] METADATA = test-meta @@ -657,33 +615,19 @@ def test_get_data_cloudinit_userdata_not_found(self): # Don't prepare the user data file - # Mock wait_for_imc_file() to return the file path - def my_wait_for_imc_file(*args, **kwargs): - if len(args): - filename = args[0] - else: - filename = kwargs['filename'] - # mock test-user can't be found - if filename == "test-user": - return None - else: - return self.tdir + "/" + filename - with mock.patch(MPATH + 'set_customization_status', return_value=('msg', b'')): - with mock.patch(MPATH + 'wait_for_imc_file', - side_effect=my_wait_for_imc_file): - with self.assertRaises(FileNotFound) as context: - wrap_and_call( - 'cloudinit.sources.DataSourceOVF', - {'dmi.read_dmi_data': 'vmware', - 'util.del_dir': True, - 'search_file': self.tdir, - 'get_nics_to_enable': ''}, - ds.get_data) + with self.assertRaises(FileNotFoundError) as context: + wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'get_nics_to_enable': ''}, + ds.get_data) - self.assertIn('cloud-init user data file is not found', - str(context.exception)) + self.assertIn('file is not found', str(context.exception)) class TestTransportIso9660(CiTestCase): From b09900934b6ff8f8a95d585329d5eb57b3aeb077 Mon Sep 17 00:00:00 2001 From: Xiaofeng Wang Date: Wed, 9 Dec 2020 03:08:42 +0000 Subject: [PATCH 6/8] [VMware] Support cloudinit raw data feature:address Chad's comments --- cloudinit/sources/DataSourceOVF.py | 80 ++++++++++----------- tests/unittests/test_datasource/test_ovf.py | 4 +- 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index ba19a7d537c..da8d832d76d 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -129,7 +129,7 @@ def _get_data(self): else: LOG.debug("Did not find the customization plugin.") - metaPath = None + md_path = None if vmwareImcConfigFilePath: imcdirpath = os.path.dirname(vmwareImcConfigFilePath) cf = ConfigFile(vmwareImcConfigFilePath) @@ -137,7 +137,7 @@ def _get_data(self): LOG.debug("Found VMware Customization Config File at %s", vmwareImcConfigFilePath) try: - (metaPath, userPath, nicspath) = collect_imc_files( + (md_path, ud_path, nicspath) = collect_imc_files( self._vmware_cust_conf) except Exception as e: _raise_error_status( @@ -150,7 +150,7 @@ def _get_data(self): LOG.debug("Did not find VMware Customization Config File") # ignore disable_vmware_customization if meta data is available - if not metaPath: + if not md_path: if util.get_cfg_option_bool(self.sys_cfg, "disable_vmware_customization", True): @@ -160,19 +160,19 @@ def _get_data(self): # customization for VMware platform vmwareImcConfigFilePath = None - if vmwareImcConfigFilePath and metaPath: + if vmwareImcConfigFilePath and md_path: + set_gc_status(self._vmware_cust_conf, "Started") + LOG.debug("Start to load cloud-init meta data and user data") try: - set_gc_status(self._vmware_cust_conf, "Started") - - LOG.debug("Start to load cloud-init meta data and user data") (md, ud, cfg, network) = load_cloudinit_data( - metaPath, userPath) + md_path, ud_path) if network: self._network_config = network else: - fallbackNetwork = self.distro.generate_fallback_config() - self._network_config = fallbackNetwork + self._network_config = ( + self.distro.generate_fallback_config() + ) except safeyaml.YAMLError as e: _raise_error_status( @@ -741,26 +741,26 @@ def _raise_error_status(prefix, error, event, config_file, conf): raise error -def load_cloudinit_data(metaPath, userPath): +def load_cloudinit_data(md_path, ud_path): """ Load the cloud-init meta data, user data, cfg and network from the given files """ LOG.debug('load meta data from: %s: user data from: %s', - metaPath, userPath) + md_path, ud_path) md = {} cfg = {} ud = None network = None - md = load(util.load_file(metaPath)) + md = load(util.load_file(md_path)) if 'network' in md: network = md['network'] del md['network'] - if userPath: - ud = util.load_file(userPath).replace("\r", "") + if ud_path: + ud = util.load_file(ud_path).replace("\r", "") return md, ud, cfg, network @@ -777,36 +777,32 @@ def load(data): def collect_imc_files(cust_conf): ''' - collect all the other imc files. Since these files are copied - before cust.cfg is copied, check if they exist or not directly. - - meta data: - mandatory if customization specification is raw cloud-init data - - user data: - optional if customization specification is raw cloud-init data - - nics.txt: - optional if customization specification is traditional one + collect all the other imc files. metadata/userdata must be present + if they are specified in customization configuration. ''' - metaPath = None - userPath = None - nicsPath = None - metaDataFile = cust_conf.meta_data_name - if metaDataFile: - metaPath = os.path.join(VMWARE_IMC_DIR, metaDataFile) - if not os.path.exists(metaPath): - raise FileNotFoundError("meta data file is not found") - - userDataFile = cust_conf.user_data_name - if userDataFile: - userPath = os.path.join(VMWARE_IMC_DIR, userDataFile) - if not os.path.exists(userPath): - raise FileNotFoundError("user data file is not found") + md_path = None + ud_path = None + nics_path = None + md_file = cust_conf.meta_data_name + if md_file: + md_path = os.path.join(VMWARE_IMC_DIR, md_file) + if not os.path.exists(md_path): + raise FileNotFoundError("meta data file is not found: %s" + % md_path) + + ud_file = cust_conf.user_data_name + if ud_file: + ud_path = os.path.join(VMWARE_IMC_DIR, ud_file) + if not os.path.exists(ud_path): + raise FileNotFoundError("user data file is not found: %s" + % ud_path) else: - nicsPath = os.path.join(VMWARE_IMC_DIR, "nics.txt") - if not os.path.exists(nicsPath): - LOG.debug('nics.txt is not exist.') - nicsPath = None + nics_path = os.path.join(VMWARE_IMC_DIR, "nics.txt") + if not os.path.exists(nics_path): + LOG.debug('%s is not exist.', nics_path) + nics_path = None - return metaPath, userPath, nicsPath + return md_path, ud_path, nics_path # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index a7192c5e80a..971c83889f4 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -525,7 +525,7 @@ def test_get_data_cloudinit_metadata_not_found(self): 'get_nics_to_enable': ''}, ds.get_data) - self.assertIn('file is not found', str(context.exception)) + self.assertIn('is not found', str(context.exception)) def test_get_data_cloudinit_userdata(self): """Test user data can be loaded to cloud-init user data. @@ -627,7 +627,7 @@ def test_get_data_cloudinit_userdata_not_found(self): 'get_nics_to_enable': ''}, ds.get_data) - self.assertIn('file is not found', str(context.exception)) + self.assertIn('is not found', str(context.exception)) class TestTransportIso9660(CiTestCase): From 27ec8d062fcf9911aba2c09856935557668ba2ed Mon Sep 17 00:00:00 2001 From: Xiaofeng Wang Date: Wed, 6 Jan 2021 03:14:11 +0000 Subject: [PATCH 7/8] [VMware] Support cloudinit raw data feature: fix typo --- tests/unittests/test_datasource/test_ovf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index 971c83889f4..45481cc6402 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -139,7 +139,7 @@ def test_get_data_false_on_none_dmi_data(self): 'DEBUG: No system-product-name found', self.logs.getvalue()) def test_get_data_no_vmware_customization_disabled(self): - """When cloud-init workflow for vmware is disabled via sys_cfgi and + """When cloud-init workflow for vmware is disabled via sys_cfg and no meta data provided, log a message. """ paths = Paths({'cloud_dir': self.tdir}) From 59cac2da4fdb62939f9007fb35354d7d44f72701 Mon Sep 17 00:00:00 2001 From: Xiaofeng Wang Date: Wed, 13 Jan 2021 04:24:49 +0000 Subject: [PATCH 8/8] [VMware] Support cloudinit raw data feature: address comments --- cloudinit/sources/DataSourceOVF.py | 49 ++++++++++++++------- tests/unittests/test_datasource/test_ovf.py | 12 ++--- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index da8d832d76d..94d9f1b9a65 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -119,7 +119,9 @@ def _get_data(self): # When the VM is powered on, the "VMware Tools" daemon # copies the customization specification file to # /var/run/vmware-imc directory. cloud-init code needs - # to search for the file in that directory. + # to search for the file in that directory which indicates + # that required metadata and userdata files are now + # present. max_wait = get_max_wait_from_cfg(self.ds_cfg) vmwareImcConfigFilePath = util.log_time( logfunc=LOG.debug, @@ -137,9 +139,9 @@ def _get_data(self): LOG.debug("Found VMware Customization Config File at %s", vmwareImcConfigFilePath) try: - (md_path, ud_path, nicspath) = collect_imc_files( + (md_path, ud_path, nicspath) = collect_imc_file_paths( self._vmware_cust_conf) - except Exception as e: + except FileNotFoundError as e: _raise_error_status( "File(s) missing in directory", e, @@ -149,7 +151,7 @@ def _get_data(self): else: LOG.debug("Did not find VMware Customization Config File") - # ignore disable_vmware_customization if meta data is available + # Honor disable_vmware_customization setting on metadata absent if not md_path: if util.get_cfg_option_bool(self.sys_cfg, "disable_vmware_customization", @@ -160,12 +162,12 @@ def _get_data(self): # customization for VMware platform vmwareImcConfigFilePath = None - if vmwareImcConfigFilePath and md_path: + use_raw_data = bool(vmwareImcConfigFilePath and md_path) + if use_raw_data: set_gc_status(self._vmware_cust_conf, "Started") LOG.debug("Start to load cloud-init meta data and user data") try: - (md, ud, cfg, network) = load_cloudinit_data( - md_path, ud_path) + (md, ud, cfg, network) = load_cloudinit_data(md_path, ud_path) if network: self._network_config = network @@ -199,6 +201,7 @@ def _get_data(self): set_gc_status(self._vmware_cust_conf, "Successful") elif vmwareImcConfigFilePath: + # Load configuration from vmware_imc self._vmware_nics_to_enable = "" try: set_gc_status(self._vmware_cust_conf, "Started") @@ -745,26 +748,29 @@ def load_cloudinit_data(md_path, ud_path): """ Load the cloud-init meta data, user data, cfg and network from the given files + + @return: 4-tuple of configuration + metadata, userdata, cfg={}, network + + @raises: FileNotFoundError if md_path or ud_path are absent """ LOG.debug('load meta data from: %s: user data from: %s', md_path, ud_path) md = {} - cfg = {} ud = None network = None - md = load(util.load_file(md_path)) + md = safeload_yaml_or_dict(util.load_file(md_path)) if 'network' in md: network = md['network'] - del md['network'] if ud_path: ud = util.load_file(ud_path).replace("\r", "") - return md, ud, cfg, network + return md, ud, {}, network -def load(data): +def safeload_yaml_or_dict(data): ''' The meta data could be JSON or YAML. Since YAML is a strict superset of JSON, we will unmarshal the data as YAML. If data is None then a new @@ -775,10 +781,21 @@ def load(data): return safeyaml.load(data) -def collect_imc_files(cust_conf): +def collect_imc_file_paths(cust_conf): ''' - collect all the other imc files. metadata/userdata must be present - if they are specified in customization configuration. + collect all the other imc files. + + metadata is preferred to nics.txt configuration data. + + If metadata file exists because it is specified in customization + configuration, then metadata is required and userdata is optional. + + @return a 3-tuple containing desired configuration file paths if present + Expected returns: + 1. user provided metadata and userdata (md_path, ud_path, None) + 2. user provided metadata (md_path, None, None) + 3. user-provided network config (None, None, nics_path) + 4. No config found (None, None, None) ''' md_path = None ud_path = None @@ -799,7 +816,7 @@ def collect_imc_files(cust_conf): else: nics_path = os.path.join(VMWARE_IMC_DIR, "nics.txt") if not os.path.exists(nics_path): - LOG.debug('%s is not exist.', nics_path) + LOG.debug('%s does not exist.', nics_path) nics_path = None return md_path, ud_path, nics_path diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index 45481cc6402..dce01f5d2d9 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -402,7 +402,7 @@ def test_get_data_cloudinit_metadata_json(self): 'util.del_dir': True, 'search_file': self.tdir, 'wait_for_imc_cfg_file': conf_file, - 'collect_imc_files': [self.tdir + '/test-meta', '', ''], + 'collect_imc_file_paths': [self.tdir + '/test-meta', '', ''], 'get_nics_to_enable': ''}, ds._get_data) @@ -450,7 +450,7 @@ def test_get_data_cloudinit_metadata_yaml(self): 'util.del_dir': True, 'search_file': self.tdir, 'wait_for_imc_cfg_file': conf_file, - 'collect_imc_files': [self.tdir + '/test-meta', '', ''], + 'collect_imc_file_paths': [self.tdir + '/test-meta', '', ''], 'get_nics_to_enable': ''}, ds._get_data) @@ -490,7 +490,9 @@ def test_get_data_cloudinit_metadata_not_valid(self): 'util.del_dir': True, 'search_file': self.tdir, 'wait_for_imc_cfg_file': conf_file, - 'collect_imc_files': [self.tdir + '/test-meta', '', ''], + 'collect_imc_file_paths': [ + self.tdir + '/test-meta', '', '' + ], 'get_nics_to_enable': ''}, ds.get_data) @@ -572,8 +574,8 @@ def test_get_data_cloudinit_userdata(self): 'util.del_dir': True, 'search_file': self.tdir, 'wait_for_imc_cfg_file': conf_file, - 'collect_imc_files': [self.tdir + '/test-meta', - self.tdir + '/test-user', ''], + 'collect_imc_file_paths': [self.tdir + '/test-meta', + self.tdir + '/test-user', ''], 'get_nics_to_enable': ''}, ds._get_data)