From 70b1fec0a5df2537bf404cff953c39de1cc3f04f Mon Sep 17 00:00:00 2001 From: Johann Queuniet Date: Tue, 8 Sep 2020 22:42:13 +0200 Subject: [PATCH 1/4] Add vendor-data support to seed parameter for NoCloud and OVF --- cloudinit/sources/DataSourceNoCloud.py | 3 ++- cloudinit/sources/DataSourceOVF.py | 4 +++- cloudinit/util.py | 13 +++++++++++-- tests/unittests/test_util.py | 7 +++++-- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index e408d7303ca..d4a175e82c5 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -157,13 +157,14 @@ def _pp2d_callback(mp, data): # This could throw errors, but the user told us to do it # so if errors are raised, let them raise - (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) + (md_seed, ud, vd) = util.read_seeded(seedfrom, timeout=None) LOG.debug("Using seeded cache data from %s", seedfrom) # Values in the command line override those from the seed mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], md_seed]) mydata['user-data'] = ud + mydata['vendor-data'] = vd found.append(seedfrom) # Now that we have exhausted any other places merge in the defaults diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index e53d2eb142f..a5ccb8f6849 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -73,6 +73,7 @@ def _get_data(self): found = [] md = {} ud = "" + vd = "" vmwareImcConfigFilePath = None nicspath = None @@ -304,7 +305,7 @@ def _get_data(self): seedfrom, self) return False - (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) + (md_seed, ud, vd) = util.read_seeded(seedfrom, timeout=None) LOG.debug("Using seeded cache data from %s", seedfrom) md = util.mergemanydict([md, md_seed]) @@ -316,6 +317,7 @@ def _get_data(self): self.seed = ",".join(found) self.metadata = md self.userdata_raw = ud + self.vendordata_raw = vd self.cfg = cfg return True diff --git a/cloudinit/util.py b/cloudinit/util.py index 64142f23dac..60a810b1ebb 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -761,8 +761,9 @@ def del_dir(path): # 'meta-data' entries def read_optional_seed(fill, base="", ext="", timeout=5): try: - (md, ud) = read_seeded(base, ext, timeout) + (md, ud, vd) = read_seeded(base, ext, timeout) fill['user-data'] = ud + fill['vendor-data'] = vd fill['meta-data'] = md return True except url_helper.UrlError as e: @@ -840,9 +841,11 @@ def load_yaml(blob, default=None, allowed=(dict,)): def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): if base.find("%s") >= 0: ud_url = base % ("user-data" + ext) + vd_url = base % ("vendor-data" + ext) md_url = base % ("meta-data" + ext) else: ud_url = "%s%s%s" % (base, "user-data", ext) + vd_url = "%s%s%s" % (base, "vendor-data", ext) md_url = "%s%s%s" % (base, "meta-data", ext) md_resp = url_helper.read_file_or_url(md_url, timeout=timeout, @@ -857,7 +860,13 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): if ud_resp.ok(): ud = ud_resp.contents - return (md, ud) + vd_resp = url_helper.read_file_or_url(vd_url, timeout=timeout, + retries=retries) + vd = None + if vd_resp.ok(): + vd = vd_resp.contents + + return (md, ud, vd) def read_conf_d(confd): diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index fc5574698cd..643ef7e6028 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -735,13 +735,16 @@ def setUp(self): def test_unicode_not_messed_up(self): ud = b"userdatablob" + vd = b"vendordatablob" helpers.populate_dir( - self.tmp, {'meta-data': "key1: val1", 'user-data': ud}) + self.tmp, {'meta-data': "key1: val1", 'user-data': ud, + 'vendor-data': vd}) sdir = self.tmp + os.path.sep - (found_md, found_ud) = util.read_seeded(sdir) + (found_md, found_ud, found_vd) = util.read_seeded(sdir) self.assertEqual(found_md, {'key1': 'val1'}) self.assertEqual(found_ud, ud) + self.assertEqual(found_vd, vd) class TestEncode(helpers.TestCase): From 885e08200fe495bbad4e0ac97cb5dfbdfd68b733 Mon Sep 17 00:00:00 2001 From: Johann Queuniet Date: Tue, 8 Sep 2020 23:05:14 +0200 Subject: [PATCH 2/4] Handle failure with vendor data --- cloudinit/util.py | 12 ++++++++---- tests/unittests/test_util.py | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 60a810b1ebb..f0c0300ec67 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -860,11 +860,15 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): if ud_resp.ok(): ud = ud_resp.contents - vd_resp = url_helper.read_file_or_url(vd_url, timeout=timeout, - retries=retries) vd = None - if vd_resp.ok(): - vd = vd_resp.contents + try: + vd_resp = url_helper.read_file_or_url(vd_url, timeout=timeout, + retries=retries) + except url_helper.UrlError: + pass + else: + if vd_resp.ok(): + vd = vd_resp.contents return (md, ud, vd) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 643ef7e6028..2a6409edb27 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -747,6 +747,25 @@ def test_unicode_not_messed_up(self): self.assertEqual(found_vd, vd) +class TestReadSeededWithoutVendorData(helpers.TestCase): + def setUp(self): + super(TestReadSeededWithoutVendorData, self).setUp() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + + def test_unicode_not_messed_up(self): + ud = b"userdatablob" + vd = None + helpers.populate_dir( + self.tmp, {'meta-data': "key1: val1", 'user-data': ud}) + sdir = self.tmp + os.path.sep + (found_md, found_ud, found_vd) = util.read_seeded(sdir) + + self.assertEqual(found_md, {'key1': 'val1'}) + self.assertEqual(found_ud, ud) + self.assertEqual(found_vd, vd) + + class TestEncode(helpers.TestCase): """Test the encoding functions""" def test_decode_binary_plain_text_with_hex(self): From 23482b91b3ef9dd87f32b1b305824002dd61868d Mon Sep 17 00:00:00 2001 From: Johann Queuniet Date: Tue, 8 Sep 2020 23:23:51 +0200 Subject: [PATCH 3/4] Fix flake8 test --- cloudinit/util.py | 2 +- tests/unittests/test_util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index f0c0300ec67..f3ea186c4a4 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -863,7 +863,7 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): vd = None try: vd_resp = url_helper.read_file_or_url(vd_url, timeout=timeout, - retries=retries) + retries=retries) except url_helper.UrlError: pass else: diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 2a6409edb27..cca53123e75 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -738,7 +738,7 @@ def test_unicode_not_messed_up(self): vd = b"vendordatablob" helpers.populate_dir( self.tmp, {'meta-data': "key1: val1", 'user-data': ud, - 'vendor-data': vd}) + 'vendor-data': vd}) sdir = self.tmp + os.path.sep (found_md, found_ud, found_vd) = util.read_seeded(sdir) From a5fd2440750eb3e27fbdb150ac9e5d6e1d0985dd Mon Sep 17 00:00:00 2001 From: Johann Queuniet Date: Sat, 12 Sep 2020 11:04:27 +0200 Subject: [PATCH 4/4] Add error logging --- cloudinit/util.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index f3ea186c4a4..e47f1cf60dd 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -864,11 +864,13 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): try: vd_resp = url_helper.read_file_or_url(vd_url, timeout=timeout, retries=retries) - except url_helper.UrlError: - pass + except url_helper.UrlError as e: + LOG.debug("Error in vendor-data response: %s", e) else: if vd_resp.ok(): vd = vd_resp.contents + else: + LOG.debug("Error in vendor-data response") return (md, ud, vd)