From 2745b8268fee201f464c96ffcb0088fabdb306ce Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Mon, 30 Mar 2020 18:45:39 +0200 Subject: [PATCH 1/4] Test acceptance of x-shellscript as cloud-config. Test case from Ryan Harper in discussion of PR#234 (https://github.com/canonical/cloud-init/pull/234). This should demonstrate that cloud-init (without the subsequent patches) will fail to accept multipart user-data documents with the cloud-config piece (mis)declared as text/x-shellscript. This happens on old heat on OpenTelekomCloud. Let's watch this fail the CI and then add the fix ... Signed-off-by: Kurt Garloff --- tests/unittests/test_data.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 74cc26ec1a8..9a9514e4e29 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -57,6 +57,40 @@ def gzip_text(text): f.close() return contents.getvalue() + def test_cloud_config_as_x_shell_script(self): + blob_cc = ''' +#cloud-config +a: b +c: d +''' + message_cc = MIMEBase("text", "x-shellscript") + message_cc.set_payload(blob_cc) + + blob_jp = ''' +#cloud-config-jsonp +[ + { "op": "replace", "path": "/a", "value": "c" }, + { "op": "remove", "path": "/c" } +] +''' + + message_jp = MIMEBase('text', "cloud-config-jsonp") + message_jp.set_payload(blob_jp) + + message = MIMEMultipart() + message.attach(message_cc) + message.attach(message_jp) + + self.reRoot() + ci = stages.Init() + ci.datasource = FakeDataSource(str(message)) + ci.fetch() + ci.consume_data() + cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) + cc = util.load_yaml(cc_contents) + self.assertEqual(1, len(cc)) + self.assertEqual('c', cc['a']) + # FIXME: these tests shouldn't be checking log output?? # Weirddddd... From ad597bdbd01533a9104547140c546a619e51c605 Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Fri, 27 Mar 2020 13:28:20 +0000 Subject: [PATCH 2/4] Fix cloud-init ignoring some mimetypes in user-data. This patch if from Ryan Harper (inspired by my bug report and a suggestion from Scott Moser) from the discussion on PR #234. cloud-init would not accept unexpected mime-types in multipart user-data for the cloud-config piece before; now it does. Problem was observed and reproduced with old heat on OTC and is now fixed. Signed-off-by: Kurt Garloff --- cloudinit/user_data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 6f41b03a19b..670dbee6960 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -25,6 +25,7 @@ NOT_MULTIPART_TYPE = handlers.NOT_MULTIPART_TYPE PART_FN_TPL = handlers.PART_FN_TPL OCTET_TYPE = handlers.OCTET_TYPE +INCLUDE_MAP = handlers.INCLUSION_TYPES_MAP # Saves typing errors CONTENT_TYPE = 'Content-Type' @@ -115,7 +116,8 @@ def find_ctype(payload): # Attempt to figure out the payloads content-type if not ctype_orig: ctype_orig = UNDEF_TYPE - if ctype_orig in TYPE_NEEDED: + if ctype_orig in TYPE_NEEDED or (ctype_orig in + INCLUDE_MAP.values()): ctype = find_ctype(payload) if ctype is None: ctype = ctype_orig From b8aad5928cad030f516f5c7b4c02831882c8a99d Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Mon, 30 Mar 2020 21:09:22 +0200 Subject: [PATCH 3/4] Fix mismerge. --- tests/unittests/test_data.py | 70 ++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 9a9514e4e29..8274e938eb9 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -57,41 +57,6 @@ def gzip_text(text): f.close() return contents.getvalue() - def test_cloud_config_as_x_shell_script(self): - blob_cc = ''' -#cloud-config -a: b -c: d -''' - message_cc = MIMEBase("text", "x-shellscript") - message_cc.set_payload(blob_cc) - - blob_jp = ''' -#cloud-config-jsonp -[ - { "op": "replace", "path": "/a", "value": "c" }, - { "op": "remove", "path": "/c" } -] -''' - - message_jp = MIMEBase('text', "cloud-config-jsonp") - message_jp.set_payload(blob_jp) - - message = MIMEMultipart() - message.attach(message_cc) - message.attach(message_jp) - - self.reRoot() - ci = stages.Init() - ci.datasource = FakeDataSource(str(message)) - ci.fetch() - ci.consume_data() - cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) - cc = util.load_yaml(cc_contents) - self.assertEqual(1, len(cc)) - self.assertEqual('c', cc['a']) - - # FIXME: these tests shouldn't be checking log output?? # Weirddddd... class TestConsumeUserData(helpers.FilesystemMockingTestCase): @@ -247,6 +212,41 @@ def test_mixed_cloud_config(self): self.assertEqual(1, len(cc)) self.assertEqual('c', cc['a']) + def test_cloud_config_as_x_shell_script(self): + blob_cc = ''' +#cloud-config +a: b +c: d +''' + message_cc = MIMEBase("text", "x-shellscript") + message_cc.set_payload(blob_cc) + + blob_jp = ''' +#cloud-config-jsonp +[ + { "op": "replace", "path": "/a", "value": "c" }, + { "op": "remove", "path": "/c" } +] +''' + + message_jp = MIMEBase('text', "cloud-config-jsonp") + message_jp.set_payload(blob_jp) + + message = MIMEMultipart() + message.attach(message_cc) + message.attach(message_jp) + + self.reRoot() + ci = stages.Init() + ci.datasource = FakeDataSource(str(message)) + ci.fetch() + ci.consume_data() + cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) + cc = util.load_yaml(cc_contents) + self.assertEqual(1, len(cc)) + self.assertEqual('c', cc['a']) + + def test_vendor_user_yaml_cloud_config(self): vendor_blob = ''' #cloud-config From 511b00421ffcf1a5e39a7f1450d07cae042cccdd Mon Sep 17 00:00:00 2001 From: Kurt Garloff Date: Tue, 31 Mar 2020 00:51:10 +0200 Subject: [PATCH 4/4] Make pycodestyle happy. --- tests/unittests/test_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 8274e938eb9..a42616092c3 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -57,6 +57,7 @@ def gzip_text(text): f.close() return contents.getvalue() + # FIXME: these tests shouldn't be checking log output?? # Weirddddd... class TestConsumeUserData(helpers.FilesystemMockingTestCase): @@ -246,7 +247,6 @@ def test_cloud_config_as_x_shell_script(self): self.assertEqual(1, len(cc)) self.assertEqual('c', cc['a']) - def test_vendor_user_yaml_cloud_config(self): vendor_blob = ''' #cloud-config