From c4c45162f5028018e45f108b96516ce623fcd957 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Thu, 1 Jul 2021 17:01:02 -0500 Subject: [PATCH 1/3] Fix MIME policy failure on python version upgrade Python 3.6 added a new `policy` attribute to `MIMEMultipart`. MIMEMultipart may be part of the cached object pickle of a datasource. Upgrading from an old version of python to 3.6+ will cause the datasource to be invalid after pickle load. This commit uses the upgrade framework to attempt to access the mime message and fail early (thus discarding the cache) if we cannot. Commit 78e89b03 should fix this issue more generally. --- cloudinit/sources/__init__.py | 7 + .../assets/trusty_with_mime.pkl | 572 ++++++++++++++++++ .../modules/test_persistence.py | 30 + 3 files changed, 609 insertions(+) create mode 100644 tests/integration_tests/assets/trusty_with_mime.pkl create mode 100644 tests/integration_tests/modules/test_persistence.py diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index a07c4b4f98f..148807473b3 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -239,6 +239,13 @@ def _unpickle(self, ci_pkl_version: int) -> None: self.vendordata2 = None if not hasattr(self, 'vendordata2_raw'): self.vendordata2_raw = None + if hasattr(self, 'userdata') and self.userdata is not None: + # If userdata stores MIME data, on < python3.6 it will be + # missing the 'policy' attribute that exists on >=python3.6. + # Calling str() on the userdata will attempt to access this + # policy attribute. This will raise an exception, causing + # the pickle load to fail, so cloud-init will discard the cache + str(self.userdata) def __str__(self): return type_utils.obj_name(self) diff --git a/tests/integration_tests/assets/trusty_with_mime.pkl b/tests/integration_tests/assets/trusty_with_mime.pkl new file mode 100644 index 00000000000..a4089ecfe3c --- /dev/null +++ b/tests/integration_tests/assets/trusty_with_mime.pkl @@ -0,0 +1,572 @@ +ccopy_reg +_reconstructor +p1 +(ccloudinit.sources.DataSourceNoCloud +DataSourceNoCloudNet +p2 +c__builtin__ +object +p3 +NtRp4 +(dp5 +S'paths' +p6 +g1 +(ccloudinit.helpers +Paths +p7 +g3 +NtRp8 +(dp9 +S'lookups' +p10 +(dp11 +S'cloud_config' +p12 +S'cloud-config.txt' +p13 +sS'userdata' +p14 +S'user-data.txt.i' +p15 +sS'vendordata' +p16 +S'vendor-data.txt.i' +p17 +sS'userdata_raw' +p18 +S'user-data.txt' +p19 +sS'boothooks' +p20 +g20 +sS'scripts' +p21 +g21 +sS'sem' +p22 +g22 +sS'data' +p23 +g23 +sS'vendor_scripts' +p24 +S'scripts/vendor' +p25 +sS'handlers' +p26 +g26 +sS'obj_pkl' +p27 +S'obj.pkl' +p28 +sS'vendordata_raw' +p29 +S'vendor-data.txt' +p30 +sS'vendor_cloud_config' +p31 +S'vendor-cloud-config.txt' +p32 +ssS'template_tpl' +p33 +S'/etc/cloud/templates/%s.tmpl' +p34 +sS'cfgs' +p35 +(dp36 +S'cloud_dir' +p37 +S'/var/lib/cloud/' +p38 +sS'templates_dir' +p39 +S'/etc/cloud/templates/' +p40 +sS'upstart_dir' +p41 +S'/etc/init/' +p42 +ssS'cloud_dir' +p43 +g38 +sS'datasource' +p44 +NsS'upstart_conf_d' +p45 +g42 +sS'boot_finished' +p46 +S'/var/lib/cloud/instance/boot-finished' +p47 +sS'instance_link' +p48 +S'/var/lib/cloud/instance' +p49 +sS'seed_dir' +p50 +S'/var/lib/cloud/seed' +p51 +sbsS'supported_seed_starts' +p52 +(S'http://' +p53 +S'https://' +p54 +S'ftp://' +p55 +tp56 +sS'sys_cfg' +p57 +(dp58 +S'output' +p59 +(dp60 +S'all' +p61 +S'| tee -a /var/log/cloud-init-output.log' +p62 +ssS'users' +p63 +(lp64 +S'default' +p65 +asS'def_log_file' +p66 +S'/var/log/cloud-init.log' +p67 +sS'cloud_final_modules' +p68 +(lp69 +S'rightscale_userdata' +p70 +aS'scripts-vendor' +p71 +aS'scripts-per-once' +p72 +aS'scripts-per-boot' +p73 +aS'scripts-per-instance' +p74 +aS'scripts-user' +p75 +aS'ssh-authkey-fingerprints' +p76 +aS'keys-to-console' +p77 +aS'phone-home' +p78 +aS'final-message' +p79 +aS'power-state-change' +p80 +asS'disable_root' +p81 +I01 +sS'syslog_fix_perms' +p82 +S'syslog:adm' +p83 +sS'log_cfgs' +p84 +(lp85 +(lp86 +S'[loggers]\nkeys=root,cloudinit\n\n[handlers]\nkeys=consoleHandler,cloudLogHandler\n\n[formatters]\nkeys=simpleFormatter,arg0Formatter\n\n[logger_root]\nlevel=DEBUG\nhandlers=consoleHandler,cloudLogHandler\n\n[logger_cloudinit]\nlevel=DEBUG\nqualname=cloudinit\nhandlers=\npropagate=1\n\n[handler_consoleHandler]\nclass=StreamHandler\nlevel=WARNING\nformatter=arg0Formatter\nargs=(sys.stderr,)\n\n[formatter_arg0Formatter]\nformat=%(asctime)s - %(filename)s[%(levelname)s]: %(message)s\n\n[formatter_simpleFormatter]\nformat=[CLOUDINIT] %(filename)s[%(levelname)s]: %(message)s\n' +p87 +aS'[handler_cloudLogHandler]\nclass=handlers.SysLogHandler\nlevel=DEBUG\nformatter=simpleFormatter\nargs=("/dev/log", handlers.SysLogHandler.LOG_USER)\n' +p88 +aa(lp89 +g87 +aS"[handler_cloudLogHandler]\nclass=FileHandler\nlevel=DEBUG\nformatter=arg0Formatter\nargs=('/var/log/cloud-init.log',)\n" +p90 +aasS'cloud_init_modules' +p91 +(lp92 +S'migrator' +p93 +aS'seed_random' +p94 +aS'bootcmd' +p95 +aS'write-files' +p96 +aS'growpart' +p97 +aS'resizefs' +p98 +aS'set_hostname' +p99 +aS'update_hostname' +p100 +aS'update_etc_hosts' +p101 +aS'ca-certs' +p102 +aS'rsyslog' +p103 +aS'users-groups' +p104 +aS'ssh' +p105 +asS'preserve_hostname' +p106 +I00 +sS'_log' +p107 +(lp108 +g87 +ag90 +ag88 +asS'datasource_list' +p109 +(lp110 +S'NoCloud' +p111 +aS'ConfigDrive' +p112 +aS'OpenNebula' +p113 +aS'Azure' +p114 +aS'AltCloud' +p115 +aS'OVF' +p116 +aS'MAAS' +p117 +aS'GCE' +p118 +aS'OpenStack' +p119 +aS'CloudSigma' +p120 +aS'Ec2' +p121 +aS'CloudStack' +p122 +aS'SmartOS' +p123 +aS'None' +p124 +asS'vendor_data' +p125 +(dp126 +S'prefix' +p127 +(lp128 +sS'enabled' +p129 +I01 +ssS'cloud_config_modules' +p130 +(lp131 +S'emit_upstart' +p132 +aS'disk_setup' +p133 +aS'mounts' +p134 +aS'ssh-import-id' +p135 +aS'locale' +p136 +aS'set-passwords' +p137 +aS'grub-dpkg' +p138 +aS'apt-pipelining' +p139 +aS'apt-configure' +p140 +aS'package-update-upgrade-install' +p141 +aS'landscape' +p142 +aS'timezone' +p143 +aS'puppet' +p144 +aS'chef' +p145 +aS'salt-minion' +p146 +aS'mcollective' +p147 +aS'disable-ec2-metadata' +p148 +aS'runcmd' +p149 +aS'byobu' +p150 +assg14 +(iemail.mime.multipart +MIMEMultipart +p151 +(dp152 +S'_headers' +p153 +(lp154 +(S'Content-Type' +p155 +S'multipart/mixed; boundary="===============4291038100093149247=="' +tp156 +a(S'MIME-Version' +p157 +S'1.0' +p158 +tp159 +a(S'Number-Attachments' +p160 +S'1' +tp161 +asS'_payload' +p162 +(lp163 +(iemail.mime.base +MIMEBase +p164 +(dp165 +g153 +(lp166 +(g157 +g158 +tp167 +a(S'Content-Type' +p168 +S'text/x-not-multipart' +tp169 +a(S'Content-Disposition' +p170 +S'attachment; filename="part-001"' +tp171 +asg162 +S'' +sS'_charset' +p172 +NsS'_default_type' +p173 +S'text/plain' +p174 +sS'preamble' +p175 +NsS'defects' +p176 +(lp177 +sS'_unixfrom' +p178 +NsS'epilogue' +p179 +Nsbasg172 +Nsg173 +g174 +sg175 +Nsg176 +(lp180 +sg178 +Nsg179 +Nsbsg16 +S'#cloud-config\n{}\n\n' +p181 +sg18 +S'Content-Type: multipart/mixed; boundary="===============1378281702283945349=="\nMIME-Version: 1.0\n\n--===============1378281702283945349==\nContent-Type: text/x-shellscript; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nContent-Disposition: attachment; filename="script1.sh"\n\nIyEvYmluL3NoCgplY2hvICdoaScgPiAvdmFyL3RtcC9oaQo=\n\n--===============1378281702283945349==\nContent-Type: text/x-shellscript; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: base64\nContent-Disposition: attachment; filename="script2.sh"\n\nIyEvYmluL2Jhc2gKCmVjaG8gJ2hpMicgPiAvdmFyL3RtcC9oaTIK\n\n--===============1378281702283945349==--\n\n#cloud-config\n# final_message: |\n# This is my final message!\n# $version\n# $timestamp\n# $datasource\n# $uptime\n# updates:\n# network:\n# when: [\'hotplug\']\n' +p182 +sg29 +NsS'dsmode' +p183 +S'net' +p184 +sS'seed' +p185 +S'/var/lib/cloud/seed/nocloud-net' +p186 +sS'cmdline_id' +p187 +S'ds=nocloud-net' +p188 +sS'ud_proc' +p189 +g1 +(ccloudinit.user_data +UserDataProcessor +p190 +g3 +NtRp191 +(dp192 +g6 +g8 +sS'ssl_details' +p193 +(dp194 +sbsg50 +g186 +sS'ds_cfg' +p195 +(dp196 +sS'distro' +p197 +g1 +(ccloudinit.distros.ubuntu +Distro +p198 +g3 +NtRp199 +(dp200 +S'osfamily' +p201 +S'debian' +p202 +sS'_paths' +p203 +g8 +sS'name' +p204 +S'ubuntu' +p205 +sS'_runner' +p206 +g1 +(ccloudinit.helpers +Runners +p207 +g3 +NtRp208 +(dp209 +g6 +g8 +sS'sems' +p210 +(dp211 +sbsS'_cfg' +p212 +(dp213 +S'paths' +p214 +(dp215 +g37 +g38 +sg39 +g40 +sg41 +g42 +ssS'default_user' +p216 +(dp217 +S'shell' +p218 +S'/bin/bash' +p219 +sS'name' +p220 +S'ubuntu' +p221 +sS'sudo' +p222 +(lp223 +S'ALL=(ALL) NOPASSWD:ALL' +p224 +asS'lock_passwd' +p225 +I01 +sS'gecos' +p226 +S'Ubuntu' +p227 +sS'groups' +p228 +(lp229 +S'adm' +p230 +aS'audio' +p231 +aS'cdrom' +p232 +aS'dialout' +p233 +aS'dip' +p234 +aS'floppy' +p235 +aS'netdev' +p236 +aS'plugdev' +p237 +aS'sudo' +p238 +aS'video' +p239 +assS'package_mirrors' +p240 +(lp241 +(dp242 +S'arches' +p243 +(lp244 +S'i386' +p245 +aS'amd64' +p246 +asS'failsafe' +p247 +(dp248 +S'security' +p249 +S'http://security.ubuntu.com/ubuntu' +p250 +sS'primary' +p251 +S'http://archive.ubuntu.com/ubuntu' +p252 +ssS'search' +p253 +(dp254 +S'security' +p255 +(lp256 +sS'primary' +p257 +(lp258 +S'http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/' +p259 +aS'http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/' +p260 +aS'http://%(region)s.clouds.archive.ubuntu.com/ubuntu/' +p261 +assa(dp262 +S'arches' +p263 +(lp264 +S'armhf' +p265 +aS'armel' +p266 +aS'default' +p267 +asS'failsafe' +p268 +(dp269 +S'security' +p270 +S'http://ports.ubuntu.com/ubuntu-ports' +p271 +sS'primary' +p272 +S'http://ports.ubuntu.com/ubuntu-ports' +p273 +ssasS'ssh_svcname' +p274 +S'ssh' +p275 +ssbsS'metadata' +p276 +(dp277 +g183 +g184 +sS'local-hostname' +p278 +S'me' +p279 +sS'instance-id' +p280 +S'me' +p281 +ssb. \ No newline at end of file diff --git a/tests/integration_tests/modules/test_persistence.py b/tests/integration_tests/modules/test_persistence.py new file mode 100644 index 00000000000..9e46768ca74 --- /dev/null +++ b/tests/integration_tests/modules/test_persistence.py @@ -0,0 +1,30 @@ +# This file is part of cloud-init. See LICENSE file for license information. +"""Test the behavior of loading/discarding pickle data""" +from pathlib import Path + +import pytest + +from tests.integration_tests.instances import IntegrationInstance +from tests.integration_tests.util import ( + ASSETS_DIR, + verify_ordered_items_in_text, +) + + +PICKLE_PATH = Path('/var/lib/cloud/instance/obj.pkl') +TEST_PICKLE = ASSETS_DIR / 'trusty_with_mime.pkl' + + +@pytest.mark.lxd_container +def test_log_message_on_missing_version_file(client: IntegrationInstance): + client.push_file(TEST_PICKLE, PICKLE_PATH) + client.restart() + assert client.execute('cloud-init status --wait').ok + log = client.read_from_file('/var/log/cloud-init.log') + verify_ordered_items_in_text([ + 'Failed loading pickled blob from /var/lib/cloud/instance/obj.pkl', + "'MIMEMultipart' object has no attribute 'policy", + 'no cache found', + 'Searching for local data source', + 'SUCCESS: found local data from DataSourceNoCloud' + ], log) From 143682da39d65540e3823bfd2be836b76526f546 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Thu, 8 Jul 2021 16:01:50 -0500 Subject: [PATCH 2/3] Review comments --- cloudinit/sources/__init__.py | 13 ++++++++++++- cloudinit/stages.py | 7 ++++++- tests/integration_tests/modules/test_persistence.py | 4 ++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 148807473b3..13f3dadcb29 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -75,6 +75,10 @@ _NETCFG_SOURCE_NAMES)(*_NETCFG_SOURCE_NAMES) +class DatasourceUnpickleUserDataError(Exception): + """Raised when userdata is unable to be unpickled due to python upgrades""" + + class DataSourceNotFoundException(Exception): pass @@ -245,7 +249,14 @@ def _unpickle(self, ci_pkl_version: int) -> None: # Calling str() on the userdata will attempt to access this # policy attribute. This will raise an exception, causing # the pickle load to fail, so cloud-init will discard the cache - str(self.userdata) + try: + str(self.userdata) + except AttributeError as e: + LOG.debug( + "Unable to unpickle datasource: %s." + " Ignoring current cache.", e + ) + raise DatasourceUnpickleUserDataError() def __str__(self): return type_utils.obj_name(self) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 3688be2eacb..20b1ffbddb3 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -28,7 +28,10 @@ EventType, userdata_to_events, ) -from cloudinit.sources import NetworkConfigSource +from cloudinit.sources import ( + DatasourceUnpickleUserDataError, + NetworkConfigSource, +) from cloudinit import cloud from cloudinit import config @@ -1070,6 +1073,8 @@ def _pkl_load(fname): return None try: return pickle.loads(pickle_contents) + except sources.DatasourceUnpickleUserDataError: + return None except Exception: util.logexc(LOG, "Failed loading pickled blob from %s", fname) return None diff --git a/tests/integration_tests/modules/test_persistence.py b/tests/integration_tests/modules/test_persistence.py index 9e46768ca74..00fdeaeaea1 100644 --- a/tests/integration_tests/modules/test_persistence.py +++ b/tests/integration_tests/modules/test_persistence.py @@ -22,8 +22,8 @@ def test_log_message_on_missing_version_file(client: IntegrationInstance): assert client.execute('cloud-init status --wait').ok log = client.read_from_file('/var/log/cloud-init.log') verify_ordered_items_in_text([ - 'Failed loading pickled blob from /var/lib/cloud/instance/obj.pkl', - "'MIMEMultipart' object has no attribute 'policy", + "Unable to unpickle datasource: 'MIMEMultipart' object has no " + "attribute 'policy'. Ignoring current cache.", 'no cache found', 'Searching for local data source', 'SUCCESS: found local data from DataSourceNoCloud' From 2021f72e4558b6c5767b6cd905f37069a2f12d00 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Thu, 8 Jul 2021 16:30:35 -0500 Subject: [PATCH 3/3] flake8/pylint --- cloudinit/sources/__init__.py | 2 +- cloudinit/stages.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 13f3dadcb29..9d25b0eec59 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -256,7 +256,7 @@ def _unpickle(self, ci_pkl_version: int) -> None: "Unable to unpickle datasource: %s." " Ignoring current cache.", e ) - raise DatasourceUnpickleUserDataError() + raise DatasourceUnpickleUserDataError() from e def __str__(self): return type_utils.obj_name(self) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 20b1ffbddb3..06e0d9b1078 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -28,10 +28,7 @@ EventType, userdata_to_events, ) -from cloudinit.sources import ( - DatasourceUnpickleUserDataError, - NetworkConfigSource, -) +from cloudinit.sources import NetworkConfigSource from cloudinit import cloud from cloudinit import config