From 121539b0f7fd70eafc30c3b16f15a5447fa956ba Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 28 Jun 2024 13:17:34 -0600 Subject: [PATCH 01/33] fix(schema): Don't report changed keys as deprecated (#5464) --- cloudinit/config/schema.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index cf7fd10763e..292c1efe664 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -362,7 +362,7 @@ def _validator( ): """Jsonschema validator for `deprecated` items. - It raises a instance of `error_type` if deprecated that must be handled, + It yields an instance of `error_type` if deprecated that must be handled, otherwise the instance is consider faulty. """ if deprecated: @@ -797,8 +797,9 @@ def validate_cloudconfig_schema( ): # pylint: disable=W1116 if ( "devel" != features.DEPRECATION_INFO_BOUNDARY - and Version.from_str(schema_error.version) - > Version.from_str(features.DEPRECATION_INFO_BOUNDARY) + and (schema_error.version == "devel" + or Version.from_str(schema_error.version) + > Version.from_str(features.DEPRECATION_INFO_BOUNDARY)) ): info_deprecations.append( SchemaProblem(path, schema_error.message) From 726159b9a4b7e4446817b7249171d906678ca278 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 28 Jun 2024 13:19:30 -0600 Subject: [PATCH 02/33] fix(test): Mock version boundary (#5464) --- tests/unittests/config/test_schema.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index e9e8aa64ce1..2ea42b62d5b 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -815,12 +815,13 @@ def test_validateconfig_strict_metaschema_do_not_raise_exception( def test_validateconfig_logs_deprecations( self, schema, config, expected_msg, log_deprecations, caplog ): - validate_cloudconfig_schema( - config, - schema=schema, - strict_metaschema=True, - log_deprecations=log_deprecations, - ) + with mock.patch.object(features, "DEPRECATION_INFO_BOUNDARY", "devel"): + validate_cloudconfig_schema( + config, + schema=schema, + strict_metaschema=True, + log_deprecations=log_deprecations, + ) if expected_msg is None: return log_record = (M_PATH[:-1], DEPRECATED_LOG_LEVEL, expected_msg) From 25669f736f015bb2e7e7a9e790ca310a7088f006 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Fri, 28 Jun 2024 15:55:23 -0500 Subject: [PATCH 03/33] fix: dont double-log deprecated INFOs (#5465) --- cloudinit/util.py | 1 - tests/unittests/test_log.py | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index f42e641440b..8cac7dc8b2d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -3255,7 +3255,6 @@ def deprecate( "devel" != features.DEPRECATION_INFO_BOUNDARY and Version.from_str(features.DEPRECATION_INFO_BOUNDARY) < version ): - LOG.info(deprecate_msg) level = logging.INFO elif hasattr(LOG, "deprecated"): level = log.DEPRECATED diff --git a/tests/unittests/test_log.py b/tests/unittests/test_log.py index e68dcc48029..78b53f5a9f9 100644 --- a/tests/unittests/test_log.py +++ b/tests/unittests/test_log.py @@ -84,7 +84,12 @@ def test_deprecated_log_level(self, caplog): ), ) def test_deprecate_log_level_based_on_features( - self, expected_log_level, deprecation_info_boundary, caplog, mocker + self, + expected_log_level, + deprecation_info_boundary, + caplog, + mocker, + clear_deprecation_log, ): """Deprecation log level depends on key deprecation_version From 5ce2ee36e243b4cc178fef28bc0da06212ac7607 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Fri, 28 Jun 2024 16:02:05 -0500 Subject: [PATCH 04/33] chore: fix schema.py formatting (#5465) --- cloudinit/config/schema.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 292c1efe664..c7c23731aac 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -795,11 +795,10 @@ def validate_cloudconfig_schema( if isinstance( schema_error, SchemaDeprecationError ): # pylint: disable=W1116 - if ( - "devel" != features.DEPRECATION_INFO_BOUNDARY - and (schema_error.version == "devel" + if "devel" != features.DEPRECATION_INFO_BOUNDARY and ( + schema_error.version == "devel" or Version.from_str(schema_error.version) - > Version.from_str(features.DEPRECATION_INFO_BOUNDARY)) + > Version.from_str(features.DEPRECATION_INFO_BOUNDARY) ): info_deprecations.append( SchemaProblem(path, schema_error.message) From 0a698a57f920cb50199cfea40a9aa994965252c0 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Fri, 28 Jun 2024 18:55:08 -0500 Subject: [PATCH 05/33] test: Fix deprecation test failures (#5466) 8906e17e is causing some test failures on older releases. Fix them --- cloudinit/config/schema.py | 14 ++++++-------- cloudinit/util.py | 17 +++++++++++++---- tests/unittests/distros/test_create_users.py | 15 +++++++++++++-- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index c7c23731aac..c72c4ed5f1c 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -31,18 +31,18 @@ import yaml -from cloudinit import features, importer, safeyaml +from cloudinit import importer, safeyaml from cloudinit.cmd.devel import read_cfg_paths from cloudinit.handlers import INCLUSION_TYPES_MAP, type_from_starts_with from cloudinit.helpers import Paths from cloudinit.sources import DataSourceNotFoundException from cloudinit.temp_utils import mkdtemp from cloudinit.util import ( - Version, error, get_modules_from_dir, load_text_file, load_yaml, + should_log_deprecation, write_file, ) @@ -795,16 +795,14 @@ def validate_cloudconfig_schema( if isinstance( schema_error, SchemaDeprecationError ): # pylint: disable=W1116 - if "devel" != features.DEPRECATION_INFO_BOUNDARY and ( - schema_error.version == "devel" - or Version.from_str(schema_error.version) - > Version.from_str(features.DEPRECATION_INFO_BOUNDARY) + if schema_error.version == "devel" or should_log_deprecation( + schema_error.version ): + deprecations.append(SchemaProblem(path, schema_error.message)) + else: info_deprecations.append( SchemaProblem(path, schema_error.message) ) - else: - deprecations.append(SchemaProblem(path, schema_error.message)) else: errors.append(SchemaProblem(path, schema_error.message)) diff --git a/cloudinit/util.py b/cloudinit/util.py index 8cac7dc8b2d..4380cce3535 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -3210,6 +3210,18 @@ def _compare_version(self, other: "Version") -> int: return -1 +def should_log_deprecation(version: str) -> bool: + """Determine if a deprecation message should be logged. + + :param version: The version in which the thing was deprecated. + + :return: True if the message should be logged, else False. + """ + return features.DEPRECATION_INFO_BOUNDARY == "devel" or Version.from_str( + version + ) <= Version.from_str(features.DEPRECATION_INFO_BOUNDARY) + + def deprecate( *, deprecated: str, @@ -3251,10 +3263,7 @@ def deprecate( f"{deprecated_version} and scheduled to be removed in " f"{version_removed}. {message}" ).rstrip() - if ( - "devel" != features.DEPRECATION_INFO_BOUNDARY - and Version.from_str(features.DEPRECATION_INFO_BOUNDARY) < version - ): + if not should_log_deprecation(deprecated_version): level = logging.INFO elif hasattr(LOG, "deprecated"): level = log.DEPRECATED diff --git a/tests/unittests/distros/test_create_users.py b/tests/unittests/distros/test_create_users.py index 039723aaad2..86c79fb1775 100644 --- a/tests/unittests/distros/test_create_users.py +++ b/tests/unittests/distros/test_create_users.py @@ -5,6 +5,7 @@ import pytest from cloudinit import distros, ssh_util +from cloudinit.util import should_log_deprecation from tests.unittests.helpers import mock from tests.unittests.util import abstract_to_concrete @@ -142,7 +143,12 @@ def test_create_groups_with_dict_deprecated( ] assert m_subp.call_args_list == expected - assert caplog.records[0].levelname in ["WARNING", "DEPRECATED"] + expected_levels = ( + ["WARNING", "DEPRECATED"] + if should_log_deprecation("23.1") + else ["INFO"] + ) + assert caplog.records[0].levelname in expected_levels assert ( "The user foo_user has a 'groups' config value of type dict" in caplog.records[0].message @@ -170,7 +176,12 @@ def test_explicit_sudo_false(self, m_subp, dist, caplog): mock.call(["passwd", "-l", USER]), ] - assert caplog.records[1].levelname in ["WARNING", "DEPRECATED"] + expected_levels = ( + ["WARNING", "DEPRECATED"] + if should_log_deprecation("22.3") + else ["INFO"] + ) + assert caplog.records[1].levelname in expected_levels assert ( "The value of 'false' in user foo_user's 'sudo' " "config is deprecated in 22.3 and scheduled to be removed" From 7693ee2a6846ab8953b37eee36d275898eb73001 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 1 Jul 2024 06:58:28 -0600 Subject: [PATCH 06/33] tests: update keyserver PPA key fur curtin-dev (#5472) Commit 8470af00 changes the integration test keyserver PPA to ppa:curtin-dev/daily. Update the assertion key fingerprint. --- tests/integration_tests/modules/test_apt_functionality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/modules/test_apt_functionality.py b/tests/integration_tests/modules/test_apt_functionality.py index 1b49722fd03..b86e217556e 100644 --- a/tests/integration_tests/modules/test_apt_functionality.py +++ b/tests/integration_tests/modules/test_apt_functionality.py @@ -124,7 +124,7 @@ r"deb-src http://badsecurity.ubuntu.com/ubuntu [a-z]+-security multiverse", ] -TEST_KEYSERVER_KEY = "110E 21D8 B0E2 A1F0 243A F682 0856 F197 B892 ACEA" +TEST_KEYSERVER_KEY = "1BC3 0F71 5A3B 8612 47A8 1A5E 55FE 7C8C 0165 013E" TEST_PPA_KEY = "3552 C902 B4DD F7BD 3842 1821 015D 28D7 4416 14D8" TEST_KEY = "1FF0 D853 5EF7 E719 E5C8 1B9C 083D 06FB E4D3 04DF" TEST_SIGNED_BY_KEY = "A2EB 2DEC 0BD7 519B 7B38 BE38 376A 290E C806 8B11" From d59724a1516c6d434063a5677bd3ea67257f83a6 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 1 Jul 2024 07:07:05 -0600 Subject: [PATCH 07/33] tests: integration tests aware of features.DEPRECATION_INFO_BOUNDARY Adapt util.should_log_deprecation to expect two params: - version: where the key/feature was deprecated - boundary_version: at which deprecation level logs occur --- cloudinit/config/schema.py | 4 +- cloudinit/util.py | 11 +++-- tests/integration_tests/cmd/test_schema.py | 42 +++++++++++-------- tests/integration_tests/cmd/test_status.py | 39 ++++++++++++----- .../modules/test_combined.py | 18 ++++++-- tests/unittests/distros/test_create_users.py | 10 +++-- 6 files changed, 83 insertions(+), 41 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index c72c4ed5f1c..e59fde4ee92 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -31,7 +31,7 @@ import yaml -from cloudinit import importer, safeyaml +from cloudinit import features, importer, safeyaml from cloudinit.cmd.devel import read_cfg_paths from cloudinit.handlers import INCLUSION_TYPES_MAP, type_from_starts_with from cloudinit.helpers import Paths @@ -796,7 +796,7 @@ def validate_cloudconfig_schema( schema_error, SchemaDeprecationError ): # pylint: disable=W1116 if schema_error.version == "devel" or should_log_deprecation( - schema_error.version + schema_error.version, features.DEPRECATION_INFO_BOUNDARY ): deprecations.append(SchemaProblem(path, schema_error.message)) else: diff --git a/cloudinit/util.py b/cloudinit/util.py index 4380cce3535..5b8f07683f6 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -3210,16 +3210,17 @@ def _compare_version(self, other: "Version") -> int: return -1 -def should_log_deprecation(version: str) -> bool: +def should_log_deprecation(version: str, boundary_version: str) -> bool: """Determine if a deprecation message should be logged. :param version: The version in which the thing was deprecated. + :param boundary_version: The version at which deprecation level is logged. :return: True if the message should be logged, else False. """ - return features.DEPRECATION_INFO_BOUNDARY == "devel" or Version.from_str( + return boundary_version == "devel" or Version.from_str( version - ) <= Version.from_str(features.DEPRECATION_INFO_BOUNDARY) + ) <= Version.from_str(boundary_version) def deprecate( @@ -3263,7 +3264,9 @@ def deprecate( f"{deprecated_version} and scheduled to be removed in " f"{version_removed}. {message}" ).rstrip() - if not should_log_deprecation(deprecated_version): + if not should_log_deprecation( + deprecated_version, features.DEPRECATION_INFO_BOUNDARY + ): level = logging.INFO elif hasattr(LOG, "deprecated"): level = log.DEPRECATED diff --git a/tests/integration_tests/cmd/test_schema.py b/tests/integration_tests/cmd/test_schema.py index 70eb0a67c6f..3155a07919b 100644 --- a/tests/integration_tests/cmd/test_schema.py +++ b/tests/integration_tests/cmd/test_schema.py @@ -3,9 +3,13 @@ import pytest +from cloudinit.util import should_log_deprecation from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.releases import CURRENT_RELEASE, MANTIC -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import ( + get_feature_flag_value, + verify_clean_log, +) USER_DATA = """\ #cloud-config @@ -62,10 +66,19 @@ class TestSchemaDeprecations: def test_clean_log(self, class_client: IntegrationInstance): log = class_client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log, ignore_deprecations=True) - assert "DEPRECATED]: Deprecated cloud-config provided:" in log - assert "apt_reboot_if_required: Default: ``false``. Deprecated " in log - assert "apt_update: Default: ``false``. Deprecated in version" in log - assert "apt_upgrade: Default: ``false``. Deprecated in version" in log + version_boundary = get_feature_flag_value( + class_client, "DEPRECATION_INFO_BOUNDARY" + ) + # the deprecation_version is 22.2 in schema for apt_* keys in + # user-data. Pass 22.2 in against the client's version_boundary. + if should_log_deprecation("22.2", version_boundary): + log_level = "DEPRECATED" + else: + log_level = "INFO" + assert f"{log_level}]: Deprecated cloud-config provided:" in log + assert "apt_reboot_if_required: Deprecated " in log + assert "apt_update: Deprecated in version" in log + assert "apt_upgrade: Deprecated in version" in log def test_network_config_schema_validation( self, class_client: IntegrationInstance @@ -139,17 +152,10 @@ def test_schema_deprecations(self, class_client: IntegrationInstance): ), "`schema` cmd must return 0 even with deprecated configs" assert not result.stderr assert "Cloud config schema deprecations:" in result.stdout + assert "apt_update: Deprecated in version" in result.stdout + assert "apt_upgrade: Deprecated in version" in result.stdout assert ( - "apt_update: Default: ``false``. Deprecated in version" - in result.stdout - ) - assert ( - "apt_upgrade: Default: ``false``. Deprecated in version" - in result.stdout - ) - assert ( - "apt_reboot_if_required: Default: ``false``. Deprecated in version" - in result.stdout + "apt_reboot_if_required: Deprecated in version" in result.stdout ) annotated_result = class_client.execute( @@ -167,9 +173,9 @@ def test_schema_deprecations(self, class_client: IntegrationInstance): apt_reboot_if_required: false\t\t# D3 # Deprecations: ------------- - # D1: Default: ``false``. Deprecated in version 22.2. Use ``package_update`` instead. - # D2: Default: ``false``. Deprecated in version 22.2. Use ``package_upgrade`` instead. - # D3: Default: ``false``. Deprecated in version 22.2. Use ``package_reboot_if_required`` instead. + # D1: Deprecated in version 22.2. Use ``package_update`` instead. + # D2: Deprecated in version 22.2. Use ``package_upgrade`` instead. + # D3: Deprecated in version 22.2. Use ``package_reboot_if_required`` instead. Valid schema /root/user-data""" # noqa: E501 diff --git a/tests/integration_tests/cmd/test_status.py b/tests/integration_tests/cmd/test_status.py index 4eda01e82fb..20c78433257 100644 --- a/tests/integration_tests/cmd/test_status.py +++ b/tests/integration_tests/cmd/test_status.py @@ -3,12 +3,14 @@ import pytest +from cloudinit.util import should_log_deprecation from tests.integration_tests.clouds import IntegrationCloud from tests.integration_tests.decorators import retry from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU, JAMMY from tests.integration_tests.util import ( + get_feature_flag_value, push_and_enable_systemd_unit, wait_for_cloud_init, ) @@ -79,17 +81,32 @@ def test_status_json_errors(client): "DEPRECATED" ) - status_json = client.execute("cloud-init status --format json").stdout - assert "Deprecated cloud-config provided:\nca-certs:" in json.loads( - status_json - )["init"]["recoverable_errors"].get("DEPRECATED").pop(0) - assert "Deprecated cloud-config provided:\nca-certs:" in json.loads( - status_json - )["recoverable_errors"].get("DEPRECATED").pop(0) - assert "cloud-config failed schema validation" in json.loads(status_json)[ - "init" - ]["recoverable_errors"].get("WARNING").pop(0) - assert "cloud-config failed schema validation" in json.loads(status_json)[ + status_json = json.loads( + client.execute("cloud-init status --format json").stdout + ) + version_boundary = get_feature_flag_value( + client, "DEPRECATION_INFO_BOUNDARY" + ) + # The deprecation_version is 22.3 in schema for ca-certs. + # Expect an extra deprecation level log in status if boundary > 22.3 + if should_log_deprecation("22.3", version_boundary): + assert "Deprecated cloud-config provided: ca-certs" in status_json[ + "init" + ]["recoverable_errors"].get("DEPRECATED").pop(0) + assert "Deprecated cloud-config provided: ca-certs" in status_json[ + "recoverable_errors" + ].get("DEPRECATED").pop(0) + + assert "Key 'ca-certs' is deprecated in 22.1" in status_json["init"][ + "recoverable_errors" + ].get("DEPRECATED").pop(0) + assert "Key 'ca-certs' is deprecated in 22.1" in status_json[ + "recoverable_errors" + ].get("DEPRECATED").pop(0) + assert "cloud-config failed schema validation" in status_json["init"][ + "recoverable_errors" + ].get("WARNING").pop(0) + assert "cloud-config failed schema validation" in status_json[ "recoverable_errors" ].get("WARNING").pop(0) diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 8a73e7e8623..4a7c2dfc9e0 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -15,7 +15,7 @@ import pytest import cloudinit.config -from cloudinit.util import is_true +from cloudinit.util import is_true, should_log_deprecation from tests.integration_tests.decorators import retry from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM @@ -131,8 +131,20 @@ def test_deprecated_message(self, class_client: IntegrationInstance): """Check that deprecated key produces a log warning""" client = class_client log = client.read_from_file("/var/log/cloud-init.log") - assert "Deprecated cloud-config provided" in log - assert "The value of 'false' in user craig's 'sudo' config is " in log + version_boundary = get_feature_flag_value( + class_client, "DEPRECATION_INFO_BOUNDARY" + ) + # the deprecation_version is 22.2 in schema for apt_* keys in + # user-data. Pass 22.2 in against the client's version_boundary. + if should_log_deprecation("22.2", version_boundary): + log_level = "DEPRECATED" + else: + log_level = "INFO" + + assert ( + f"[{log_level}]: The value of 'false' in user craig's 'sudo'" + " config is deprecated" in log + ) assert 2 == log.count("DEPRECATE") def test_ntp_with_apt(self, class_client: IntegrationInstance): diff --git a/tests/unittests/distros/test_create_users.py b/tests/unittests/distros/test_create_users.py index 86c79fb1775..893b47060d4 100644 --- a/tests/unittests/distros/test_create_users.py +++ b/tests/unittests/distros/test_create_users.py @@ -4,7 +4,7 @@ import pytest -from cloudinit import distros, ssh_util +from cloudinit import distros, features, ssh_util from cloudinit.util import should_log_deprecation from tests.unittests.helpers import mock from tests.unittests.util import abstract_to_concrete @@ -145,7 +145,9 @@ def test_create_groups_with_dict_deprecated( expected_levels = ( ["WARNING", "DEPRECATED"] - if should_log_deprecation("23.1") + if should_log_deprecation( + "23.1", features.DEPRECATION_INFO_BOUNDARY + ) else ["INFO"] ) assert caplog.records[0].levelname in expected_levels @@ -178,7 +180,9 @@ def test_explicit_sudo_false(self, m_subp, dist, caplog): expected_levels = ( ["WARNING", "DEPRECATED"] - if should_log_deprecation("22.3") + if should_log_deprecation( + "22.3", features.DEPRECATION_INFO_BOUNDARY + ) else ["INFO"] ) assert caplog.records[1].levelname in expected_levels From 681b7de1a598bc5ab1b7d9868dcf2f32fd45aba3 Mon Sep 17 00:00:00 2001 From: Alberto Contreras Date: Mon, 1 Jul 2024 21:04:21 +0200 Subject: [PATCH 08/33] fix(rh_subscription): add string type to org (#5453) Per [1], org's correct type is string. Added as new type and deprecated integer. References: [1] https://github.com/candlepin/subscription-manager/blob/b6fad11e7783ae414fe88fdecee57d8db5c8e292/man/subscription-manager.8#L589 Fixes GH-5382 Co-authored-by: pneigel-ca --- .../config/schemas/schema-cloud-config-v1.json | 14 ++++++++++++-- doc/module-docs/cc_rh_subscription/example2.yaml | 2 +- .../unittests/config/test_cc_rh_subscription.py | 16 +++++++++++++++- tools/.github-cla-signers | 1 + 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/schemas/schema-cloud-config-v1.json b/cloudinit/config/schemas/schema-cloud-config-v1.json index f1ed3dd8f4d..f5609c539fc 100644 --- a/cloudinit/config/schemas/schema-cloud-config-v1.json +++ b/cloudinit/config/schemas/schema-cloud-config-v1.json @@ -2535,8 +2535,18 @@ "description": "The activation key to use. Must be used with ``org``. Should not be used with ``username`` or ``password``" }, "org": { - "type": "integer", - "description": "The organization number to use. Must be used with ``activation-key``. Should not be used with ``username`` or ``password``" + "description": "The organization to use. Must be used with ``activation-key``. Should not be used with ``username`` or ``password``", + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer", + "deprecated": true, + "deprecated_version": "24.2", + "deprecated_description": "Use of type integer for this value is deprecated. Use a string instead." + } + ] }, "auto-attach": { "type": "boolean", diff --git a/doc/module-docs/cc_rh_subscription/example2.yaml b/doc/module-docs/cc_rh_subscription/example2.yaml index b6fff8c44d1..72328f93811 100644 --- a/doc/module-docs/cc_rh_subscription/example2.yaml +++ b/doc/module-docs/cc_rh_subscription/example2.yaml @@ -1,4 +1,4 @@ #cloud-config rh_subscription: activation-key: foobar - org: 12345 + org: "ABC" diff --git a/tests/unittests/config/test_cc_rh_subscription.py b/tests/unittests/config/test_cc_rh_subscription.py index 955b092ba16..d811d16a5b6 100644 --- a/tests/unittests/config/test_cc_rh_subscription.py +++ b/tests/unittests/config/test_cc_rh_subscription.py @@ -184,7 +184,7 @@ class TestBadInput(CiTestCase): "rh_subscription": { "activation-key": "abcdef1234", "fookey": "bar", - "org": "123", + "org": "ABC", } } @@ -330,6 +330,20 @@ class TestRhSubscriptionSchema: {"rh_subscription": {"disable-repo": "name"}}, "'name' is not of type 'array'", ), + ( + { + "rh_subscription": { + "activation-key": "foobar", + "org": "ABC", + } + }, + None, + ), + ( + {"rh_subscription": {"activation-key": "foobar", "org": 314}}, + "Deprecated in version 24.2. Use of type integer for this" + " value is deprecated. Use a string instead.", + ), ], ) @skipUnlessJsonSchema() diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 3aea01b0023..d9accd11460 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -144,6 +144,7 @@ outscale-mdr philsphicas phsm phunyguy +pneigel-ca qubidt r00ta RedKrieg From ee1b25b73d8c08bf079a34af3837760e283715b0 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 1 Jul 2024 13:53:20 -0600 Subject: [PATCH 09/33] tests: update nocloud deprecation test for boundary version (#5474) --- .../integration_tests/datasources/test_nocloud.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/datasources/test_nocloud.py b/tests/integration_tests/datasources/test_nocloud.py index a21f91cfa21..c6c440840a3 100644 --- a/tests/integration_tests/datasources/test_nocloud.py +++ b/tests/integration_tests/datasources/test_nocloud.py @@ -6,10 +6,12 @@ from pycloudlib.lxd.instance import LXDInstance from cloudinit.subp import subp +from cloudinit.util import should_log_deprecation from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, FOCAL from tests.integration_tests.util import ( + get_feature_flag_value, override_kernel_command_line, verify_clean_boot, verify_clean_log, @@ -193,9 +195,18 @@ def test_smbios_seed_network(self, client: IntegrationInstance): assert client.execute("cloud-init clean --logs").ok client.restart() assert client.execute("test -f /var/tmp/smbios_test_file").ok - assert "'nocloud-net' datasource name is deprecated" in client.execute( - "cloud-init status --format json" + version_boundary = get_feature_flag_value( + client, "DEPRECATION_INFO_BOUNDARY" ) + # nocloud-net deprecated in version 24.1 + if should_log_deprecation("24.1", version_boundary): + log_level = "DEPRECATED" + else: + log_level = "INFO" + client.execute( + rf"grep \"{log_level}]: The 'nocloud-net' datasource name is" + ' deprecated" /var/log/cloud-init.log' + ).ok @pytest.mark.skipif(PLATFORM != "lxd_vm", reason="Modifies grub config") From c45280e981a0658d637af8736036e653ffe9646e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 1 Jul 2024 15:04:45 -0600 Subject: [PATCH 10/33] tests: update ubuntu_pro test to account for info-level deprecations (#5475) The ubuntu_advantage key is deprecated in 24.1 in favor of ubuntu_pro. Any version less than 24.1 will have only an info level log resulting in no DEPRECATED keys from cloud-init status --format=yaml. Parse log level messages in /var/log/cloud-init.log instead. --- .../modules/test_ubuntu_pro.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/integration_tests/modules/test_ubuntu_pro.py b/tests/integration_tests/modules/test_ubuntu_pro.py index 20bdd5b9510..f4438163425 100644 --- a/tests/integration_tests/modules/test_ubuntu_pro.py +++ b/tests/integration_tests/modules/test_ubuntu_pro.py @@ -5,6 +5,7 @@ import pytest from pycloudlib.cloud import ImageType +from cloudinit.util import should_log_deprecation from tests.integration_tests.clouds import IntegrationCloud from tests.integration_tests.conftest import get_validated_source from tests.integration_tests.instances import ( @@ -19,7 +20,10 @@ IS_UBUNTU, JAMMY, ) -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import ( + get_feature_flag_value, + verify_clean_log, +) LOG = logging.getLogger("integration_testing.test_ubuntu_pro") @@ -135,12 +139,18 @@ def test_valid_token(self, client: IntegrationInstance): "sed -i 's/ubuntu_pro$/ubuntu_advantage/' /etc/cloud/cloud.cfg" ) client.restart() - status_resp = client.execute("cloud-init status --format json") - status = json.loads(status_resp.stdout) - assert ( - "Module has been renamed from cc_ubuntu_advantage to cc_ubuntu_pro" - in "\n".join(status["recoverable_errors"]["DEPRECATED"]) + version_boundary = get_feature_flag_value( + client, "DEPRECATION_INFO_BOUNDARY" ) + # ubuntu_advantage key is deprecated in version 24.1 + if should_log_deprecation("24.1", version_boundary): + log_level = "DEPRECATED" + else: + log_level = "INFO" + client.execute( + rf"grep \"{log_level}]: Module has been renamed from" + " cc_ubuntu_advantage to cc_ubuntu_pro /var/log/cloud-init.log" + ).ok assert is_attached(client) @pytest.mark.user_data(ATTACH.format(token=CLOUD_INIT_UA_TOKEN)) From 40e2eb4aea83299b3de497b6237cb64ae48cfa74 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Mon, 1 Jul 2024 16:40:49 -0500 Subject: [PATCH 11/33] test: Update integration tests to pass on focal (#5476) - Update deprecation test to include focal. - Update test using `cloud-id` appropriately --- tests/integration_tests/cmd/test_status.py | 45 +++++++++------------ tests/integration_tests/test_ds_identify.py | 4 +- 2 files changed, 19 insertions(+), 30 deletions(-) diff --git a/tests/integration_tests/cmd/test_status.py b/tests/integration_tests/cmd/test_status.py index 20c78433257..84d9b694f63 100644 --- a/tests/integration_tests/cmd/test_status.py +++ b/tests/integration_tests/cmd/test_status.py @@ -3,14 +3,12 @@ import pytest -from cloudinit.util import should_log_deprecation from tests.integration_tests.clouds import IntegrationCloud from tests.integration_tests.decorators import retry from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU, JAMMY from tests.integration_tests.util import ( - get_feature_flag_value, push_and_enable_systemd_unit, wait_for_cloud_init, ) @@ -65,8 +63,10 @@ def test_wait_when_no_datasource(session_cloud: IntegrationCloud, setup_image): USER_DATA = """\ #cloud-config +users: + - name: something + ssh-authorized-keys: ["something"] ca-certs: - remove_defaults: false invalid_key: true """ @@ -81,32 +81,23 @@ def test_status_json_errors(client): "DEPRECATED" ) - status_json = json.loads( - client.execute("cloud-init status --format json").stdout + status_json = client.execute("cloud-init status --format json").stdout + assert ( + "Deprecated cloud-config provided: users.0.ssh-authorized-keys" + in json.loads(status_json)["init"]["recoverable_errors"] + .get("DEPRECATED") + .pop(0) ) - version_boundary = get_feature_flag_value( - client, "DEPRECATION_INFO_BOUNDARY" + assert ( + "Deprecated cloud-config provided: users.0.ssh-authorized-keys:" + in json.loads(status_json)["recoverable_errors"] + .get("DEPRECATED") + .pop(0) ) - # The deprecation_version is 22.3 in schema for ca-certs. - # Expect an extra deprecation level log in status if boundary > 22.3 - if should_log_deprecation("22.3", version_boundary): - assert "Deprecated cloud-config provided: ca-certs" in status_json[ - "init" - ]["recoverable_errors"].get("DEPRECATED").pop(0) - assert "Deprecated cloud-config provided: ca-certs" in status_json[ - "recoverable_errors" - ].get("DEPRECATED").pop(0) - - assert "Key 'ca-certs' is deprecated in 22.1" in status_json["init"][ - "recoverable_errors" - ].get("DEPRECATED").pop(0) - assert "Key 'ca-certs' is deprecated in 22.1" in status_json[ - "recoverable_errors" - ].get("DEPRECATED").pop(0) - assert "cloud-config failed schema validation" in status_json["init"][ - "recoverable_errors" - ].get("WARNING").pop(0) - assert "cloud-config failed schema validation" in status_json[ + assert "cloud-config failed schema validation" in json.loads(status_json)[ + "init" + ]["recoverable_errors"].get("WARNING").pop(0) + assert "cloud-config failed schema validation" in json.loads(status_json)[ "recoverable_errors" ].get("WARNING").pop(0) diff --git a/tests/integration_tests/test_ds_identify.py b/tests/integration_tests/test_ds_identify.py index 2375dde7909..59d3edd778f 100644 --- a/tests/integration_tests/test_ds_identify.py +++ b/tests/integration_tests/test_ds_identify.py @@ -1,7 +1,7 @@ """test that ds-identify works as expected""" from tests.integration_tests.instances import IntegrationInstance -from tests.integration_tests.integration_settings import OS_IMAGE, PLATFORM +from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.util import verify_clean_log, wait_for_cloud_init DATASOURCE_LIST_FILE = "/etc/cloud/cloud.cfg.d/90_dpkg.cfg" @@ -29,8 +29,6 @@ def test_ds_identify(client: IntegrationInstance): assert client.execute("cloud-init status --wait") datasource = MAP_PLATFORM_TO_DATASOURCE.get(PLATFORM, PLATFORM) - if "lxd" == datasource and "focal" == OS_IMAGE: - datasource = "nocloud" cloud_id = client.execute("cloud-id") assert cloud_id.ok assert datasource == cloud_id.stdout.rstrip() From 41c375c4f011ac030e2cf59d6741b24a28fbe257 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 31 May 2024 12:56:50 -0600 Subject: [PATCH 12/33] ci(mypy): Set default follow_imports value (#5350) Remove code for detecting a virtualenv. This has been broken since ~3.7. --- setup.py | 17 +++++++---------- setup_utils.py | 9 --------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index 3becd7bbb1d..3e33d0062bd 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ # isort: off from setup_utils import ( # noqa: E402 get_version, - in_virtualenv, is_f, is_generator, pkg_config_read, @@ -266,13 +265,12 @@ def finalize_options(self): self.distribution.reinitialize_command("install_data", True) -if not in_virtualenv(): - USR = "/" + USR - ETC = "/" + ETC - USR_LIB_EXEC = "/" + USR_LIB_EXEC - LIB = "/" + LIB - for k in INITSYS_ROOTS.keys(): - INITSYS_ROOTS[k] = "/" + INITSYS_ROOTS[k] +USR = "/" + USR +ETC = "/" + ETC +USR_LIB_EXEC = "/" + USR_LIB_EXEC +LIB = "/" + LIB +for k in INITSYS_ROOTS.keys(): + INITSYS_ROOTS[k] = "/" + INITSYS_ROOTS[k] data_files = [ (ETC + "/cloud", [render_tmpl("config/cloud.cfg.tmpl", is_yaml=True)]), @@ -308,8 +306,7 @@ def finalize_options(self): ] if not platform.system().endswith("BSD"): RULES_PATH = pkg_config_read("udev", "udevdir") - if not in_virtualenv(): - RULES_PATH = "/" + RULES_PATH + RULES_PATH = "/" + RULES_PATH data_files.extend( [ diff --git a/setup_utils.py b/setup_utils.py index f88905afdfb..0ff7581070c 100644 --- a/setup_utils.py +++ b/setup_utils.py @@ -35,15 +35,6 @@ def pkg_config_read(library: str, var: str) -> str: return path -def in_virtualenv() -> bool: - # TODO: sys.real_prefix doesn't exist on any currently supported - # version of python. This function can never return True - try: - return sys.real_prefix != sys.prefix - except AttributeError: - return False - - def version_to_pep440(version: str) -> str: # read-version can spit out something like 22.4-15-g7f97aee24 # which is invalid under PEP 440. If we replace the first - with a + From 16018b244922f71f6e2882567c58f8cbf5b4cdff Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 31 May 2024 18:14:34 -0600 Subject: [PATCH 13/33] fix(typing): Remove invalid type annotations (#5350) Invalid type annotations confuse mypy. When it sees a type annotation that doesn't conflict with other annotations, it is trusted. This means that future attempts to conditionally check a variable's type will cause mypy to produce an unreachable code path warning. --- cloudinit/config/cc_lxd.py | 4 +--- tests/unittests/sources/test_akamai.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 85e50679209..aa77e9ef901 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -30,9 +30,7 @@ } # type: ignore -def supplemental_schema_validation( - init_cfg: dict, bridge_cfg: dict, preseed_str: str -): +def supplemental_schema_validation(init_cfg, bridge_cfg, preseed_str): """Validate user-provided lxd network and bridge config option values. @raises: ValueError describing invalid values provided. diff --git a/tests/unittests/sources/test_akamai.py b/tests/unittests/sources/test_akamai.py index 7b6987bfb47..2480269f6e6 100644 --- a/tests/unittests/sources/test_akamai.py +++ b/tests/unittests/sources/test_akamai.py @@ -1,5 +1,5 @@ from contextlib import suppress -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Union import pytest @@ -224,7 +224,7 @@ def test_get_network_context_managers( get_interfaces_by_mac, local_stage: bool, ds_cfg: Dict[str, Any], - expected_manager_config: List[Tuple[Tuple[bool, bool], bool]], + expected_manager_config: List, expected_interface: str, ): """ From 555028524368facc375e28e3b4631d595e80c95f Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 31 May 2024 18:28:00 -0600 Subject: [PATCH 14/33] fix(typing): Remove type annotation for unused variable (#5350) Later use of the same variable name leads mypy to report unreachable code paths. --- cloudinit/config/cc_power_state_change.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 69420d6f960..5bd6b96d847 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -78,8 +78,8 @@ def check_condition(cond): def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: try: - (args, timeout, condition) = load_power_state(cfg, cloud.distro) - if args is None: + (arg_list, timeout, condition) = load_power_state(cfg, cloud.distro) + if arg_list is None: LOG.debug("no power_state provided. doing nothing") return except Exception as e: @@ -99,7 +99,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: devnull_fp = open(os.devnull, "w") - LOG.debug("After pid %s ends, will execute: %s", mypid, " ".join(args)) + LOG.debug("After pid %s ends, will execute: %s", mypid, " ".join(arg_list)) util.fork_cb( run_after_pid_gone, @@ -108,7 +108,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None: timeout, condition, execmd, - [args, devnull_fp], + [arg_list, devnull_fp], ) From a8cf24b7c8d777913974ea07e25e6f1cd50b82bd Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 31 May 2024 18:46:37 -0600 Subject: [PATCH 15/33] fix(typing): Add / update type annotations (#5350) Mypy infers an unannotated variable's type from its initial value. When a variable's type may change (such as from None to non-None), the variable's type needs to be annotated, otherwise mypy may warn of unreachable code paths. --- cloudinit/config/schema.py | 2 +- cloudinit/net/dhcp.py | 4 +++- cloudinit/net/ephemeral.py | 4 ++-- cloudinit/sources/DataSourceAzure.py | 6 +++--- cloudinit/url_helper.py | 3 ++- tests/unittests/cmd/test_status.py | 2 +- 6 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index e59fde4ee92..790fe1d1e80 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -127,7 +127,7 @@ class MetaSchema(TypedDict): title: str description: str distros: typing.List[str] - examples: typing.List[str] + examples: typing.List[Union[dict, str]] frequency: str activate_by_schema_keys: NotRequired[List[str]] diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 49d5ae80147..6b3aabc8603 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -82,7 +82,9 @@ class NoDHCPLeaseMissingDhclientError(NoDHCPLeaseError): """Raised when unable to find dhclient.""" -def maybe_perform_dhcp_discovery(distro, nic=None, dhcp_log_func=None): +def maybe_perform_dhcp_discovery( + distro, nic=None, dhcp_log_func=None +) -> Dict[str, Any]: """Perform dhcp discovery if nic valid and dhclient command exists. If the nic is invalid or undiscoverable or dhclient command is not found, diff --git a/cloudinit/net/ephemeral.py b/cloudinit/net/ephemeral.py index 4ac2aa12736..c8730fb1e8a 100644 --- a/cloudinit/net/ephemeral.py +++ b/cloudinit/net/ephemeral.py @@ -285,8 +285,8 @@ def __init__( dhcp_log_func=None, ): self.iface = iface - self._ephipv4 = None - self.lease = None + self._ephipv4: Optional[EphemeralIPv4Network] = None + self.lease: Optional[Dict[str, Any]] = None self.dhcp_log_func = dhcp_log_func self.connectivity_url_data = connectivity_url_data self.distro = distro diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 04a928c31af..44b1e194fa4 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -331,7 +331,7 @@ def __init__(self, sys_cfg, distro, paths): ) self._iso_dev = None self._network_config = None - self._ephemeral_dhcp_ctx = None + self._ephemeral_dhcp_ctx: Optional[EphemeralDHCPv4] = None self._route_configured_for_imds = False self._route_configured_for_wireserver = False self._wireserver_endpoint = DEFAULT_WIRESERVER_ENDPOINT @@ -426,7 +426,7 @@ def _setup_ephemeral_networking( dhcp_log_func=dhcp_log_cb, ) - lease = None + lease: Optional[Dict[str, Any]] = None start_time = monotonic() deadline = start_time + timeout_minutes * 60 with events.ReportEventStack( @@ -1252,7 +1252,7 @@ def _wait_for_pps_unknown_reuse(self): def _poll_imds(self) -> bytes: """Poll IMDs for reprovisiondata XML document data.""" dhcp_attempts = 0 - reprovision_data = None + reprovision_data: Optional[bytes] = None while not reprovision_data: if not self._is_ephemeral_networking_up(): dhcp_attempts += 1 diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index a897194661a..777aa6a70e3 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -279,7 +279,8 @@ def __init__(self, response: requests.Response): @property def contents(self) -> bytes: if self._response.content is None: - return b"" + # typeshed bug: https://github.com/python/typeshed/pull/12180 + return b"" # type: ignore return self._response.content @property diff --git a/tests/unittests/cmd/test_status.py b/tests/unittests/cmd/test_status.py index ddf738560f8..22241139ff1 100644 --- a/tests/unittests/cmd/test_status.py +++ b/tests/unittests/cmd/test_status.py @@ -746,7 +746,7 @@ def test_status_output( assert_file, cmdargs: MyArgs, expected_retcode: int, - expected_status: str, + expected_status: Union[str, dict], config: Config, capsys, ): From b26d388ceb7c8f0cf2e8fc214049f9477090ddc4 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 31 May 2024 18:52:01 -0600 Subject: [PATCH 16/33] refactor(typing): Remove unused code paths (#5350) Enable mypy's "warn_unreachable" checks. Remove adjacent unused mocks. Note: mypy can incorrectly report typing errors as "unused code paths" --- cloudinit/config/schema.py | 10 ----- cloudinit/sources/DataSourceLXD.py | 4 -- cloudinit/sources/DataSourceNWCS.py | 12 +++-- cloudinit/sources/__init__.py | 10 ++++- cloudinit/temp_utils.py | 12 +---- cloudinit/util.py | 2 - pyproject.toml | 1 + tests/integration_tests/test_upgrade.py | 2 - .../unittests/config/test_cc_apk_configure.py | 11 +++-- tests/unittests/config/test_cc_ntp.py | 44 +++++++++---------- tests/unittests/config/test_schema.py | 23 ---------- tests/unittests/test_temp_utils.py | 5 --- 12 files changed, 42 insertions(+), 94 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 790fe1d1e80..062ab92ecd8 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -939,16 +939,6 @@ def annotate( if not schema_errors and not schema_deprecations: return self._original_content lines = self._original_content.split("\n") - if not isinstance(self._cloudconfig, dict): - # Return a meaningful message on empty cloud-config - return "\n".join( - lines - + [ - self._build_footer( - "Errors", ["# E1: Cloud-config is not a YAML dict."] - ) - ] - ) errors_by_line = self._build_errors_by_line(schema_errors) deprecations_by_line = self._build_errors_by_line(schema_deprecations) annotated_content = self._annotate_content( diff --git a/cloudinit/sources/DataSourceLXD.py b/cloudinit/sources/DataSourceLXD.py index a85853ec44a..4f69d90eb70 100644 --- a/cloudinit/sources/DataSourceLXD.py +++ b/cloudinit/sources/DataSourceLXD.py @@ -213,10 +213,6 @@ def _get_data(self) -> bool: user_metadata = _raw_instance_data_to_dict( "user.meta-data", user_metadata ) - if not isinstance(self.metadata, dict): - self.metadata = util.mergemanydict( - [util.load_yaml(self.metadata), user_metadata] - ) if "user-data" in self._crawled_metadata: self.userdata_raw = self._crawled_metadata["user-data"] if "network-config" in self._crawled_metadata: diff --git a/cloudinit/sources/DataSourceNWCS.py b/cloudinit/sources/DataSourceNWCS.py index 03a86254891..7c89713cb33 100644 --- a/cloudinit/sources/DataSourceNWCS.py +++ b/cloudinit/sources/DataSourceNWCS.py @@ -45,6 +45,11 @@ def __init__(self, sys_cfg, distro, paths): self.dsmode = sources.DSMODE_NETWORK self.metadata_full = None + def _unpickle(self, ci_pkl_version: int) -> None: + super()._unpickle(ci_pkl_version) + if not self._network_config: + self._network_config = sources.UNSET + def _get_data(self): md = self.get_metadata() @@ -95,13 +100,6 @@ def get_metadata(self): def network_config(self): LOG.debug("Attempting network configuration") - if self._network_config is None: - LOG.warning( - "Found None as cached _network_config, resetting to %s", - sources.UNSET, - ) - self._network_config = sources.UNSET - if self._network_config != sources.UNSET: return self._network_config diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index df19b764559..1420862ea21 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -317,7 +317,7 @@ def __init__(self, sys_cfg, distro: Distro, paths: Paths, ud_proc=None): self.sys_cfg = sys_cfg self.distro = distro self.paths = paths - self.userdata = None + self.userdata: Optional[Any] = None self.metadata: dict = {} self.userdata_raw: Optional[str] = None self.vendordata = None @@ -359,7 +359,6 @@ def _unpickle(self, ci_pkl_version: int) -> None: if not hasattr(self, "check_if_fallback_is_allowed"): setattr(self, "check_if_fallback_is_allowed", lambda: False) - if hasattr(self, "userdata") and self.userdata is not None: # If userdata stores MIME data, on < python3.6 it will be # missing the 'policy' attribute that exists on >=python3.6. @@ -376,6 +375,7 @@ def _unpickle(self, ci_pkl_version: int) -> None: ) raise DatasourceUnpickleUserDataError() from e + def __str__(self): return type_utils.obj_name(self) @@ -484,6 +484,12 @@ def get_data(self) -> bool: """ self._dirty_cache = True return_value = self._check_and_get_data() + # TODO: verify that datasource types are what they are expected to be + # each datasource uses different logic to get userdata, metadata, etc + # and then the rest of the codebase assumes the types of this data + # it would be prudent to have a type check here that warns, when the + # datatype is incorrect, rather than assuming types and throwing + # exceptions later if/when they get used incorrectly. if not return_value: return return_value self.persist_instance_data() diff --git a/cloudinit/temp_utils.py b/cloudinit/temp_utils.py index 957433474aa..faa4aaa287a 100644 --- a/cloudinit/temp_utils.py +++ b/cloudinit/temp_utils.py @@ -10,7 +10,6 @@ from cloudinit import util LOG = logging.getLogger(__name__) -_TMPDIR = None _ROOT_TMPDIR = "/run/cloud-init/tmp" _EXE_ROOT_TMPDIR = "/var/tmp/cloud-init" @@ -20,8 +19,6 @@ def get_tmp_ancestor(odir=None, needs_exe: bool = False): return odir if needs_exe: return _EXE_ROOT_TMPDIR - if _TMPDIR: - return _TMPDIR if os.getuid() == 0: return _ROOT_TMPDIR return os.environ.get("TMPDIR", "/tmp") @@ -53,18 +50,11 @@ def _tempfile_dir_arg(odir=None, needs_exe: bool = False): " mounted as noexec", tdir, ) - - if odir is None and not needs_exe: - global _TMPDIR - _TMPDIR = tdir - return tdir def ExtendedTemporaryFile(**kwargs): - kwargs["dir"] = _tempfile_dir_arg( - kwargs.pop("dir", None), kwargs.pop("needs_exe", False) - ) + kwargs["dir"] = _tempfile_dir_arg() fh = tempfile.NamedTemporaryFile(**kwargs) # Replace its unlink with a quiet version # that does not raise errors when the diff --git a/cloudinit/util.py b/cloudinit/util.py index 5b8f07683f6..93ba6e0dcbf 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -357,8 +357,6 @@ def read_conf(fname, *, instance_data_file=None) -> Dict: config_file, repr(e), ) - if config_file is None: - return {} return load_yaml(config_file, default={}) # pyright: ignore diff --git a/pyproject.toml b/pyproject.toml index 49811eb7477..a0419d31666 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ follow_imports = "silent" check_untyped_defs = true warn_redundant_casts = true warn_unused_ignores = true +warn_unreachable = true exclude = [] [[tool.mypy.overrides]] diff --git a/tests/integration_tests/test_upgrade.py b/tests/integration_tests/test_upgrade.py index 4d29b29f832..970a2406d8a 100644 --- a/tests/integration_tests/test_upgrade.py +++ b/tests/integration_tests/test_upgrade.py @@ -62,7 +62,6 @@ def test_clean_boot_of_upgraded_package(session_cloud: IntegrationCloud): source = get_validated_source(session_cloud) if not source.installs_new_version(): pytest.skip(UNSUPPORTED_INSTALL_METHOD_MSG.format(source)) - return # type checking doesn't understand that skip raises launch_kwargs = { "image_id": session_cloud.initial_image_id, } @@ -194,7 +193,6 @@ def test_subsequent_boot_of_upgraded_package(session_cloud: IntegrationCloud): pytest.fail(UNSUPPORTED_INSTALL_METHOD_MSG.format(source)) else: pytest.skip(UNSUPPORTED_INSTALL_METHOD_MSG.format(source)) - return # type checking doesn't understand that skip raises launch_kwargs = {"image_id": session_cloud.initial_image_id} diff --git a/tests/unittests/config/test_cc_apk_configure.py b/tests/unittests/config/test_cc_apk_configure.py index 544820a9335..6d09c5738ad 100644 --- a/tests/unittests/config/test_cc_apk_configure.py +++ b/tests/unittests/config/test_cc_apk_configure.py @@ -10,7 +10,7 @@ import pytest -from cloudinit import cloud, helpers, temp_utils, util +from cloudinit import cloud, helpers, util from cloudinit.config import cc_apk_configure from cloudinit.config.schema import ( SchemaValidationError, @@ -51,7 +51,7 @@ def test_no_config(self): class TestConfig(FilesystemMockingTestCase): def setUp(self): - super(TestConfig, self).setUp() + super().setUp() self.new_root = self.tmp_dir() self.new_root = self.reRoot(root=self.new_root) for dirname in ["tmp", "etc/apk"]: @@ -60,11 +60,14 @@ def setUp(self): self.name = "apk_configure" self.cloud = cloud.Cloud(None, self.paths, None, None, None) self.args = [] - temp_utils._TMPDIR = self.new_root + self.mock = mock.patch( + "cloudinit.temp_utils.get_tmp_ancestor", lambda *_: self.new_root + ) + self.mock.start() def tearDown(self): + self.mock.stop() super().tearDown() - temp_utils._TMPDIR = None @mock.patch(CC_APK + "._write_repositories_file") def test_no_repo_settings(self, m_write_repos): diff --git a/tests/unittests/config/test_cc_ntp.py b/tests/unittests/config/test_cc_ntp.py index 6f6c3360de7..ead6f5213f3 100644 --- a/tests/unittests/config/test_cc_ntp.py +++ b/tests/unittests/config/test_cc_ntp.py @@ -138,16 +138,14 @@ def test_write_ntp_config_template_uses_ntp_conf_distro_no_servers(self): servers = [] pools = ["10.0.0.1", "10.0.0.2"] (confpath, template_fn) = self._generate_template() - mock_path = "cloudinit.config.cc_ntp.temp_utils._TMPDIR" - with mock.patch(mock_path, self.new_root): - cc_ntp.write_ntp_config_template( - "ubuntu", - servers=servers, - pools=pools, - path=confpath, - template_fn=template_fn, - template=None, - ) + cc_ntp.write_ntp_config_template( + "ubuntu", + servers=servers, + pools=pools, + path=confpath, + template_fn=template_fn, + template=None, + ) self.assertEqual( "servers []\npools ['10.0.0.1', '10.0.0.2']\n", util.load_text_file(confpath), @@ -163,16 +161,14 @@ def test_write_ntp_config_template_defaults_pools_w_empty_lists(self): pools = cc_ntp.generate_server_names(distro) servers = [] (confpath, template_fn) = self._generate_template() - mock_path = "cloudinit.config.cc_ntp.temp_utils._TMPDIR" - with mock.patch(mock_path, self.new_root): - cc_ntp.write_ntp_config_template( - distro, - servers=servers, - pools=pools, - path=confpath, - template_fn=template_fn, - template=None, - ) + cc_ntp.write_ntp_config_template( + distro, + servers=servers, + pools=pools, + path=confpath, + template_fn=template_fn, + template=None, + ) self.assertEqual( "servers []\npools {0}\n".format(pools), util.load_text_file(confpath), @@ -672,8 +668,8 @@ def test_ntp_user_provided_config_with_template(self, m_install): } for distro in cc_ntp.distros: mycloud = self._get_cloud(distro) - mock_path = "cloudinit.config.cc_ntp.temp_utils._TMPDIR" - with mock.patch(mock_path, self.new_root): + mock_path = "cloudinit.config.cc_ntp.temp_utils.get_tmp_ancestor" + with mock.patch(mock_path, lambda *_: self.new_root): cc_ntp.handle("notimportant", cfg, mycloud, None) self.assertEqual( "servers []\npools ['mypool.org']\n%s" % custom, @@ -712,8 +708,8 @@ def test_ntp_user_provided_config_template_only( ) confpath = ntpconfig["confpath"] m_select.return_value = ntpconfig - mock_path = "cloudinit.config.cc_ntp.temp_utils._TMPDIR" - with mock.patch(mock_path, self.new_root): + mock_path = "cloudinit.config.cc_ntp.temp_utils.get_tmp_ancestor" + with mock.patch(mock_path, lambda *_: self.new_root): cc_ntp.handle("notimportant", {"ntp": cfg}, mycloud, None) self.assertEqual( "servers []\npools ['mypool.org']\n%s" % custom, diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 2ea42b62d5b..184857583fb 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -1767,29 +1767,6 @@ def test_annotated_cloudconfig_file_no_schema_errors(self): schema_errors=[], ) - def test_annotated_cloudconfig_file_with_non_dict_cloud_config(self): - """Error when empty non-dict cloud-config is provided. - - OurJSON validation when user-data is None type generates a bunch - schema validation errors of the format: - ('', "None is not of type 'object'"). Ignore those symptoms and - report the general problem instead. - """ - content = "\n\n\n" - expected = "\n".join( - [ - content, - "# Errors: -------------", - "# E1: Cloud-config is not a YAML dict.\n\n", - ] - ) - assert expected == annotated_cloudconfig_file( - None, - content, - schemamarks={}, - schema_errors=[SchemaProblem("", "None is not of type 'object'")], - ) - def test_annotated_cloudconfig_file_schema_annotates_and_adds_footer(self): """With schema_errors, error lines are annotated and a footer added.""" content = dedent( diff --git a/tests/unittests/test_temp_utils.py b/tests/unittests/test_temp_utils.py index d47852d66d8..6cbc372823c 100644 --- a/tests/unittests/test_temp_utils.py +++ b/tests/unittests/test_temp_utils.py @@ -25,7 +25,6 @@ def fake_mkdtemp(*args, **kwargs): { "os.getuid": 1000, "tempfile.mkdtemp": {"side_effect": fake_mkdtemp}, - "_TMPDIR": {"new": None}, "os.path.isdir": True, }, mkdtemp, @@ -46,7 +45,6 @@ def fake_mkdtemp(*args, **kwargs): { "os.getuid": 1000, "tempfile.mkdtemp": {"side_effect": fake_mkdtemp}, - "_TMPDIR": {"new": None}, "os.path.isdir": True, "util.has_mount_opt": True, }, @@ -69,7 +67,6 @@ def fake_mkdtemp(*args, **kwargs): { "os.getuid": 0, "tempfile.mkdtemp": {"side_effect": fake_mkdtemp}, - "_TMPDIR": {"new": None}, "os.path.isdir": True, }, mkdtemp, @@ -90,7 +87,6 @@ def fake_mkstemp(*args, **kwargs): { "os.getuid": 1000, "tempfile.mkstemp": {"side_effect": fake_mkstemp}, - "_TMPDIR": {"new": None}, "os.path.isdir": True, }, mkstemp, @@ -111,7 +107,6 @@ def fake_mkstemp(*args, **kwargs): { "os.getuid": 0, "tempfile.mkstemp": {"side_effect": fake_mkstemp}, - "_TMPDIR": {"new": None}, "os.path.isdir": True, }, mkstemp, From 97146b57f85e4faafc232b61e9ca0aa3bb0a2812 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Fri, 31 May 2024 19:41:46 -0600 Subject: [PATCH 17/33] chore(typing): Remove type ignores and casts (#5350) Python's provides type annotation features which allow validating types throughout the code base. Use, rather than ignore, these features. Note that since a dict's .get() may return None, using this method when the only possible return value is a literal will confuse mypy. --- cloudinit/distros/__init__.py | 3 ++- cloudinit/distros/bsd.py | 2 +- cloudinit/net/netplan.py | 25 +++++++++++++---------- cloudinit/sources/__init__.py | 1 - cloudinit/url_helper.py | 4 ++-- cloudinit/util.py | 9 ++++---- tests/unittests/conftest.py | 2 +- tests/unittests/net/test_network_state.py | 2 +- tests/unittests/test_log.py | 7 ++----- 9 files changed, 28 insertions(+), 27 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 789af4a7d69..3c956408090 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -50,6 +50,7 @@ from cloudinit.distros.parsers import hosts from cloudinit.features import ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES from cloudinit.net import activators, dhcp, renderers +from cloudinit.net.netops import NetOps from cloudinit.net.network_state import parse_net_config_data from cloudinit.net.renderer import Renderer @@ -142,7 +143,7 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta): # This is used by self.shutdown_command(), and can be overridden in # subclasses shutdown_options_map = {"halt": "-H", "poweroff": "-P", "reboot": "-r"} - net_ops = iproute2.Iproute2 + net_ops: Type[NetOps] = iproute2.Iproute2 _ci_pkl_version = 1 prefer_fqdn = False diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py index 5bef9203c3d..25b374ba3bc 100644 --- a/cloudinit/distros/bsd.py +++ b/cloudinit/distros/bsd.py @@ -28,7 +28,7 @@ class BSD(distros.Distro): # There is no update/upgrade on OpenBSD pkg_cmd_update_prefix: Optional[List[str]] = None pkg_cmd_upgrade_prefix: Optional[List[str]] = None - net_ops = bsd_netops.BsdNetOps # type: ignore + net_ops = bsd_netops.BsdNetOps def __init__(self, name, cfg, paths): super().__init__(name, cfg, paths) diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 4e1a01ae94b..532442dcb83 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -6,7 +6,7 @@ import os import textwrap from tempfile import SpooledTemporaryFile -from typing import Callable, List, Optional, cast +from typing import Callable, List, Optional from cloudinit import features, safeyaml, subp, util from cloudinit.net import ( @@ -454,10 +454,7 @@ def _render_content(self, network_state: NetworkState) -> str: bond_config = {} # extract bond params and drop the bond_ prefix as it's # redundant in v2 yaml format - v2_bond_map = cast(dict, NET_CONFIG_TO_V2.get("bond")) - # Previous cast is needed to help mypy to know that the key is - # present in `NET_CONFIG_TO_V2`. This could probably be removed - # by using `Literal` when supported. + v2_bond_map = NET_CONFIG_TO_V2["bond"] for match in ["bond_", "bond-"]: bond_params = _get_params_dict_by_match(ifcfg, match) for param, value in bond_params.items(): @@ -478,9 +475,18 @@ def _render_content(self, network_state: NetworkState) -> str: elif if_type == "bridge": # required_keys = ['name', 'bridge_ports'] + # + # Rather than raise an exception on `sorted(None)`, log a + # warning and skip this interface when invalid configuration is + # received. bridge_ports = ifcfg.get("bridge_ports") - # mypy wrong error. `copy(None)` is supported: - ports = sorted(copy.copy(bridge_ports)) # type: ignore + if bridge_ports is None: + LOG.warning( + "Invalid config. The key", + f"'bridge_ports' is required in {config}.", + ) + continue + ports = sorted(copy.copy(bridge_ports)) bridge: dict = { "interfaces": ports, } @@ -492,10 +498,7 @@ def _render_content(self, network_state: NetworkState) -> str: # v2 yaml uses different names for the keys # and at least one value format change - v2_bridge_map = cast(dict, NET_CONFIG_TO_V2.get("bridge")) - # Previous cast is needed to help mypy to know that the key is - # present in `NET_CONFIG_TO_V2`. This could probably be removed - # by using `Literal` when supported. + v2_bridge_map = NET_CONFIG_TO_V2["bridge"] for param, value in params.items(): newname = v2_bridge_map.get(param) if newname is None: diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 1420862ea21..27c37ee1e13 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -375,7 +375,6 @@ def _unpickle(self, ci_pkl_version: int) -> None: ) raise DatasourceUnpickleUserDataError() from e - def __str__(self): return type_utils.obj_name(self) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 777aa6a70e3..d409e322858 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -560,7 +560,7 @@ def dual_stack( """ return_result = None returned_address = None - last_exception = None + last_exception: Optional[BaseException] = None exceptions = [] is_done = threading.Event() @@ -620,7 +620,7 @@ def dual_stack( "Timed out waiting for addresses: %s, " "exception(s) raised while waiting: %s", " ".join(addresses), - " ".join(exceptions), # type: ignore + " ".join(map(str, exceptions)), ) finally: executor.shutdown(wait=False) diff --git a/cloudinit/util.py b/cloudinit/util.py index 93ba6e0dcbf..98dd66d59fc 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -3251,8 +3251,8 @@ def deprecate( Note: uses keyword-only arguments to improve legibility """ - if not hasattr(deprecate, "_log"): - deprecate._log = set() # type: ignore + if not hasattr(deprecate, "log"): + setattr(deprecate, "log", set()) message = extra_message or "" dedup = hash(deprecated + message + deprecated_version + str(schedule)) version = Version.from_str(deprecated_version) @@ -3270,8 +3270,9 @@ def deprecate( level = log.DEPRECATED else: level = logging.WARN - if not skip_log and dedup not in deprecate._log: # type: ignore - deprecate._log.add(dedup) # type: ignore + log_cache = getattr(deprecate, "log") + if not skip_log and dedup not in log_cache: + log_cache.add(dedup) LOG.log(level, deprecate_msg) return DeprecationLog(level, deprecate_msg) diff --git a/tests/unittests/conftest.py b/tests/unittests/conftest.py index e912c90ad28..e0baa63b99b 100644 --- a/tests/unittests/conftest.py +++ b/tests/unittests/conftest.py @@ -152,7 +152,7 @@ def clear_deprecation_log(): # Since deprecations are de-duped, the existance (or non-existance) of # a deprecation warning in a previous test can cause the next test to # fail. - util.deprecate._log = set() + setattr(util.deprecate, "log", set()) PYTEST_VERSION_TUPLE = tuple(map(int, pytest.__version__.split("."))) diff --git a/tests/unittests/net/test_network_state.py b/tests/unittests/net/test_network_state.py index 19b1b1b30e2..eaad90dc8e1 100644 --- a/tests/unittests/net/test_network_state.py +++ b/tests/unittests/net/test_network_state.py @@ -215,7 +215,7 @@ def test_v2_warns_deprecated_gateways( In netplan targets we perform a passthrough and the warning is not needed. """ - util.deprecate._log = set() # type: ignore + util.deprecate.__dict__["log"] = set() ncfg = yaml.safe_load( cfg.format( gateway4="gateway4: 10.54.0.1", diff --git a/tests/unittests/test_log.py b/tests/unittests/test_log.py index 78b53f5a9f9..87996310349 100644 --- a/tests/unittests/test_log.py +++ b/tests/unittests/test_log.py @@ -63,8 +63,7 @@ def test_logger_uses_gmtime(self): class TestDeprecatedLogs: def test_deprecated_log_level(self, caplog): - logger = logging.getLogger() - logger.deprecated("deprecated message") + logging.getLogger().deprecated("deprecated message") assert "DEPRECATED" == caplog.records[0].levelname assert "deprecated message" in caplog.text @@ -116,7 +115,6 @@ def test_deprecate_log_level_based_on_features( ) def test_log_deduplication(self, caplog): - log.define_deprecation_logger() util.deprecate( deprecated="stuff", deprecated_version="19.1", @@ -139,6 +137,5 @@ def test_log_deduplication(self, caplog): def test_logger_prints_to_stderr(capsys): message = "to stdout" log.setup_basic_logging() - LOG = logging.getLogger() - LOG.warning(message) + logging.getLogger().warning(message) assert message in capsys.readouterr().err From e4c05e80b8ba7f1449674ffd0240dff435ebbdf2 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Mon, 3 Jun 2024 15:05:31 -0600 Subject: [PATCH 18/33] fix(tox): Update tox.ini (#5350) Enable positional arguments for various environments. Add missing deps for tip-mypy. The -tip envs were broken when a non-tip env existed. In these cases, {[testenv:mypy]commands} resolved to the non-tip environment. Fix it. --- tox.ini | 47 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/tox.ini b/tox.ini index 37e61f120c4..52d6505e9ad 100644 --- a/tox.ini +++ b/tox.ini @@ -45,31 +45,33 @@ version = cloudinit/config/schemas/versions.schema.cloud-config.json [testenv:ruff] deps = ruff=={[format_deps]ruff} -commands = {envpython} -m ruff check . +commands = {envpython} -m ruff check {posargs:.} [testenv:pylint] deps = pylint=={[format_deps]pylint} -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt -commands = {envpython} -m pylint {posargs:cloudinit/ tests/ tools/ conftest.py setup.py} +commands = {envpython} -m pylint {posargs:.} [testenv:black] deps = black=={[format_deps]black} -commands = {envpython} -m black . --check +commands = {envpython} -m black --check {posargs:.} [testenv:isort] deps = isort=={[format_deps]isort} -commands = {envpython} -m isort . --check-only --diff +commands = {envpython} -m isort --check-only --diff {posargs:.} [testenv:mypy] deps = + -r{toxinidir}/test-requirements.txt + -r{toxinidir}/integration-requirements.txt + -r{toxinidir}/doc-requirements.txt hypothesis=={[format_deps]hypothesis} hypothesis_jsonschema=={[format_deps]hypothesis_jsonschema} mypy=={[format_deps]mypy} - pytest=={[format_deps]pytest} types-jsonschema=={[format_deps]types-jsonschema} types-Jinja2=={[format_deps]types-Jinja2} types-passlib=={[format_deps]types-passlib} @@ -78,7 +80,7 @@ deps = types-requests=={[format_deps]types-requests} types-setuptools=={[format_deps]types-setuptools} typing-extensions=={[format_deps]typing-extensions} -commands = {envpython} -m mypy cloudinit/ tests/ tools/ +commands = {envpython} -m mypy {posargs:cloudinit/ tests/ tools/} [testenv:check_format] deps = @@ -89,16 +91,18 @@ deps = isort=={[format_deps]isort} mypy=={[format_deps]mypy} pylint=={[format_deps]pylint} - pytest=={[format_deps]pytest} types-jsonschema=={[format_deps]types-jsonschema} + types-Jinja2=={[format_deps]types-Jinja2} types-oauthlib=={[format_deps]types-oauthlib} types-passlib=={[format_deps]types-passlib} types-pyyaml=={[format_deps]types-PyYAML} + types-oauthlib=={[format_deps]types-oauthlib} types-requests=={[format_deps]types-requests} types-setuptools=={[format_deps]types-setuptools} typing-extensions=={[format_deps]typing-extensions} -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt + -r{toxinidir}/doc-requirements.txt commands = {[testenv:black]commands} {[testenv:ruff]commands} @@ -115,15 +119,17 @@ deps = isort mypy pylint - pytest types-jsonschema + types-Jinja2 types-oauthlib + types-passlib types-pyyaml + types-oauthlib types-requests types-setuptools - typing-extensions -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt + -r{toxinidir}/doc-requirements.txt commands = {[testenv:check_format]commands} @@ -151,7 +157,8 @@ commands = {envpython} -m pytest \ -vvvv --showlocals \ --durations 10 \ -m "not hypothesis_slow" \ - {posargs:--cov=cloudinit --cov-branch tests/unittests} + --cov=cloudinit --cov-branch \ + {posargs:tests/unittests} # experimental [testenv:py3-fast] @@ -168,15 +175,16 @@ deps = -r{toxinidir}/test-requirements.txt commands = {envpython} -m pytest \ -m hypothesis_slow \ - {posargs:--hypothesis-show-statistics tests/unittests} + --hypothesis-show-statistics \ + {posargs:tests/unittests} #commands = {envpython} -X tracemalloc=40 -Werror::ResourceWarning:cloudinit -m pytest \ [testenv:py3-leak] deps = {[testenv:py3]deps} commands = {envpython} -X tracemalloc=40 -Wall -m pytest \ --durations 10 \ - {posargs:--cov=cloudinit --cov-branch \ - tests/unittests} + --cov=cloudinit --cov-branch \ + {posargs:tests/unittests} [testenv:lowest-supported] @@ -242,16 +250,24 @@ commands = {[testenv:ruff]commands} [testenv:tip-mypy] deps = + -r{toxinidir}/test-requirements.txt + -r{toxinidir}/integration-requirements.txt + -r{toxinidir}/doc-requirements.txt hypothesis hypothesis_jsonschema mypy pytest + types-Jinja2 types-jsonschema types-oauthlib types-PyYAML + types-passlib + types-pyyaml + types-oauthlib types-requests types-setuptools -commands = {[testenv:mypy]commands} + typing-extensions +commands = {envpython} -m mypy {posargs:cloudinit/ tests/ tools/} [testenv:tip-pylint] deps = @@ -260,7 +276,8 @@ deps = # test-requirements -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt -commands = {[testenv:pylint]commands} +commands = {envpython} -m pylint {posargs:.} + [testenv:tip-black] deps = black From 73a5c512a5e1bd9dcdbf6741b6b97d3cfc644c3b Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Mon, 3 Jun 2024 15:18:18 -0600 Subject: [PATCH 19/33] chore: Auto-format network jsonschema in ci (#5350) Fix the current network v2 schema as well. --- .../schemas/schema-network-config-v2.json | 112 ++++++++++-------- tox.ini | 4 + 2 files changed, 68 insertions(+), 48 deletions(-) diff --git a/cloudinit/config/schemas/schema-network-config-v2.json b/cloudinit/config/schemas/schema-network-config-v2.json index 64c29f5ba16..0a3741d65ac 100644 --- a/cloudinit/config/schemas/schema-network-config-v2.json +++ b/cloudinit/config/schemas/schema-network-config-v2.json @@ -10,43 +10,43 @@ ] }, "dhcp-overrides": { - "type": "object", - "description": "DHCP behaviour overrides. Overrides will only have an effect if the corresponding DHCP type is enabled.", - "additionalProperties": false, - "properties": { - "hostname": { - "type": "string", - "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." - }, - "route-metric": { - "type": "integer", - "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." - }, - "send-hostname": { - "type": "boolean", - "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." - }, - "use-dns": { - "type": "boolean" - }, - "use-domains": { - "type": "string" - }, - "use-hostname": { - "type": "boolean" - }, - "use-mtu": { - "type": "boolean", - "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." - }, - "use-ntp": { - "type": "boolean" - }, - "use-routes": { - "type": "boolean", - "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." - } + "type": "object", + "description": "DHCP behaviour overrides. Overrides will only have an effect if the corresponding DHCP type is enabled.", + "additionalProperties": false, + "properties": { + "hostname": { + "type": "string", + "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." + }, + "route-metric": { + "type": "integer", + "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." + }, + "send-hostname": { + "type": "boolean", + "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." + }, + "use-dns": { + "type": "boolean" + }, + "use-domains": { + "type": "string" + }, + "use-hostname": { + "type": "boolean" + }, + "use-mtu": { + "type": "boolean", + "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." + }, + "use-ntp": { + "type": "boolean" + }, + "use-routes": { + "type": "boolean", + "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." } + } }, "gateway": { "type": "string", @@ -56,17 +56,33 @@ "type": "object", "properties": { "renderer": { - "$ref": "#/$defs/renderer" + "$ref": "#/$defs/renderer" }, "dhcp4": { - "type": ["boolean", "string"], + "type": [ + "boolean", + "string" + ], "description": "Enable DHCP for IPv4. Off by default.", - "enum": ["yes", "no", true, false] + "enum": [ + "yes", + "no", + true, + false + ] }, "dhcp6": { - "type": ["boolean", "string"], + "type": [ + "boolean", + "string" + ], "description": "Enable DHCP for IPv6. Off by default.", - "enum": ["yes", "no", true, false] + "enum": [ + "yes", + "no", + true, + false + ] }, "dhcp4-overrides": { "$ref": "#/$defs/dhcp-overrides" @@ -89,7 +105,7 @@ }, "mtu": { "type": "integer", - "description": "The MTU key represents a device’s Maximum Transmission Unit, the largest size packet or frame, specified in octets (eight-bit bytes), that can be sent in a packet- or frame-based network. Specifying mtu is optional." + "description": "The MTU key represents a device\u2019s Maximum Transmission Unit, the largest size packet or frame, specified in octets (eight-bit bytes), that can be sent in a packet- or frame-based network. Specifying mtu is optional." }, "nameservers": { "type": "object", @@ -152,7 +168,7 @@ }, "macaddress": { "type": "string", - "description": "Device’s MAC address in the form xx:xx:xx:xx:xx:xx. Globs are not allowed. Letters must be lowercase." + "description": "Device\u2019s MAC address in the form xx:xx:xx:xx:xx:xx. Globs are not allowed. Letters must be lowercase." }, "driver": { "type": "string", @@ -162,7 +178,7 @@ }, "set-name": { "type": "string", - "description": "When matching on unique properties such as path or MAC, or with additional assumptions such as ''there will only ever be one wifi device'', match rules can be written so that they only match one device. Then this property can be used to give that device a more specific/desirable/nicer name than the default from udev’s ifnames. Any additional device that satisfies the match rules will then fail to get renamed and keep the original kernel name (and dmesg will show an error)." + "description": "When matching on unique properties such as path or MAC, or with additional assumptions such as ''there will only ever be one wifi device'', match rules can be written so that they only match one device. Then this property can be used to give that device a more specific/desirable/nicer name than the default from udev\u2019s ifnames. Any additional device that satisfies the match rules will then fail to get renamed and keep the original kernel name (and dmesg will show an error)." }, "wakeonlan": { "type": "boolean", @@ -257,7 +273,7 @@ "description": "Configure how ARP replies are to be validated when using ARP link monitoring.", "enum": [ "none", - "active", + "active", "backup", "all" ] @@ -356,7 +372,7 @@ }, "stp": { "type": "boolean", - "description": "Define whether the bridge should use Spanning Tree Protocol. The default value is “true”, which means that Spanning Tree should be used." + "description": "Define whether the bridge should use Spanning Tree Protocol. The default value is \u201ctrue\u201d, which means that Spanning Tree should be used." } } } @@ -393,13 +409,13 @@ ] }, "renderer": { - "$ref": "#/$defs/renderer" + "$ref": "#/$defs/renderer" }, "ethernets": { "type": "object", "additionalProperties": { "$ref": "#/$defs/mapping_physical" - } + } }, "bonds": { "type": "object", diff --git a/tox.ini b/tox.ini index 52d6505e9ad..a43ef53f3c2 100644 --- a/tox.ini +++ b/tox.ini @@ -41,6 +41,8 @@ typing-extensions==4.1.1 [files] schema = cloudinit/config/schemas/schema-cloud-config-v1.json version = cloudinit/config/schemas/versions.schema.cloud-config.json +network_v1 = cloudinit/config/schemas/schema-network-config-v1.json +network_v2 = cloudinit/config/schemas/schema-network-config-v2.json [testenv:ruff] deps = @@ -142,6 +144,8 @@ commands = {envpython} -m black . {envpython} -m json.tool --indent 2 {[files]schema} {[files]schema} {envpython} -m json.tool --indent 2 {[files]version} {[files]version} + {envpython} -m json.tool --indent 2 {[files]network_v1} {[files]network_v1} + {envpython} -m json.tool --indent 2 {[files]network_v2} {[files]network_v2} [testenv:do_format_tip] deps = From 8ad0bbaf6a35020eb78e002a58e17d64fc71a757 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Mon, 10 Jun 2024 17:52:35 -0600 Subject: [PATCH 20/33] type: Add stub types for network v1/v2 config (#5350) This doesn't add anything useful on its own, but it defines a custom type which can easily be overridden. See GH-5398 for more details. --- cloudinit/config/__init__.py | 2 ++ cloudinit/stages.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py index e5670257208..01da83fcffb 100644 --- a/cloudinit/config/__init__.py +++ b/cloudinit/config/__init__.py @@ -1 +1,3 @@ Config = dict +Netv1 = dict +Netv2 = dict diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 894eeac5960..52876e72434 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -11,7 +11,7 @@ import sys from collections import namedtuple from contextlib import suppress -from typing import Dict, Iterable, List, Optional, Set +from typing import Dict, Iterable, List, Optional, Set, Tuple, Union from cloudinit import ( atomic_helper, @@ -26,6 +26,7 @@ type_utils, util, ) +from cloudinit.config import Netv1, Netv2 from cloudinit.event import EventScope, EventType, userdata_to_events # Default handlers (used if not overridden) @@ -944,7 +945,7 @@ def _consume_userdata(self, frequency=PER_INSTANCE): # Run the handlers self._do_handlers(user_data_msg, c_handlers_list, frequency) - def _get_network_key_contents(self, cfg) -> dict: + def _get_network_key_contents(self, cfg) -> Union[Netv1, Netv2, None]: """ Network configuration can be passed as a dict under a "network" key, or optionally at the top level. In both cases, return the config. @@ -953,7 +954,9 @@ def _get_network_key_contents(self, cfg) -> dict: return cfg["network"] return cfg - def _find_networking_config(self): + def _find_networking_config( + self, + ) -> Tuple[Union[Netv1, Netv2, None], Union[NetworkConfigSource, str]]: disable_file = os.path.join( self.paths.get_cpath("data"), "upgraded-network" ) @@ -978,7 +981,9 @@ def _find_networking_config(self): order = sources.DataSource.network_config_sources for cfg_source in order: if not isinstance(cfg_source, NetworkConfigSource): - LOG.warning( + # This won't happen in the cloud-init codebase, but out-of-tree + # datasources might have an invalid type that mypy cannot know. + LOG.warning( # type: ignore "data source specifies an invalid network cfg_source: %s", cfg_source, ) From c95ec1a4f8bf3dd9c2c34c7f3daea6ffca125dad Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Thu, 20 Jun 2024 20:11:18 -0600 Subject: [PATCH 21/33] chore(mypy): Drop unused missing import exclusions (#5350) --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a0419d31666..7408488f975 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,6 @@ exclude = [] module = [ "apport.*", "BaseHTTPServer", - "cloudinit.feature_overrides", "configobj", "debconf", "httplib", @@ -33,7 +32,6 @@ module = [ "paramiko.*", "pip.*", "pycloudlib.*", - "responses", "serial", "tests.integration_tests.user_settings", "uaclient.*", From 2bca69a9a41e2f1e3e822cedd0860d2536e9cda3 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Mon, 1 Jul 2024 11:48:31 -0600 Subject: [PATCH 22/33] fix(test): Fix ip printing for non-lxd instances (#5350) This also satisfies mypy via type narrowing, which hasattr() does not accomplish. --- tests/integration_tests/instances.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py index bff28da7605..da7c4fb9ae3 100644 --- a/tests/integration_tests/instances.py +++ b/tests/integration_tests/instances.py @@ -8,6 +8,7 @@ from tempfile import NamedTemporaryFile from typing import Union +from pycloudlib.lxd.instance import LXDInstance from pycloudlib.gce.instance import GceInstance from pycloudlib.instance import BaseInstance from pycloudlib.result import Result @@ -289,7 +290,7 @@ def ip(self) -> str: try: # in some cases that ssh is not used, an address is not assigned if ( - hasattr(self.instance, "execute_via_ssh") + isinstance(self.instance, LXDInstance) and self.instance.execute_via_ssh ): self._ip = self.instance.ip From 291aabeb74c25b8af05449b481db197c36d8788a Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Mon, 1 Jul 2024 11:50:04 -0600 Subject: [PATCH 23/33] fix(test): Fix pycloudlib types in integration tests (#5350) --- tests/integration_tests/clouds.py | 31 ++++++++++--------- tests/integration_tests/instances.py | 2 +- .../modules/test_apt_functionality.py | 4 +-- .../modules/test_combined.py | 8 +++++ 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py index b7786edb280..82acde409e2 100644 --- a/tests/integration_tests/clouds.py +++ b/tests/integration_tests/clouds.py @@ -21,7 +21,7 @@ Openstack, Qemu, ) -from pycloudlib.cloud import BaseCloud, ImageType +from pycloudlib.cloud import ImageType from pycloudlib.ec2.instance import EC2Instance from pycloudlib.lxd.cloud import _BaseLXD from pycloudlib.lxd.instance import BaseInstance, LXDInstance @@ -55,7 +55,6 @@ def _get_ubuntu_series() -> list: class IntegrationCloud(ABC): datasource: str - cloud_instance: BaseCloud def __init__( self, @@ -64,7 +63,7 @@ def __init__( ): self._image_type = image_type self.settings = settings - self.cloud_instance: BaseCloud = self._get_cloud_instance() + self.cloud_instance = self._get_cloud_instance() self.initial_image_id = self._get_initial_image() self.snapshot_id = None @@ -183,7 +182,7 @@ def snapshot(self, instance): def delete_snapshot(self): if self.snapshot_id: - if self.settings.KEEP_IMAGE: + if self.settings.KEEP_IMAGE: # type: ignore log.info( "NOT deleting snapshot image created for this testrun " "because KEEP_IMAGE is True: %s", @@ -200,7 +199,7 @@ def delete_snapshot(self): class Ec2Cloud(IntegrationCloud): datasource = "ec2" - def _get_cloud_instance(self): + def _get_cloud_instance(self) -> EC2: return EC2(tag="ec2-integration-test") def _get_initial_image(self, **kwargs) -> str: @@ -228,7 +227,7 @@ def _perform_launch( class GceCloud(IntegrationCloud): datasource = "gce" - def _get_cloud_instance(self): + def _get_cloud_instance(self) -> GCE: return GCE( tag="gce-integration-test", ) @@ -243,7 +242,7 @@ class AzureCloud(IntegrationCloud): datasource = "azure" cloud_instance: Azure - def _get_cloud_instance(self): + def _get_cloud_instance(self) -> Azure: return Azure(tag="azure-integration-test") def _get_initial_image(self, **kwargs) -> str: @@ -265,7 +264,7 @@ def destroy(self): class OciCloud(IntegrationCloud): datasource = "oci" - def _get_cloud_instance(self): + def _get_cloud_instance(self) -> OCI: return OCI( tag="oci-integration-test", ) @@ -276,11 +275,7 @@ class _LxdIntegrationCloud(IntegrationCloud): instance_tag: str cloud_instance: _BaseLXD - def _get_cloud_instance(self): - return self.pycloudlib_instance_cls(tag=self.instance_tag) - - @staticmethod - def _get_or_set_profile_list(release): + def _get_or_set_profile_list(self, release): return None @staticmethod @@ -355,15 +350,21 @@ class LxdContainerCloud(_LxdIntegrationCloud): pycloudlib_instance_cls = LXDContainer instance_tag = "lxd-container-integration-test" + def _get_cloud_instance(self) -> LXDContainer: + return self.pycloudlib_instance_cls(tag=self.instance_tag) + class LxdVmCloud(_LxdIntegrationCloud): datasource = "lxd_vm" cloud_instance: LXDVirtualMachine pycloudlib_instance_cls = LXDVirtualMachine instance_tag = "lxd-vm-integration-test" - _profile_list = None + _profile_list: list = [] - def _get_or_set_profile_list(self, release): + def _get_cloud_instance(self) -> LXDVirtualMachine: + return self.pycloudlib_instance_cls(tag=self.instance_tag) + + def _get_or_set_profile_list(self, release) -> list: if self._profile_list: return self._profile_list self._profile_list = self.cloud_instance.build_necessary_profiles( diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py index da7c4fb9ae3..32281756cd1 100644 --- a/tests/integration_tests/instances.py +++ b/tests/integration_tests/instances.py @@ -8,9 +8,9 @@ from tempfile import NamedTemporaryFile from typing import Union -from pycloudlib.lxd.instance import LXDInstance from pycloudlib.gce.instance import GceInstance from pycloudlib.instance import BaseInstance +from pycloudlib.lxd.instance import LXDInstance from pycloudlib.result import Result from tests.helpers import cloud_init_project_dir diff --git a/tests/integration_tests/modules/test_apt_functionality.py b/tests/integration_tests/modules/test_apt_functionality.py index b86e217556e..2af9e590ce0 100644 --- a/tests/integration_tests/modules/test_apt_functionality.py +++ b/tests/integration_tests/modules/test_apt_functionality.py @@ -152,10 +152,10 @@ def get_keys(self, class_client: IntegrationInstance): keys = class_client.execute(list_cmd + cc_apt_configure.APT_LOCAL_KEYS) files = class_client.execute( "ls " + cc_apt_configure.APT_TRUSTED_GPG_DIR - ) + ).stdout for file in files.split(): path = cc_apt_configure.APT_TRUSTED_GPG_DIR + file - keys += class_client.execute(list_cmd + path) or "" + keys += class_client.execute(list_cmd + path).stdout class_client.execute("gpgconf --homedir /root/tmpdir --kill all") return keys diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 4a7c2dfc9e0..0b25a0fbe1a 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -13,6 +13,8 @@ from pathlib import Path import pytest +from pycloudlib.ec2.instance import EC2Instance +from pycloudlib.gce.instance import GceInstance import cloudinit.config from cloudinit.util import is_true, should_log_deprecation @@ -473,6 +475,9 @@ def test_instance_json_ec2(self, class_client: IntegrationInstance): "/run/cloud-init/cloud-id-aws" ) assert v1_data["subplatform"].startswith("metadata") + + # type narrow since availability_zone is not a BaseInstance attribute + assert isinstance(client.instance, EC2Instance) assert ( v1_data["availability_zone"] == client.instance.availability_zone ) @@ -495,6 +500,9 @@ def test_instance_json_gce(self, class_client: IntegrationInstance): "/run/cloud-init/cloud-id-gce" ) assert v1_data["subplatform"].startswith("metadata") + # type narrow since zone and instance_id are not BaseInstance + # attributes + assert isinstance(client.instance, GceInstance) assert v1_data["availability_zone"] == client.instance.zone assert v1_data["instance_id"] == client.instance.instance_id assert v1_data["local_hostname"] == client.instance.name From b0d6c7ddc4aa6adad064464e87ee0431037d57bd Mon Sep 17 00:00:00 2001 From: James Falcon Date: Tue, 2 Jul 2024 10:56:29 -0500 Subject: [PATCH 24/33] test: Add jsonschema guard in test_cc_ubuntu_pro.py (#5479) jsonschema is still a soft dependency, so don't fail it not present. --- tests/unittests/config/test_cc_ubuntu_pro.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unittests/config/test_cc_ubuntu_pro.py b/tests/unittests/config/test_cc_ubuntu_pro.py index f68a688f9fc..df47e7ae41e 100644 --- a/tests/unittests/config/test_cc_ubuntu_pro.py +++ b/tests/unittests/config/test_cc_ubuntu_pro.py @@ -5,7 +5,6 @@ import sys from collections import namedtuple -import jsonschema import pytest from cloudinit import subp @@ -28,6 +27,11 @@ from tests.unittests.helpers import does_not_raise, mock, skipUnlessJsonSchema from tests.unittests.util import get_cloud +try: + import jsonschema +except ImportError: + jsonschema = None # type: ignore + # Module path used in mocks MPATH = "cloudinit.config.cc_ubuntu_pro" From 0a7036ed748387e5855e301e194c77b3830f14a8 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 2 Jul 2024 07:12:31 -0600 Subject: [PATCH 25/33] fix: correct deprecated_version=22.2 for users.sudo Update test to reduce expected deprecation count when below deprecation boundary. changed_version messages are still logged. --- cloudinit/distros/__init__.py | 2 +- tests/integration_tests/modules/test_combined.py | 9 +++++++-- tests/unittests/distros/test_create_users.py | 6 +++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 3c956408090..4557d4320ee 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -847,7 +847,7 @@ def create_user(self, name, **kwargs): util.deprecate( deprecated=f"The value of 'false' in user {name}'s " "'sudo' config", - deprecated_version="22.3", + deprecated_version="22.2", extra_message="Use 'null' instead.", ) diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 0b25a0fbe1a..0bf1b3d49e8 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -136,18 +136,23 @@ def test_deprecated_message(self, class_client: IntegrationInstance): version_boundary = get_feature_flag_value( class_client, "DEPRECATION_INFO_BOUNDARY" ) - # the deprecation_version is 22.2 in schema for apt_* keys in + # the changed_version is 22.2 in schema for user.sudo key in # user-data. Pass 22.2 in against the client's version_boundary. if should_log_deprecation("22.2", version_boundary): log_level = "DEPRECATED" + deprecation_count = 2 else: + # Expect the distros deprecated call to be redacted. + # jsonschema still emits deprecation log due to changed_version + # instead of deprecated_version log_level = "INFO" + deprecation_count = 1 assert ( f"[{log_level}]: The value of 'false' in user craig's 'sudo'" " config is deprecated" in log ) - assert 2 == log.count("DEPRECATE") + assert deprecation_count == log.count("DEPRECATE") def test_ntp_with_apt(self, class_client: IntegrationInstance): """LP #1628337. diff --git a/tests/unittests/distros/test_create_users.py b/tests/unittests/distros/test_create_users.py index 893b47060d4..8fa7f0cc092 100644 --- a/tests/unittests/distros/test_create_users.py +++ b/tests/unittests/distros/test_create_users.py @@ -181,15 +181,15 @@ def test_explicit_sudo_false(self, m_subp, dist, caplog): expected_levels = ( ["WARNING", "DEPRECATED"] if should_log_deprecation( - "22.3", features.DEPRECATION_INFO_BOUNDARY + "22.2", features.DEPRECATION_INFO_BOUNDARY ) else ["INFO"] ) assert caplog.records[1].levelname in expected_levels assert ( "The value of 'false' in user foo_user's 'sudo' " - "config is deprecated in 22.3 and scheduled to be removed" - " in 27.3. Use 'null' instead." + "config is deprecated in 22.2 and scheduled to be removed" + " in 27.2. Use 'null' instead." ) in caplog.text def test_explicit_sudo_none(self, m_subp, dist, caplog): From 0a4c43d47cebd25c5f6d2f7631752349e956e32e Mon Sep 17 00:00:00 2001 From: James Falcon Date: Tue, 2 Jul 2024 13:22:28 -0500 Subject: [PATCH 26/33] test: Fix no default user in test_status.py (#5478) Also use ca_certs instead of ca-certs deprecated key --- tests/integration_tests/cmd/test_status.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration_tests/cmd/test_status.py b/tests/integration_tests/cmd/test_status.py index 84d9b694f63..23509c57cef 100644 --- a/tests/integration_tests/cmd/test_status.py +++ b/tests/integration_tests/cmd/test_status.py @@ -66,7 +66,8 @@ def test_wait_when_no_datasource(session_cloud: IntegrationCloud, setup_image): users: - name: something ssh-authorized-keys: ["something"] -ca-certs: + - default +ca_certs: invalid_key: true """ From fd63297550a3c153b2f92720100a1fa5d266144c Mon Sep 17 00:00:00 2001 From: James Falcon Date: Tue, 2 Jul 2024 20:44:08 -0500 Subject: [PATCH 27/33] Release 24.2 (#5481) Bump the version in cloudinit/version.py to 24.2 and update ChangeLog. --- ChangeLog | 364 +++++++++++++++++++++++++++++++++++++++++++ cloudinit/version.py | 2 +- 2 files changed, 365 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 880d134750f..260c460df46 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,367 @@ +24.2 + - test: Fix no default user in test_status.py (#5478) + - fix: correct deprecated_version=22.2 for users.sudo + - test: Add jsonschema guard in test_cc_ubuntu_pro.py (#5479) + - fix(test): Fix pycloudlib types in integration tests (#5350) + - fix(test): Fix ip printing for non-lxd instances (#5350) + - chore(mypy): Drop unused missing import exclusions (#5350) + - type: Add stub types for network v1/v2 config (#5350) + - chore: Auto-format network jsonschema in ci (#5350) + - fix(tox): Update tox.ini (#5350) + - chore(typing): Remove type ignores and casts (#5350) + - refactor(typing): Remove unused code paths (#5350) + - fix(typing): Add / update type annotations (#5350) + - fix(typing): Remove type annotation for unused variable (#5350) + - fix(typing): Remove invalid type annotations (#5350) + - ci(mypy): Set default follow_imports value (#5350) + - test: Update integration tests to pass on focal (#5476) + - tests: update ubuntu_pro test to account for info-level deprecations + (#5475) + - tests: update nocloud deprecation test for boundary version (#5474) + - fix(rh_subscription): add string type to org (#5453) + - tests: integration tests aware of features.DEPRECATION_INFO_BOUNDARY + - tests: update keyserver PPA key fur curtin-dev (#5472) + - test: Fix deprecation test failures (#5466) + - chore: fix schema.py formatting (#5465) + - fix: dont double-log deprecated INFOs (#5465) + - fix(test): Mock version boundary (#5464) + - fix(schema): Don't report changed keys as deprecated (#5464) + - test: fix unit test openstack vlan mac_address (#5367) + - fix: Ensure properties for bonded interfaces are properly translated + (#5367) [Curt Moore] + - fix(schema): permit deprecated hyphenated keys under users key (#5456) + - fix: Do not add the vlan_mac_address field into the VLAN object (#5365) + [Curt Moore] + - doc(refactor): Convert module docs to new system (#5427) [Sally] + - test: Add unit tests for features.DEPRECATION_INFO_BOUNDARY (#5411) + - feat: Add deprecation boundary support to schema validator (#5411) + - feat: Add deprecation boundary to logger (#5411) + - fix: Gracefully handle missing files (#5397) [Curt Moore] + - test(openstack): Test bond mac address (#5369) + - fix(openstack): Fix bond mac_address (#5369) [Curt Moore] + - test: Add ds-identify integration test coverage (#5394) + - chore(cmdline): Update comments (#5458) + - fix: Add get_connection_with_tls_context() for requests 2.32.2+ (#5435) + [eaglegai] + - fix(net): klibc ipconfig PROTO compatibility (#5437) + [Alexsander de Souza] (LP: #2065787) + - Support metalink in yum repository config (#5444) [Ani Sinha] + - tests: hard-code curtin-dev ppa instead of canonical-kernel-team (#5450) + - ci: PR update checklist GH- anchors to align w/ later template (#5449) + - test: update validate error message in test_networking (#5436) + - ci: Add PR checklist (#5446) + - chore: fix W0105 in t/u/s/h/test_netlink.py (#5409) + - chore(pyproject.toml): migrate to booleans (#5409) + - typing: add check_untyped_defs (#5409) + - fix(openstack): Append interface / scope_id for IPv6 link-local metadata + address (#5419) [Christian Rohmann] + - test: Update validation error in test_cli.py test (#5430) + - test: Update schema validation error in integration test (#5429) + - test: bump pycloudlib to get azure oracular images (#5428) + - fix(azure): fix discrepancy for monotonic() vs time() (#5420) + [Chris Patterson] + - fix(pytest): Fix broken pytest gdb flag (#5415) + - fix: Use monotonic time (#5423) + - docs: Remove mention of resolv.conf (#5424) + - perf(netplan): Improve network v1 -> network v2 performance (#5391) + - perf(set_passwords): Run module in Network stage (#5395) + - fix(test): Remove temporary directory side effect (#5416) + - Improve schema validator warning messages (#5404) [Ani Sinha] + - feat(sysconfig): Add DNS from interface config to resolv.conf (#5401) + [Ani Sinha] + - typing: add no_implicit_optional lint (#5408) + - doc: update examples to reflect alternative ways to provide `sudo` + option (#5418) [Ani Sinha] + - fix(jsonschema): Add missing sudo definition (#5418) + - chore(doc): migrate cc modules i through r to templates (#5313) + - chore(doc): migrate grub_dpkg to tmpl add changed/deprecation (#5313) + - chore(json): migrate cc_apt_configure and json schema indents (#5313) + - chore(doc): migrate ca_certs/chef to template, flatten schema (#5313) + - chore(doc): migrate cc_byobu to templates (#5313) + - chore(doc): migrate cc_bootcmd to templates (#5313) + - fix(apt): Enable calling apt update multiple times (#5230) + - chore(VMware): Modify section of instance-id in the customization config + (#5356) [PengpengSun] + - fix(treewide): Remove dead code (#5332) [Shreenidhi Shedi] + - doc: network-config v2 ethernets are of type object (#5381) [Malte Poll] + - Release 24.1.7 (#5375) + - fix(azure): url_helper: specify User-Agent when using headers_cb with + readurl() (#5298) [Ksenija Stanojevic] + - fix: Stop attempting to resize ZFS in cc_growpart on Linux (#5370) + - doc: update docs adding YAML 1.1 spec and jinja template references + - fix(final_message): do not warn on datasourcenone when single ds + - fix(growpart): correct growpart log message to include value of mode + - feat(hotplug): disable hotplugd.socket (#5058) + - feat(hotlug): trigger hotplug after cloud-init.service (#5058) + - test: add function to push and enable systemd units (#5058) + - test(util): fix wait_until_cloud_init exit code 2 (#5058) + - test(hotplug): fix race getting ipv6 (#5271) + - docs: Adjust CSS to increase font weight across the docs (#5363) [Sally] + - fix(ec2): Correctly identify netplan renderer (#5361) + - tests: fix expect logging from growpart on devent with partition (#5360) + - test: Add v2 test coverage to test_net.py (#5247) + - refactor: Simplify collect_logs() in logs.py (#5268) + - fix: Ensure no subp from logs.py import (#5268) + - tests: fix integration tests for ubuntu pro 32.3 release (#5351) + - tests: add oracular's hello package for pkg upgrade test (#5354) + - growpart: Fix behaviour for ZFS datasets (#5169) [Mina Galić] + - device_part_info: do not recurse if we did not match anything (#5169) + [Mina Galić] + - feat(alpine): add support for Busybox adduser/addgroup (#5176) + [dermotbradley] + - ci: Move lint tip and py3-dev jobs to daily (#5347) + - fix(netplan): treat netplan warnings on stderr as debug for cloud-init + (#5348) + - feat(disk_setup): Add support for nvme devices (#5263) + - fix(log): Do not warn when doing requested operation (#5263) + - Support sudoers in the "/usr/usr merge" location (#5161) + [Robert Schweikert] + - doc(nocloud): Document network-config file (#5204) + - fix(netplan): Fix predictable interface rename issue (#5339) + - cleanup: Don't execute code on import (#5295) + - fix(net): Make duplicate route add succeed. (#5343) + - fix(freebsd): correct configuration of IPv6 routes (#5291) [Théo Bertin] + - fix(azure): disable use-dns for secondary nics (#5314) + - chore: fix lint failure (#5320) + - Update pylint version to support python 3.12 (#5338) [Ani Sinha] + - fix(tests): use regex to avoid focal whitespace in jinja debug test + (#5335) + - chore: Add docstrings and types to Version class (#5262) + - ci(mypy): add type-jinja2 stubs (#5337) + - tests(alpine): github trust lxc mounted source dir cloud-init-ro (#5329) + - test: Add oracular release to integration tests (#5328) + - Release 24.1.6 (#5326) + - test: Fix failing test_ec2.py test (#5324) + - fix: Check renderer for netplan-specific code (#5321) + - docs: Removal of top-level --file breaking change (#5308) + - fix: typo correction of delaycompress (#5317) + - docs: Renderers/Activators have downstream overrides (#5322) + - fix(ec2): Ensure metadata exists before configuring PBR (#5287) + - fix(lxd): Properly handle unicode from LXD socket (#5309) + - docs: Prefer "artifact" over "artefact" (#5311) [Arthur Le Maitre] + - chore(doc): migrate cc_byobu to templates + - chore(doc): migrate cc_bootcmd to templates + - chore(doc): migrate apt_pipelining and apk_configure to templates + - tests: in_place mount module-docs into lxd vm/container + - feat(docs): generate rtd module schema from rtd/module-docs + - feat: Set RH ssh key permissions when no 'ssh_keys' group (#5296) + [Ani Sinha] + - test: Avoid circular import in Azure tests (#5280) + - test: Fix test_failing_userdata_modules_exit_codes (#5279) + - chore: Remove CPY check from ruff (#5281) + - chore: Clean up docstrings + - chore(ruff): Bump to version 0.4.3 + - feat(systemd): Improve AlmaLinux OS and CloudLinux OS support (#5265) + [Elkhan Mammadli] + - feat(ca_certs): Add AlmaLinux OS and CloudLinux OS support (#5264) + [Elkhan Mammadli] + - docs: cc_apt_pipelining docstring typo fix (#5273) [Alex Ratner] + - feat(azure): add request identifier to IMDS requests (#5218) + [Ksenija Stanojevic] + - test: Fix TestFTP integration test (#5237) [d1r3ct0r] + - feat(ifconfig): prepare for CIDR output (#5272) [Mina Galić] + - fix: stop manually dropping dhcp6 key in integration test (#5267) + [Alec Warren] + - test: Remove some CiTestCase tests (#5256) + - fix: Warn when signal is handled (#5186) + - fix(snapd): ubuntu do not snap refresh when snap absent (LP: #2064300) + - feat(landscape-client): handle already registered client (#4784) + [Fabian Lichtenegger-Lukas] + - doc: Show how to debug external services blocking cloud-init (#5255) + - fix(pdb): Enable running cloud-init under pdb (#5217) + - chore: Update systemd description (#5250) + - fix(time): Harden cloud-init to system clock changes + - fix: Update analyze timestamp uptime + - fix(schema): no network validation on netplan systems without API + - fix(mount): Don't run cloud-init.service if cloud-init disabled (#5226) + - fix(ntp): Fix AlmaLinux OS and CloudLinux OS support (#5235) + [Elkhan Mammadli] + - tests: force version of cloud-init from PPA regardless of version (#5251) + - ci: Print isort diff (#5242) + - test: Fix integration test dependencies (#5248) + - fix(ec2): Fix broken uuid match with other-endianness (#5236) + - fix(schema): allow networkv2 schema without top-level key (#5239) + [Cat Red] + - fix(cmd): Do not hardcode reboot command (#5208) + - test: Run Alpine tests without network (#5220) + - docs: Add base config reference from explanation (#5241) + - docs: Remove preview from WSL tutorial (#5225) + - chore: Remove broken maas code (#5219) + - feat(WSL): Add support for Ubuntu Pro configs (#5116) [Ash] + - chore: sync ChangeLog and version.py from 24.1.x (#5228) + - bug(package_update): avoid snap refresh in images without snap command + (LP: #2064132) + - ci: Skip package build on tox runs (#5210) + - chore: Fix test skip message + - test(ec2): adopt pycloudlib public ip creation while launching instances + - test(ec2): add ipv6 testing for multi-nic instances + - test(ec2): adopt pycloudlib enable_ipv6 while launching instances + - feat: tool to print diff between netplan and networkv2 schema (#5200) + [Cat Red] + - test: mock internet access in test_upgrade (#5212) + - ci: Add timezone for alpine unit tests (#5216) + - fix: Ensure dump timestamps parsed as UTC (#5214) + - docs: Add WSL tutorial (#5206) + - feature(schema): add networkv2 schema (#4892) [Cat Red] + - Add alpine unittests to ci (#5121) + - test: Fix invalid openstack datasource name (#4905) + - test: Fix MAAS test and mark xfail (#4905) + - chore(ds-identify): Update shellcheck ignores (#4905) + - fix(ds-identify): Prevent various false positives and false negatives + (#4905) + - Use grep for faster parsing of cloud config in ds-identify (#4905) + [Scott Moser] (LP: #2030729) + - tests: validate netplan API YAML instead of strict content (#5195) + - chore(templates): update ubuntu universe wording (#5199) + - Deprecate the users ssh-authorized-keys property (#5162) + [Anders Björklund] + - doc(nocloud): Describe ftp and ftp over tls implementation (#5193) + - feat(net): provide network config to netplan.State for render (#4981) + - docs: Add breaking datasource identification changes (#5171) + - fix(openbsd): Update build-on-openbsd python dependencies (#5172) + [Hyacinthe Cartiaux] + - fix: Add subnet ipv4/ipv6 to network schema (#5191) + - docs: Add deprecated system_info to schema (#5168) + - docs: Add DataSourceNone documentation (#5165) + - test: Skip test if console log is None (#5188) + - fix(dhcp): Enable interactively running cloud-init init --local (#5166) + - test: Update message for netplan apply dbus issue + - test: install software-properties-common if absent during PPA setup + - test: bump pycloudlib to use latest version + - test: Update version of hello package installed on noble + - test: universally ignore netplan apply dbus issue (#5178) + - chore: Remove obsolete nose workaround + - feat: Add support for FTP and FTP over TLS (#4834) + - feat(opennebula): Add support for posix shell + - test: Make analyze tests not depend on GNU date + - test: Eliminate bash dependency from subp tests + - docs: Add breaking changes section to reference docs (#5147) [Cat Red] + - util: add log_level kwarg for logexc() (#5125) [Chris Patterson] + - refactor: Make device info part of distro definition (#5067) + - refactor: Distro-specific growpart code (#5067) + - test(ec2): fix mocking with responses==0.9.0 (focal) (#5163) + - chore(safeyaml): Remove unicode helper for Python2 (#5142) + - Revert "test: fix upgrade dhcp6 on ec2 (#5131)" (#5148) + - refactor(net): Reuse netops code + - refactor(iproute2): Make expressions multi-line for legibility + - feat(freebsd): support freebsd find part by gptid and ufsid (#5122) + [jinkangkang] + - feat: Determining route metric based on NIC name (#5070) [qidong.ld] + - test: Enable profiling in integration tests (#5130) + - dhcp: support configuring static routes for dhclient's unknown-121 + option (#5146) [Chris Patterson] + - feat(azure): parse ProvisionGuestProxyAgent as bool (#5126) + [Ksenija Stanojevic] + - fix(url_helper): fix TCP connection leak on readurl() retries (#5144) + [Chris Patterson] + - test: pytest-ify t/u/sources/test_ec2.py + - Revert "ec2: Do not enable dhcp6 on EC2 (#5104)" (#5145) [Major Hayden] + - fix: Logging sensitive data + - test: Mock ds-identify systemd path (#5119) + - fix(dhcpcd): Make lease parsing more robust (#5129) + - test: fix upgrade dhcp6 on ec2 (#5131) + - net/dhcp: raise InvalidDHCPLeaseFileError on error parsing dhcpcd lease + (#5128) [Chris Patterson] + - fix: Fix runtime file locations for cloud-init (#4820) + - ci: fix linkcheck.yml invalid yaml (#5123) + - net/dhcp: bump dhcpcd timeout to 300s (#5127) [Chris Patterson] + - ec2: Do not enable dhcp6 on EC2 (#5104) [Major Hayden] + - fix: Fall back to cached local ds if no valid ds found (#4997) + [PengpengSun] + - ci: Make linkcheck a scheduled job (#5118) + - net: Warn when interface rename fails + - ephemeral(dhcpcd): Set dhcpcd interface down + - Release 24.1.3 + - chore: Handle all level 1 TiCS security violations (#5103) + - fix: Always use single datasource if specified (#5098) + - fix(tests): Leaked mocks (#5097) + - fix(rhel)!: Fix network boot order in upstream cloud-init + - fix(rhel): Fix network ordering in sysconfig + - feat: Use NetworkManager renderer by default in RHEL family + - fix: Allow caret at the end of apt package (#5099) + - test: Add missing mocks to prevent bleed through (#5082) + [Robert Schweikert] + - fix: Ensure network config in DataSourceOracle can be unpickled (#5073) + - docs: set the home directory using homedir, not home (#5101) + [Olivier Gayot] (LP: #2047796) + - fix(cacerts): Correct configuration customizations for Photon (#5077) + [Christopher McCann] + - fix(test): Mock systemd fs path for non-systemd distros + - fix(tests): Leaked subp.which mock + - fix(networkd): add GatewayOnLink flag when necessary (#4996) [王煎饼] + - Release 24.1.2 + - test: fix `disable_sysfs_net` mock (#5065) + - refactor: don't import subp function directly (#5065) + - test: Remove side effects from tests (#5074) + - refactor: Import log module rather than functions (#5074) + - fix: Fix breaking changes in package install (#5069) + - fix: Undeprecate 'network' in schema route definition (#5072) + - refactor(ec2): simplify convert_ec2_metadata_network_config + - fix(ec2): fix ipv6 policy routing + - fix: document and add 'accept-ra' to network schema (#5060) + - bug(maas): register the correct DatasourceMAASLocal in init-local + (#5068) (LP: #2057763) + - ds-identify: Improve ds-identify testing flexibility (#5047) + - fix(ansible): Add verify_commit and inventory to ansible.pull schema + (#5032) [Fionn Fitzmaurice] + - doc: Explain breaking change in status code (#5049) + - gpg: Handle temp directory containing files (#5063) + - distro(freebsd): add_user: respect homedir (#5061) [Mina Galić] + - doc: Install required dependencies (#5054) + - networkd: Always respect accept-ra if set (#4928) [Phil Sphicas] + - chore: ignore all cloud-init_*.tar.gz in .gitignore (#5059) + - test: Don't assume ordering of ThreadPoolExecutor submissions (#5052) + - feat: Add new distro 'azurelinux' for Microsoft Azure Linux. (#4931) + [Dan Streetman] + - fix(gpg): Make gpg resilient to host configuration changes (#5026) + - Sync 24.1.1 changelog and version + - DS VMware: Fix ipv6 addr converter from netinfo to netifaces (#5029) + [PengpengSun] + - packages/debian: remove dependency on isc-dhcp-client (#5041) + [Chris Patterson] + - test: Allow fake_filesystem to work with TemporaryDirectory (#5035) + - tests: Don't wait for GCE instance teardown (#5037) + - fix: Include DataSourceCloudStack attribute in unpickle test (#5039) + - bug(vmware): initialize new DataSourceVMware attributes at unpickle + (#5021) (LP: #2056439) + - fix(apt): Don't warn on apt 822 source format (#5028) + - fix(atomic_helper.py): ensure presence of parent directories (#4938) + [Shreenidhi Shedi] + - fix: Add "broadcast" to network v1 schema (#5034) (LP: #2056460) + - pro: honor but warn on custom ubuntu_advantage in /etc/cloud/cloud.cfg + (#5030) + - net/dhcp: handle timeouts for dhcpcd (#5022) [Chris Patterson] + - fix: Make wait_for_url respect explicit arguments + - test: Fix scaleway retry assumptions + - fix: Make DataSourceOracle more resilient to early network issues + (#5025) (LP: #2056194) + - chore(cmd-modules): fix exit code when --mode init (#5017) + - feat: pylint: enable W0201 - attribute-defined-outside-init + - refactor: Ensure no attributes defined outside __init__ + - chore: disable attribute-defined-outside-init check in tests + - refactor: Use _unpickle rather than hasattr() in sources + - chore: remove unused vendordata "_pure" variables + - chore(cmd-modules): deprecate --mode init (#5005) + - tests: drop CiTestCase and convert to pytest + - bug(tests): mock reads of host's /sys/class/net via get_sys_class_path + - fix: log correct disabled path in ds-identify (#5016) + - tests: ec2 dont spend > 1 second retrying 19 times when 3 times will do + - tests: openstack mock expected ipv6 IMDS + - bug(wait_for_url): when exceptions occur url is unset, use url_exc + (LP: #2055077) + - feat(run-container): Run from arbitrary commitish (#5015) + - tests: Fix wsl test (#5008) + - feat(ds-identify): Don't run unnecessary systemd-detect-virt (#4633) + - chore(ephemeral): add debug log when bringing up ephemeral network + (#5010) [Alec Warren] + - release: sync changelog and version (#5011) + - Cleanup test_net.py (#4840) + - refactor: remove dependency on netifaces (#4634) [Cat Red] + - feat: make lxc binary configurable (#5000) + - docs: update 404 page for new doc site and bug link + - test(aws): local network connectivity on multi-nics (#4982) + - test: Make integration test output more useful (#4984) + 24.1.7 - fix(ec2): Correctly identify netplan renderer (#5361) diff --git a/cloudinit/version.py b/cloudinit/version.py index 9b141b4f87e..b6bc8227d66 100644 --- a/cloudinit/version.py +++ b/cloudinit/version.py @@ -4,7 +4,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. -__VERSION__ = "24.1.7" +__VERSION__ = "24.2" _PACKAGED_VERSION = "@@PACKAGED_VERSION@@" FEATURES = [ From 359a1953539673187599694d29419ff5ecdf7578 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 2 Jul 2024 20:19:12 -0600 Subject: [PATCH 28/33] update changelog --- debian/changelog | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 9ef10614a86..cc55ae5064b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,8 @@ cloud-init (24.1.3-0ubuntu1~20.04.4) UNRELEASED; urgency=medium * d/apport-general-hook.py: Move apport hook to main branch - * d/cloud-init.maintscript: remove /etc/cloud/clean.d/README * d/cloud-init.logrotate: add logrotate config for cloud-init + * d/cloud-init.maintscript: remove /etc/cloud/clean.d/README * d/cloud-init.postinst: change priority of hotplug rules. Avoids LP #1946003 on upgraded systems. References: [0] https://github.com/canonical/cloud-init/pull/4799 @@ -14,8 +14,6 @@ cloud-init (24.1.3-0ubuntu1~20.04.4) UNRELEASED; urgency=medium * d/p/drop-unsupported-systemd-condition-environment.patch: drop ConditionEnvironment from unit files because systemd 245.4 ignores those keys and emits warnings at systemctl status - * d/p/revert-551f560d-cloud-config-after-snap-seeding.patch: retain systemd - ordering cloud-config.service After=snapd.seeded.service * d/p/add-deprecation-info-boundary.patch: Update DEPRECATION_INFO_BOUNDARY to ensure new deprecations don't trigger warnings. From 335e243c3457e941ea5d9a39ad6eee113b4e7f81 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 2 Jul 2024 20:19:44 -0600 Subject: [PATCH 29/33] d/changelog: sync released changelog from focal-24.1.x branch --- debian/changelog | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/debian/changelog b/debian/changelog index cc55ae5064b..cf125c2dac1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -31,6 +31,29 @@ cloud-init (24.1.3-0ubuntu1~20.04.4) UNRELEASED; urgency=medium -- James Falcon Fri, 28 Jun 2024 11:17:04 -0500 +cloud-init (24.1.3-0ubuntu1~20.04.5) focal; urgency=medium + + * Upstream bug fix release based on 24.1.7 + + functional fixes in debian/patches: + - cpick-417ee551: fix(ec2): Ensure metadata exists before configuring PBR. + (LP: #2066979) + - cpick-d6776632: fix: Check renderer for netplan-specific code (#5321) + (LP: #2066985) + - cpick d771d1f4: fix(ec2): Correctly identify netplan renderer (#5361) + (LP: #2066985) + + test fixes in debian/patches: + - cpick-74dc7cce: test: Fix failing test_ec2.py test (#5324) + + -- James Falcon Wed, 05 Jun 2024 12:40:38 -0500 + +cloud-init (24.1.3-0ubuntu1~20.04.4) focal; urgency=medium + + * cherry-pick 51c6569f: fix(snapd): ubuntu do not snap refresh when + snap absent (LP: #2064132) + - fix in 24.1.3-0ubuntu1~20.04.2 did not handle package_upgrade case + + -- Chad Smith Fri, 03 May 2024 15:38:58 -0600 + cloud-init (24.1.3-0ubuntu1~20.04.3) focal; urgency=medium * d/p/cli-retain-file-argument-as-main-cmd-arg.patch: retain ability to From 7a2163c281511d7a03e0089931520fb92bcaa9ad Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 2 Jul 2024 20:20:58 -0600 Subject: [PATCH 30/33] fix(systemd): drop unsupported CondtionEnvironment from hotplug Systemd on focal does not support ConditionEnvironment in units which raises a warning in journalctl. Drop these unused conditions from cloud-init-hotplug.service and socket units. Fixes GH-5402 --- ...ported-systemd-condition-environment.patch | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/debian/patches/drop-unsupported-systemd-condition-environment.patch b/debian/patches/drop-unsupported-systemd-condition-environment.patch index 09f2a65a2e4..4f4eabc35c5 100644 --- a/debian/patches/drop-unsupported-systemd-condition-environment.patch +++ b/debian/patches/drop-unsupported-systemd-condition-environment.patch @@ -55,3 +55,23 @@ This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ ConditionPathExists=!/etc/cloud/cloud-init.disabled ConditionKernelCommandLine=!cloud-init=disabled -ConditionEnvironment=!KERNEL_CMDLINE=cloud-init=disabled +--- a/systemd/cloud-init-hotplugd.service ++++ b/systemd/cloud-init-hotplugd.service +@@ -16,7 +16,6 @@ After=cloud-init.target + Requires=cloud-init-hotplugd.socket + ConditionPathExists=!/etc/cloud/cloud-init.disabled + ConditionKernelCommandLine=!cloud-init=disabled +-ConditionEnvironment=!KERNEL_CMDLINE=cloud-init=disabled + + [Service] + Type=oneshot +--- a/systemd/cloud-init-hotplugd.socket ++++ b/systemd/cloud-init-hotplugd.socket +@@ -8,7 +8,6 @@ Description=cloud-init hotplug hook sock + After=cloud-config.target + ConditionPathExists=!/etc/cloud/cloud-init.disabled + ConditionKernelCommandLine=!cloud-init=disabled +-ConditionEnvironment=!KERNEL_CMDLINE=cloud-init=disabled + + [Socket] + ListenFIFO=/run/cloud-init/hook-hotplug-cmd From ba162418d19d9c6bd52f947cd9b59f82c075b992 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 2 Jul 2024 20:22:51 -0600 Subject: [PATCH 31/33] refresh patches against 24.2 patches: debian/patches/deprecation-version-boundary.patch debian/patches/revert-551f560d-cloud-config-after-snap-seeding.patch --- debian/patches/deprecation-version-boundary.patch | 4 ++-- .../revert-551f560d-cloud-config-after-snap-seeding.patch | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/debian/patches/deprecation-version-boundary.patch b/debian/patches/deprecation-version-boundary.patch index a4e980bd2dd..1e2a7804a17 100644 --- a/debian/patches/deprecation-version-boundary.patch +++ b/debian/patches/deprecation-version-boundary.patch @@ -7,10 +7,10 @@ Author: James Falcon Last-Update: 2024-06-28 --- a/cloudinit/features.py +++ b/cloudinit/features.py -@@ -87,7 +87,7 @@ On Debian and Ubuntu systems, cc_apt_configure will write a deb822 compatible +@@ -87,7 +87,7 @@ On Debian and Ubuntu systems, cc_apt_con to write /etc/apt/sources.list directly. """ - + -DEPRECATION_INFO_BOUNDARY = "devel" +DEPRECATION_INFO_BOUNDARY = "20.1" """ diff --git a/debian/patches/revert-551f560d-cloud-config-after-snap-seeding.patch b/debian/patches/revert-551f560d-cloud-config-after-snap-seeding.patch index d6523e0efdb..a338d91f217 100644 --- a/debian/patches/revert-551f560d-cloud-config-after-snap-seeding.patch +++ b/debian/patches/revert-551f560d-cloud-config-after-snap-seeding.patch @@ -27,7 +27,7 @@ Last-Update: 2024-02-14 def get_template_filename(self, name): --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py -@@ -81,7 +81,6 @@ def handle(name: str, cfg: Config, cloud +@@ -79,7 +79,6 @@ def handle(name: str, cfg: Config, cloud f" '{type(lxd_cfg).__name__}'" ) @@ -67,7 +67,7 @@ Last-Update: 2024-02-14 if TYPE_CHECKING: # Avoid circular import -@@ -3066,18 +3066,6 @@ def wait_for_files(flist, maxwait, naple +@@ -3064,18 +3064,6 @@ def wait_for_files(flist, maxwait, naple return need From 99f7411d9173d91a6c02ecdb24ea3e015250ee78 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 2 Jul 2024 20:25:25 -0600 Subject: [PATCH 32/33] update changelog (new upstream snapshot) --- debian/changelog | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index cf125c2dac1..dc95ca12af2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -cloud-init (24.1.3-0ubuntu1~20.04.4) UNRELEASED; urgency=medium +cloud-init (24.2-0ubuntu1~20.04.1) UNRELEASED; urgency=medium * d/apport-general-hook.py: Move apport hook to main branch * d/cloud-init.logrotate: add logrotate config for cloud-init @@ -27,9 +27,11 @@ cloud-init (24.1.3-0ubuntu1~20.04.4) UNRELEASED; urgency=medium - d/p/status-do-not-remove-duplicated-data.patch - d/p/status-retain-recoverable-error-exit-code.patch - d/p/revert-551f560d-cloud-config-after-snap-seeding.patch - * Upstream snapshot based on upstream/main at debafbc9. + * Upstream snapshot based on 24.2. (LP: #2071762). + List of changes from upstream can be found at + https://raw.githubusercontent.com/canonical/cloud-init/24.2/ChangeLog - -- James Falcon Fri, 28 Jun 2024 11:17:04 -0500 + -- Chad Smith Tue, 02 Jul 2024 20:25:25 -0600 cloud-init (24.1.3-0ubuntu1~20.04.5) focal; urgency=medium From f8dccc37296aa25e424ada8ec761d0ed222e45a6 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 2 Jul 2024 20:25:34 -0600 Subject: [PATCH 33/33] releasing cloud-init version 24.2-0ubuntu1~20.04.1 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index dc95ca12af2..bcbe94707fc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -cloud-init (24.2-0ubuntu1~20.04.1) UNRELEASED; urgency=medium +cloud-init (24.2-0ubuntu1~20.04.1) focal; urgency=medium * d/apport-general-hook.py: Move apport hook to main branch * d/cloud-init.logrotate: add logrotate config for cloud-init @@ -31,7 +31,7 @@ cloud-init (24.2-0ubuntu1~20.04.1) UNRELEASED; urgency=medium List of changes from upstream can be found at https://raw.githubusercontent.com/canonical/cloud-init/24.2/ChangeLog - -- Chad Smith Tue, 02 Jul 2024 20:25:25 -0600 + -- Chad Smith Tue, 02 Jul 2024 20:25:32 -0600 cloud-init (24.1.3-0ubuntu1~20.04.5) focal; urgency=medium