From facd9a267980bf3f1887c595d9c2e369c261bf68 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Sun, 23 Jan 2022 20:44:16 -0700 Subject: [PATCH 01/13] schema: migrate legacy cc_bootcmd schema definition Move definition into cloud-init-schema.json --- cloudinit/config/cc_bootcmd.py | 23 +---- cloudinit/config/cloud-init-schema.json | 19 +++- doc/examples/cloud-config-disk-setup.txt | 2 +- tests/unittests/config/test_cc_bootcmd.py | 100 +++++++++++----------- tests/unittests/config/test_schema.py | 4 +- 5 files changed, 72 insertions(+), 76 deletions(-) diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index bff11a247d1..6585bfbf38a 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -13,7 +13,7 @@ from textwrap import dedent from cloudinit import subp, temp_utils, util -from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema +from cloudinit.config.schema import get_meta_doc from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS @@ -62,25 +62,7 @@ "frequency": PER_ALWAYS, } -schema = { - "type": "object", - "properties": { - "bootcmd": { - "type": "array", - "items": { - "oneOf": [ - {"type": "array", "items": {"type": "string"}}, - {"type": "string"}, - ] - }, - "additionalItems": False, # Reject items of non-string non-list - "additionalProperties": False, - "minItems": 1, - } - }, -} - -__doc__ = get_meta_doc(meta, schema) # Supplement python help() +__doc__ = get_meta_doc(meta) def handle(name, cfg, cloud, log, _args): @@ -91,7 +73,6 @@ def handle(name, cfg, cloud, log, _args): ) return - validate_cloudconfig_schema(cfg, schema) with temp_utils.ExtendedTemporaryFile(suffix=".sh") as tmpf: try: content = util.shellify(cfg["bootcmd"]) diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index afaed285ece..5385a8f9d26 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -60,10 +60,27 @@ ] } } + }, + "cc_bootcmd": { + "type": "object", + "properties": { + "bootcmd": { + "type": "array", + "items": { + "oneOf": [ + {"type": "array", "items": {"type": "string"}}, + {"type": "string"} + ] + }, + "additionalItems": false, + "minItems": 1 + } + } } }, "allOf": [ { "$ref": "#/$defs/cc_apk_configure" }, - { "$ref": "#/$defs/cc_apt_pipelining" } + { "$ref": "#/$defs/cc_apt_pipelining" }, + { "$ref": "#/$defs/cc_bootcmd" } ] } diff --git a/doc/examples/cloud-config-disk-setup.txt b/doc/examples/cloud-config-disk-setup.txt index 5c6de77e319..a36e6cfbad9 100644 --- a/doc/examples/cloud-config-disk-setup.txt +++ b/doc/examples/cloud-config-disk-setup.txt @@ -80,7 +80,7 @@ fs_setup: disk_setup: ephmeral0: table_type: 'mbr' - layout: 'auto' + layout: true /dev/xvdh: table_type: 'mbr' layout: diff --git a/tests/unittests/config/test_cc_bootcmd.py b/tests/unittests/config/test_cc_bootcmd.py index 17033596e3c..34b16b857a5 100644 --- a/tests/unittests/config/test_cc_bootcmd.py +++ b/tests/unittests/config/test_cc_bootcmd.py @@ -1,15 +1,18 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging +import re import tempfile +import pytest + from cloudinit import subp, util -from cloudinit.config.cc_bootcmd import handle, schema -from tests.unittests.helpers import ( - CiTestCase, - SchemaTestCaseMixin, - mock, - skipUnlessJsonSchema, +from cloudinit.config.cc_bootcmd import handle +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, ) +from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema from tests.unittests.util import get_cloud LOG = logging.getLogger(__name__) @@ -65,44 +68,13 @@ def test_handler_invalid_command_set(self): str(context_manager.exception), ) - @skipUnlessJsonSchema() - def test_handler_schema_validation_warns_non_array_type(self): - """Schema validation warns of non-array type for bootcmd key. - - Schema validation is not strict, so bootcmd attempts to shellify the - invalid content. - """ - invalid_config = {"bootcmd": 1} - cc = get_cloud() - with self.assertRaises(TypeError): - handle("cc_bootcmd", invalid_config, cc, LOG, []) - self.assertIn( - "Invalid cloud-config provided:\nbootcmd: 1 is not of type" - " 'array'", - self.logs.getvalue(), - ) - self.assertIn("Failed to shellify", self.logs.getvalue()) - - @skipUnlessJsonSchema() - def test_handler_schema_validation_warns_non_array_item_type(self): - """Schema validation warns of non-array or string bootcmd items. - - Schema validation is not strict, so bootcmd attempts to shellify the - invalid content. - """ invalid_config = { "bootcmd": ["ls /", 20, ["wget", "http://stuff/blah"], {"a": "n"}] } cc = get_cloud() with self.assertRaises(TypeError) as context_manager: handle("cc_bootcmd", invalid_config, cc, LOG, []) - expected_warnings = [ - "bootcmd.1: 20 is not valid under any of the given schemas", - "bootcmd.3: {'a': 'n'} is not valid under any of the given schema", - ] logs = self.logs.getvalue() - for warning in expected_warnings: - self.assertIn(warning, logs) self.assertIn("Failed to shellify", logs) self.assertEqual( "Unable to shellify type 'int'. Expected list, string, tuple. " @@ -146,22 +118,48 @@ def test_handler_runs_bootcmd_script_with_error(self): @skipUnlessJsonSchema() -class TestSchema(CiTestCase, SchemaTestCaseMixin): +class TestBootCMDSchema: """Directly test schema rather than through handle.""" - schema = schema - - def test_duplicates_are_fine_array_array(self): - """Duplicated commands array/array entries are allowed.""" - self.assertSchemaValid( - ["byebye", "byebye"], "command entries can be duplicate" - ) - - def test_duplicates_are_fine_array_string(self): - """Duplicated commands array/string entries are allowed.""" - self.assertSchemaValid( - ["echo bye", "echo bye"], "command entries can be duplicate." - ) + @pytest.mark.parametrize( + "config, error_msg", + ( + # Valid schemas tested by meta.examples in test_schema + # Invalid schemas + ( + {"bootcmd": 1}, + "Cloud config schema errors: bootcmd: 1 is not of type" + " 'array'", + ), + ({"bootcmd": []}, re.escape("bootcmd: [] is too short")), + ( + {"bootcmd": []}, + re.escape( + "Cloud config schema errors: bootcmd: [] is too short" + ), + ), + ( + { + "bootcmd": [ + "ls /", + 20, + ["wget", "http://stuff/blah"], + {"a": "n"}, + ] + }, + "Cloud config schema errors: bootcmd.1: 20 is not valid under" + " any of the given schemas, bootcmd.3: {'a': 'n'} is not" + " valid under any of the given schemas", + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + """Assert expected schema validation and error messages.""" + # New-style schema $defs exist in config/cloud-init-schema*.json + schema = get_schema() + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) # vi: ts=4 expandtab diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 1647f6e58e6..32c25fca5fe 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -113,6 +113,7 @@ def test_get_schema_coalesces_known_schema(self): expected_subschema_defs = [ {"$ref": "#/$defs/cc_apk_configure"}, {"$ref": "#/$defs/cc_apt_pipelining"}, + {"$ref": "#/$defs/cc_bootcmd"}, ] found_subschema_defs = [] legacy_schema_keys = [] @@ -126,7 +127,6 @@ def test_get_schema_coalesces_known_schema(self): # This list will dwindle as we move legacy schema to new $defs assert [ "apt", - "bootcmd", "chef", "drivers", "keyboard", @@ -153,7 +153,7 @@ class TestLoadDoc: "module_name", ( "cc_apt_pipelining", # new style composite schema file - "cc_bootcmd", # legacy sub-schema defined in module + "cc_zypper_add_repo", # legacy sub-schema defined in module ), ) def test_report_docs_for_legacy_and_consolidated_schema(self, module_name): From f9b85d314b67d7f7727d66a04cd036fc49038d68 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 24 Jan 2022 14:16:29 -0700 Subject: [PATCH 02/13] schema: pull in james cc_byobu schema into cloud-init-schema.json --- cloudinit/config/cc_byobu.py | 40 ++++++++++--------- cloudinit/config/cloud-init-schema.json | 21 +++++++++- tests/unittests/config/test_cc_byobu.py | 51 +++++++++++++++++++++++++ tests/unittests/config/test_schema.py | 2 + 4 files changed, 95 insertions(+), 19 deletions(-) create mode 100644 tests/unittests/config/test_cc_byobu.py diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 53b6d0c8a20..b96736a4483 100755 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -6,11 +6,14 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Byobu ------ -**Summary:** enable/disable byobu system wide and for default user +"""Byobu: Enable/disable byobu system wide and for default user.""" + +from cloudinit import subp, util +from cloudinit.config.schema import get_meta_doc +from cloudinit.distros import ug_util +from cloudinit.settings import PER_INSTANCE +MODULE_DESCRIPTION = """\ This module controls whether byobu is enabled or disabled system wide and for the default system user. If byobu is to be enabled, this module will ensure it is installed. Likewise, if it is to be disabled, it will be removed if @@ -26,23 +29,24 @@ - ``disable``: disable byobu for all users - ``user``: alias for ``enable-user`` - ``system``: alias for ``enable-system`` - -**Internal name:** ``cc_byobu`` - -**Module frequency:** per instance - -**Supported distros:** ubuntu, debian - -**Config keys**:: - - byobu_by_default: """ - -from cloudinit import subp, util -from cloudinit.distros import ug_util - distros = ["ubuntu", "debian"] +meta = { + "id": "cc_byobu", + "name": "Byobu", + "title": "Enable/disable byobu system wide and for default user", + "description": MODULE_DESCRIPTION, + "distros": distros, + "frequency": PER_INSTANCE, + "examples": [ + "byobu_by_default: enable-user", + "byobu_by_default: disable-system", + ], +} + +__doc__ = get_meta_doc(meta) + def handle(name, cfg, cloud, log, args): if len(args) != 0: diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index 5385a8f9d26..56b57ef5cc1 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -76,11 +76,30 @@ "minItems": 1 } } + }, + "cc_byobu": { + "type": "object", + "properties": { + "byobu_by_default": { + "type": "string", + "enum": [ + "enable-system", + "enable-user", + "disable-system", + "disable-user", + "enable", + "disable", + "user", + "system" + ] + } + } } }, "allOf": [ { "$ref": "#/$defs/cc_apk_configure" }, { "$ref": "#/$defs/cc_apt_pipelining" }, - { "$ref": "#/$defs/cc_bootcmd" } + { "$ref": "#/$defs/cc_bootcmd" }, + { "$ref": "#/$defs/cc_byobu" } ] } diff --git a/tests/unittests/config/test_cc_byobu.py b/tests/unittests/config/test_cc_byobu.py new file mode 100644 index 00000000000..fbdf340386f --- /dev/null +++ b/tests/unittests/config/test_cc_byobu.py @@ -0,0 +1,51 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import re + +import pytest + +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import skipUnlessJsonSchema + + +class TestByobuSchema: + """Directly test schema rather than through handle.""" + + @pytest.mark.parametrize( + "config, error_msg", + ( + # Supplement valid schemas tested by meta.examples in test_schema + ({"byobu_by_default": "enable"}, None), + # Invalid schemas + ( + {"byobu_by_default": 1}, + "byobu_by_default: 1 is not of type 'string'", + ), + ( + {"byobu_by_default": "bogusenum"}, + re.escape( + "byobu_by_default: 'bogusenum' is not one of" + " ['enable-system', 'enable-user', 'disable-system'," + " 'disable-user', 'enable', 'disable'," + " 'user', 'system']" + ), + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + """Assert expected schema validation and error messages.""" + # New-style schema $defs exist in config/cloud-init-schema*.json + schema = get_schema() + if error_msg is None: + validate_cloudconfig_schema(config, schema, strict=True) + else: + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 32c25fca5fe..b467f3867ab 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -91,6 +91,7 @@ def test_get_schema_coalesces_known_schema(self): "cc_apt_configure", "cc_apt_pipelining", "cc_bootcmd", + "cc_byobu", "cc_keyboard", "cc_locale", "cc_ntp", @@ -114,6 +115,7 @@ def test_get_schema_coalesces_known_schema(self): {"$ref": "#/$defs/cc_apk_configure"}, {"$ref": "#/$defs/cc_apt_pipelining"}, {"$ref": "#/$defs/cc_bootcmd"}, + {"$ref": "#/$defs/cc_byobu"}, ] found_subschema_defs = [] legacy_schema_keys = [] From 9e8c9f8313a381ccb753b0721a3359eb17ff91da Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 24 Jan 2022 15:53:15 -0700 Subject: [PATCH 03/13] schema: migrate james cc_apt_configure schema to cloud-init-schema Migrate legacy apt_configure schema to new cloud-init-schea.json Add common $defs for 'primary' and 'security' keys. Use patterProperties for opaque "apt.sources" key. Make schema more strict by adding minProperties:1 and additionalProperties: False where applicable for objects. this should avoid some typos and accidental ommisions. --- cloudinit/config/cc_apt_configure.py | 273 +----------------- cloudinit/config/cloud-init-schema.json | 128 ++++++++ .../unittests/config/test_cc_apt_configure.py | 202 +++++++++++++ tests/unittests/config/test_schema.py | 2 +- 4 files changed, 333 insertions(+), 272 deletions(-) create mode 100644 tests/unittests/config/test_cc_apt_configure.py diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 37077a9f5d2..3a63d9730ae 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -17,7 +17,7 @@ from cloudinit import gpg from cloudinit import log as logging from cloudinit import subp, templater, util -from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema +from cloudinit.config.schema import get_meta_doc from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) @@ -147,275 +147,7 @@ "frequency": frequency, } -schema = { - "type": "object", - "properties": { - "apt": { - "type": "object", - "additionalProperties": False, - "properties": { - "preserve_sources_list": { - "type": "boolean", - "default": False, - "description": dedent( - """\ - By default, cloud-init will generate a new sources - list in ``/etc/apt/sources.list.d`` based on any - changes specified in cloud config. To disable this - behavior and preserve the sources list from the - pristine image, set ``preserve_sources_list`` - to ``true``. - - The ``preserve_sources_list`` option overrides - all other config keys that would alter - ``sources.list`` or ``sources.list.d``, - **except** for additional sources to be added - to ``sources.list.d``.""" - ), - }, - "disable_suites": { - "type": "array", - "items": {"type": "string"}, - "uniqueItems": True, - "description": dedent( - """\ - Entries in the sources list can be disabled using - ``disable_suites``, which takes a list of suites - to be disabled. If the string ``$RELEASE`` is - present in a suite in the ``disable_suites`` list, - it will be replaced with the release name. If a - suite specified in ``disable_suites`` is not - present in ``sources.list`` it will be ignored. - For convenience, several aliases are provided for - ``disable_suites``: - - - ``updates`` => ``$RELEASE-updates`` - - ``backports`` => ``$RELEASE-backports`` - - ``security`` => ``$RELEASE-security`` - - ``proposed`` => ``$RELEASE-proposed`` - - ``release`` => ``$RELEASE``. - - When a suite is disabled using ``disable_suites``, - its entry in ``sources.list`` is not deleted; it - is just commented out.""" - ), - }, - "primary": { - **mirror_property, - "description": dedent( - """\ - The primary and security archive mirrors can - be specified using the ``primary`` and - ``security`` keys, respectively. Both the - ``primary`` and ``security`` keys take a list - of configs, allowing mirrors to be specified - on a per-architecture basis. Each config is a - dictionary which must have an entry for - ``arches``, specifying which architectures - that config entry is for. The keyword - ``default`` applies to any architecture not - explicitly listed. The mirror url can be specified - with the ``uri`` key, or a list of mirrors to - check can be provided in order, with the first - mirror that can be resolved being selected. This - allows the same configuration to be used in - different environment, with different hosts used - for a local apt mirror. If no mirror is provided - by ``uri`` or ``search``, ``search_dns`` may be - used to search for dns names in the format - ``-mirror`` in each of the following: - - - fqdn of this host per cloud metadata, - - localdomain, - - domains listed in ``/etc/resolv.conf``. - - If there is a dns entry for ``-mirror``, - then it is assumed that there is a distro mirror - at ``http://-mirror./``. - If the ``primary`` key is defined, but not the - ``security`` key, then then configuration for - ``primary`` is also used for ``security``. - If ``search_dns`` is used for the ``security`` - key, the search pattern will be - ``-security-mirror``. - - Each mirror may also specify a key to import via - any of the following optional keys: - - - ``keyid``: a key to import via shortid or \ - fingerprint. - - ``key``: a raw PGP key. - - ``keyserver``: alternate keyserver to pull \ - ``keyid`` key from. - - If no mirrors are specified, or all lookups fail, - then default mirrors defined in the datasource - are used. If none are present in the datasource - either the following defaults are used: - - - ``primary`` => \ - ``http://archive.ubuntu.com/ubuntu``. - - ``security`` => \ - ``http://security.ubuntu.com/ubuntu`` - """ - ), - }, - "security": { - **mirror_property, - "description": dedent( - """\ - Please refer to the primary config documentation""" - ), - }, - "add_apt_repo_match": { - "type": "string", - "default": ADD_APT_REPO_MATCH, - "description": dedent( - """\ - All source entries in ``apt-sources`` that match - regex in ``add_apt_repo_match`` will be added to - the system using ``add-apt-repository``. If - ``add_apt_repo_match`` is not specified, it - defaults to ``{}``""".format( - ADD_APT_REPO_MATCH - ) - ), - }, - "debconf_selections": { - "type": "object", - "items": {"type": "string"}, - "description": dedent( - """\ - Debconf additional configurations can be specified as a - dictionary under the ``debconf_selections`` config - key, with each key in the dict representing a - different set of configurations. The value of each key - must be a string containing all the debconf - configurations that must be applied. We will bundle - all of the values and pass them to - ``debconf-set-selections``. Therefore, each value line - must be a valid entry for ``debconf-set-selections``, - meaning that they must possess for distinct fields: - - ``pkgname question type answer`` - - Where: - - - ``pkgname`` is the name of the package. - - ``question`` the name of the questions. - - ``type`` is the type of question. - - ``answer`` is the value used to ansert the \ - question. - - For example: \ - ``ippackage ippackage/ip string 127.0.01`` - """ - ), - }, - "sources_list": { - "type": "string", - "description": dedent( - """\ - Specifies a custom template for rendering - ``sources.list`` . If no ``sources_list`` template - is given, cloud-init will use sane default. Within - this template, the following strings will be - replaced with the appropriate values: - - - ``$MIRROR`` - - ``$RELEASE`` - - ``$PRIMARY`` - - ``$SECURITY`` - - ``$KEY_FILE``""" - ), - }, - "conf": { - "type": "string", - "description": dedent( - """\ - Specify configuration for apt, such as proxy - configuration. This configuration is specified as a - string. For multiline apt configuration, make sure - to follow yaml syntax.""" - ), - }, - "https_proxy": { - "type": "string", - "description": dedent( - """\ - More convenient way to specify https apt proxy. - https proxy url is specified in the format - ``https://[[user][:pass]@]host[:port]/``.""" - ), - }, - "http_proxy": { - "type": "string", - "description": dedent( - """\ - More convenient way to specify http apt proxy. - http proxy url is specified in the format - ``http://[[user][:pass]@]host[:port]/``.""" - ), - }, - "proxy": { - "type": "string", - "description": "Alias for defining a http apt proxy.", - }, - "ftp_proxy": { - "type": "string", - "description": dedent( - """\ - More convenient way to specify ftp apt proxy. - ftp proxy url is specified in the format - ``ftp://[[user][:pass]@]host[:port]/``.""" - ), - }, - "sources": { - "type": "object", - "items": {"type": "string"}, - "description": dedent( - """\ - Source list entries can be specified as a - dictionary under the ``sources`` config key, with - each key in the dict representing a different source - file. The key of each source entry will be used - as an id that can be referenced in other config - entries, as well as the filename for the source's - configuration under ``/etc/apt/sources.list.d``. - If the name does not end with ``.list``, it will - be appended. If there is no configuration for a - key in ``sources``, no file will be written, but - the key may still be referred to as an id in other - ``sources`` entries. - - Each entry under ``sources`` is a dictionary which - may contain any of the following optional keys: - - - ``source``: a sources.list entry \ - (some variable replacements apply). - - ``keyid``: a key to import via shortid or \ - fingerprint. - - ``key``: a raw PGP key. - - ``keyserver``: alternate keyserver to pull \ - ``keyid`` key from. - - ``filename``: specify the name of the .list file - - The ``source`` key supports variable - replacements for the following strings: - - - ``$MIRROR`` - - ``$PRIMARY`` - - ``$SECURITY`` - - ``$RELEASE`` - - ``$KEY_FILE``""" - ), - }, - }, - } - }, -} - -__doc__ = get_meta_doc(meta, schema) +__doc__ = get_meta_doc(meta) # place where apt stores cached repository data @@ -474,7 +206,6 @@ def handle(name, ocfg, cloud, log, _): ) ) - validate_cloudconfig_schema(cfg, schema) apply_debconf_selections(cfg, target) apply_apt(cfg, cloud, target) diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index 56b57ef5cc1..83bdc464eb0 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -1,6 +1,35 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "$defs": { + "apt_configure.mirror": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["arches"], + "properties": { + "arches": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1 + }, + "uri": {"type": "string", "format": "uri"}, + "search": { + "type": "array", + "items": {"type": "string", "format": "uri"}, + "minItems": 1 + }, + "search_dns": { + "type": "boolean" + }, + "keyid": {"type": "string"}, + "key": {"type": "string"}, + "keyserver": {"type": "string"} + }, + "additionalProperties": false + }, + "minItems": 1 + }, "cc_apk_configure": { "type": "object", "properties": { @@ -49,6 +78,104 @@ } } }, + "cc_apt_configure": { + "properties": { + "apt": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "preserve_sources_list": { + "type": "boolean", + "default": false, + "description": "By default, cloud-init will generate a new sources list in ``/etc/apt/sources.list.d`` based on any changes specified in cloud config. To disable this behavior and preserve the sources list from the pristine image, set ``preserve_sources_list`` to ``true``.\n\nThe ``preserve_sources_list`` option overrides all other config keys that would alter ``sources.list`` or ``sources.list.d``, **except** for additional sources to be added to ``sources.list.d``." + }, + "disable_suites": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + "uniqueItems": true, + "description": "Entries in the sources list can be disabled using ``disable_suites``, which takes a list of suites to be disabled. If the string ``$RELEASE`` is present in a suite in the ``disable_suites`` list, it will be replaced with the release name. If a suite specified in ``disable_suites`` is not present in ``sources.list`` it will be ignored. For convenience, several aliases are provided for`` disable_suites``:\n\n - ``updates`` => ``$RELEASE-updates``\n - ``backports`` => ``$RELEASE-backports``\n - ``security`` => ``$RELEASE-security``\n - ``proposed`` => ``$RELEASE-proposed``\n - ``release`` => ``$RELEASE``.\n\nWhen a suite is disabled using ``disable_suites``, its entry in ``sources.list`` is not deleted; it is just commented out." + }, + "primary": { + "$ref": "#/$defs/apt_configure.mirror", + "description": "The primary and security archive mirrors can be specified using the ``primary`` and ``security`` keys, respectively. Both the ``primary`` and ``security`` keys take a list of configs, allowing mirrors to be specified on a per-architecture basis. Each config is a dictionary which must have an entry for ``arches``, specifying which architectures that config entry is for. The keyword ``default`` applies to any architecture not explicitly listed. The mirror url can be specified with the ``uri`` key, or a list of mirrors to check can be provided in order, with the first mirror that can be resolved being selected. This allows the same configuration to be used in different environment, with different hosts used for a local APT mirror. If no mirror is provided by ``uri`` or ``search``, ``search_dns`` may be used to search for dns names in the format ``-mirror`` in each of the following:\n\n - fqdn of this host per cloud metadata,\n - localdomain,\n - domains listed in ``/etc/resolv.conf``.\n\nIf there is a dns entry for ``-mirror``, then it is assumed that there is a distro mirror at ``http://-mirror./``. If the ``primary`` key is defined, but not the ``security`` key, then then configuration for ``primary`` is also used for ``security``. If ``search_dns`` is used for the ``security`` key, the search pattern will be ``-security-mirror``.\n\nEach mirror may also specify a key to import via any of the following optional keys:\n\n - ``keyid``: a key to import via shortid or fingerprint.\n - ``key``: a raw PGP key.\n - ``keyserver``: alternate keyserver to pull ``keyid`` key from.\n\nIf no mirrors are specified, or all lookups fail, then default mirrors defined in the datasource are used. If none are present in the datasource either the following defaults are used:\n\n - ``primary`` => ``http://archive.ubuntu.com/ubuntu``.\n - ``security`` => ``http://security.ubuntu.com/ubuntu``" + }, + "security": { + "$ref": "#/$defs/apt_configure.mirror", + "description": "Please refer to the primary config documentation" + }, + "add_apt_repo_match": { + "type": "string", + "default": "^[\\w-]+:\\w", + "description": "All source entries in ``apt-sources`` that match regex in ``add_apt_repo_match`` will be added to the system using ``add-apt-repository``. If ``add_apt_repo_match`` is not specified, it defaults to ``^[\\w-]+:\\w``" + }, + "debconf_selections": { + "type": "object", + "minProperties": 1, + "patternProperties": { + "^.+$": { + "type": "string" + } + }, + "description": "Debconf additional configurations can be specified as a dictionary under the ``debconf_selections`` config key, with each key in the dict representing a different set of configurations. The value of each key must be a string containing all the debconf configurations that must be applied. We will bundle all of the values and pass them to ``debconf-set-selections``. Therefore, each value line must be a valid entry for ``debconf-set-selections``, meaning that they must possess for distinct fields:\n\n``pkgname question type answer``\n\nWhere:\n\n - ``pkgname`` is the name of the package.\n - ``question`` the name of the questions.\n - ``type`` is the type of question.\n - ``answer`` is the value used to answer the question.\n\nFor example: ``ippackage ippackage/ip string 127.0.01``" + }, + "sources_list": { + "type": "string", + "description": "Specifies a custom template for rendering ``sources.list`` . If no ``sources_list`` template is given, cloud-init will use sane default. Within this template, the following strings will be replaced with the appropriate values:\n\n - ``$MIRROR``\n - ``$RELEASE``\n - ``$PRIMARY``\n - ``$SECURITY``\n - ``$KEY_FILE``" + }, + "conf": { + "type": "string", + "description": "Specify configuration for apt, such as proxy configuration. This configuration is specified as a string. For multiline APT configuration, make sure to follow yaml syntax." + }, + "https_proxy": { + "type": "string", + "description": "More convenient way to specify https APT proxy. https proxy url is specified in the format ``https://[[user][:pass]@]host[:port]/``." + }, + "http_proxy": { + "type": "string", + "description": "More convenient way to specify http APT proxy. http proxy url is specified in the format ``http://[[user][:pass]@]host[:port]/``." + }, + "proxy": { + "type": "string", + "description": "Alias for defining a http APT proxy." + }, + "ftp_proxy": { + "type": "string", + "description": "More convenient way to specify ftp APT proxy. ftp proxy url is specified in the format ``ftp://[[user][:pass]@]host[:port]/``." + }, + "sources": { + "type": "object", + "patternProperties": { + "^.+$": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "keyid": { + "type": "string" + }, + "key": { + "type": "string" + }, + "keyserver": { + "type": "string" + }, + "filename": { + "type": "string" + } + }, + "additionalProperties": false, + "minProperties": 1 + } + }, + "description": "Source list entries can be specified as a dictionary under the ``sources`` config key, with each key in the dict representing a different source file. The key of each source entry will be used as an id that can be referenced in other config entries, as well as the filename for the source's configuration under ``/etc/apt/sources.list.d``. If the name does not end with ``.list``, it will be appended. If there is no configuration for a key in ``sources``, no file will be written, but the key may still be referred to as an id in other ``sources`` entries.\n\nEach entry under ``sources`` is a dictionary which may contain any of the following optional keys:\n - ``source``: a sources.list entry (some variable replacements apply).\n - ``keyid``: a key to import via shortid or fingerprint.\n - ``key``: a raw PGP key.\n - ``keyserver``: alternate keyserver to pull ``keyid`` key from.\n - ``filename``: specify the name of the list file\n\nThe ``source`` key supports variable replacements for the following strings:\n\n - ``$MIRROR``\n - ``$PRIMARY``\n - ``$SECURITY``\n - ``$RELEASE``\n - ``$KEY_FILE``" + } + } + } + } + }, "cc_apt_pipelining": { "type": "object", "properties": { @@ -98,6 +225,7 @@ }, "allOf": [ { "$ref": "#/$defs/cc_apk_configure" }, + { "$ref": "#/$defs/cc_apt_configure" }, { "$ref": "#/$defs/cc_apt_pipelining" }, { "$ref": "#/$defs/cc_bootcmd" }, { "$ref": "#/$defs/cc_byobu" } diff --git a/tests/unittests/config/test_cc_apt_configure.py b/tests/unittests/config/test_cc_apt_configure.py new file mode 100644 index 00000000000..bd1bb963c96 --- /dev/null +++ b/tests/unittests/config/test_cc_apt_configure.py @@ -0,0 +1,202 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +""" Tests for cc_apt_configure module """ + +import re + +import pytest + +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import skipUnlessJsonSchema + + +class TestAPTConfigureSchema: + @pytest.mark.parametrize( + "config, error_msg", + ( + # Supplement valid schemas from examples tested in test_schema + ({"apt": {"preserve_sources_list": True}}, None), + # Invalid schemas + ( + {"apt": "nonobject"}, + "apt: 'nonobject' is not of type 'object", + ), + ( + {"apt": {"boguskey": True}}, + re.escape( + "apt: Additional properties are not allowed" + " ('boguskey' was unexpected)" + ), + ), + ({"apt": {}}, "apt: {} does not have enough properties"), + ( + {"apt": {"preserve_sources_list": 1}}, + "apt.preserve_sources_list: 1 is not of type 'boolean'", + ), + ( + {"apt": {"disable_suites": 1}}, + "apt.disable_suites: 1 is not of type 'array'", + ), + ( + {"apt": {"disable_suites": []}}, + re.escape("apt.disable_suites: [] is too short"), + ), + ( + {"apt": {"disable_suites": [1]}}, + "apt.disable_suites.0: 1 is not of type 'string'", + ), + ( + {"apt": {"disable_suites": ["a", "a"]}}, + re.escape( + "apt.disable_suites: ['a', 'a'] has non-unique elements" + ), + ), + # All apt: primary tests are applicable for "security" key too. + # Those apt:security tests are exercised in the unittest below + ( + {"apt": {"primary": "nonlist"}}, + "apt.primary: 'nonlist' is not of type 'array'", + ), + ( + {"apt": {"primary": []}}, + re.escape("apt.primary: [] is too short"), + ), + ( + {"apt": {"primary": ["nonobj"]}}, + "apt.primary.0: 'nonobj' is not of type 'object'", + ), + ( + {"apt": {"primary": [{}]}}, + "apt.primary.0: 'arches' is a required property", + ), + ( + {"apt": {"primary": [{"boguskey": True}]}}, + re.escape( + "apt.primary.0: Additional properties are not allowed" + " ('boguskey' was unexpected)" + ), + ), + ( + {"apt": {"primary": [{"arches": True}]}}, + "apt.primary.0.arches: True is not of type 'array'", + ), + ( + {"apt": {"primary": [{"uri": True}]}}, + "apt.primary.0.uri: True is not of type 'string'", + ), + ( + { + "apt": { + "primary": [ + {"arches": ["amd64"], "search": "non-array"} + ] + } + }, + "apt.primary.0.search: 'non-array' is not of type 'array'", + ), + ( + {"apt": {"primary": [{"arches": ["amd64"], "search": []}]}}, + re.escape("apt.primary.0.search: [] is too short"), + ), + ( + { + "apt": { + "primary": [{"arches": ["amd64"], "search_dns": "a"}] + } + }, + "apt.primary.0.search_dns: 'a' is not of type 'boolean'", + ), + ( + {"apt": {"primary": [{"arches": ["amd64"], "keyid": 1}]}}, + "apt.primary.0.keyid: 1 is not of type 'string'", + ), + ( + {"apt": {"primary": [{"arches": ["amd64"], "key": 1}]}}, + "apt.primary.0.key: 1 is not of type 'string'", + ), + ( + {"apt": {"primary": [{"arches": ["amd64"], "keyserver": 1}]}}, + "apt.primary.0.keyserver: 1 is not of type 'string'", + ), + ( + {"apt": {"add_apt_repo_match": True}}, + "apt.add_apt_repo_match: True is not of type 'string'", + ), + ( + {"apt": {"debconf_selections": True}}, + "apt.debconf_selections: True is not of type 'object'", + ), + ( + {"apt": {"debconf_selections": {}}}, + "apt.debconf_selections: {} does not have enough properties", + ), + ( + {"apt": {"sources_list": True}}, + "apt.sources_list: True is not of type 'string'", + ), + ( + {"apt": {"conf": True}}, + "apt.conf: True is not of type 'string'", + ), + ( + {"apt": {"http_proxy": True}}, + "apt.http_proxy: True is not of type 'string'", + ), + ( + {"apt": {"https_proxy": True}}, + "apt.https_proxy: True is not of type 'string'", + ), + ( + {"apt": {"proxy": True}}, + "apt.proxy: True is not of type 'string'", + ), + ( + {"apt": {"ftp_proxy": True}}, + "apt.ftp_proxy: True is not of type 'string'", + ), + ( + {"apt": {"sources": True}}, + "apt.sources: True is not of type 'object'", + ), + ( + {"apt": {"sources": {"opaquekey": True}}}, + "apt.sources.opaquekey: True is not of type 'object'", + ), + ( + {"apt": {"sources": {"opaquekey": {}}}}, + "apt.sources.opaquekey: {} does not have enough properties", + ), + ( + {"apt": {"sources": {"opaquekey": {"boguskey": True}}}}, + re.escape( + "apt.sources.opaquekey: Additional properties are not" + " allowed ('boguskey' was unexpected)" + ), + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + schema = get_schema() + if error_msg is None: + validate_cloudconfig_schema(config, schema, strict=True) + else: + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) + # Note apt['primary'] and apt['security'] have same defition + # Avoid test setup duplicates by running same test using 'security' + if isinstance(config.get("apt"), dict) and config["apt"].get( + "primary" + ): + # To exercise security schema, rename test key from primary + config["apt"]["security"] = config["apt"].pop("primary") + error_msg = error_msg.replace("primary", "security") + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index b467f3867ab..0f0b1fdbb87 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -113,6 +113,7 @@ def test_get_schema_coalesces_known_schema(self): # New style schema should be defined in static schema file in $defs expected_subschema_defs = [ {"$ref": "#/$defs/cc_apk_configure"}, + {"$ref": "#/$defs/cc_apt_configure"}, {"$ref": "#/$defs/cc_apt_pipelining"}, {"$ref": "#/$defs/cc_bootcmd"}, {"$ref": "#/$defs/cc_byobu"}, @@ -128,7 +129,6 @@ def test_get_schema_coalesces_known_schema(self): assert expected_subschema_defs == found_subschema_defs # This list will dwindle as we move legacy schema to new $defs assert [ - "apt", "chef", "drivers", "keyboard", From 72c20c9a163e4dc230c5f7680b5855de1397e468 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 24 Jan 2022 19:44:50 -0700 Subject: [PATCH 04/13] schema: pull in james' ca_certs schema. Migrate legacy schema --- cloudinit/config/cc_ca_certs.py | 79 ++++++++++++---------- cloudinit/config/cloud-init-schema.json | 26 ++++++- tests/unittests/config/test_cc_ca_certs.py | 60 +++++++++++++++- tests/unittests/config/test_schema.py | 2 + 4 files changed, 129 insertions(+), 38 deletions(-) diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 9de065ab0fb..d9aa6f5d36a 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -2,46 +2,14 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -CA Certs --------- -**Summary:** add ca certificates - -This module adds CA certificates to ``/etc/ca-certificates.conf`` and updates -the ssl cert cache using ``update-ca-certificates``. The default certificates -can be removed from the system with the configuration option -``remove-defaults``. - -.. note:: - certificates must be specified using valid yaml. in order to specify a - multiline certificate, the yaml multiline list syntax must be used - -.. note:: - For Alpine Linux the "remove-defaults" functionality works if the - ca-certificates package is installed but not if the - ca-certificates-bundle package is installed. - -**Internal name:** ``cc_ca_certs`` - -**Module frequency:** per instance - -**Supported distros:** alpine, debian, ubuntu, rhel - -**Config keys**:: - - ca-certs: - remove-defaults: - trusted: - - - - | - -----BEGIN CERTIFICATE----- - YOUR-ORGS-TRUSTED-CA-CERT-HERE - -----END CERTIFICATE----- -""" +"""CA Certs: Add ca certificates.""" import os +from textwrap import dedent from cloudinit import subp, util +from cloudinit.config.schema import get_meta_doc +from cloudinit.settings import PER_INSTANCE DEFAULT_CONFIG = { "ca_cert_path": "/usr/share/ca-certificates/", @@ -60,9 +28,48 @@ } } +MODULE_DESCRIPTION = """\ +This module adds CA certificates to ``/etc/ca-certificates.conf`` and updates +the ssl cert cache using ``update-ca-certificates``. The default certificates +can be removed from the system with the configuration option +``remove-defaults``. + +.. note:: + certificates must be specified using valid yaml. in order to specify a + multiline certificate, the yaml multiline list syntax must be used +.. note:: + For Alpine Linux the "remove-defaults" functionality works if the + ca-certificates package is installed but not if the + ca-certificates-bundle package is installed. +""" distros = ["alpine", "debian", "ubuntu", "rhel"] +meta = { + "id": "cc_ca_certs", + "name": "CA Certificates", + "title": "Add ca certificates", + "description": MODULE_DESCRIPTION, + "distros": distros, + "frequency": PER_INSTANCE, + "examples": [ + dedent( + """\ + ca-certs: + remove-defaults: true + trusted: + - single_line_cert + - | + -----BEGIN CERTIFICATE----- + YOUR-ORGS-TRUSTED-CA-CERT-HERE + -----END CERTIFICATE----- + """ + ) + ], +} + +__doc__ = get_meta_doc(meta) + def _distro_ca_certs_configs(distro_name): """Return a distro-specific ca_certs config dictionary diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index 83bdc464eb0..ce8b1213a3b 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -221,6 +221,29 @@ ] } } + }, + "cc_ca_certs": { + "type": "object", + "properties": { + "ca_certs": { + "type": "object", + "properties": { + "remove_defaults": { + "description": "Remove default CA certificates if true. Default: false", + "type": "boolean", + "default": false + }, + "trusted": { + "description": "List of trusted CA certificates to add.", + "type": "array", + "items": {"type": "string"}, + "minItems": 1 + } + }, + "additionalProperties": false, + "minProperties": 1 + } + } } }, "allOf": [ @@ -228,6 +251,7 @@ { "$ref": "#/$defs/cc_apt_configure" }, { "$ref": "#/$defs/cc_apt_pipelining" }, { "$ref": "#/$defs/cc_bootcmd" }, - { "$ref": "#/$defs/cc_byobu" } + { "$ref": "#/$defs/cc_byobu" }, + { "$ref": "#/$defs/cc_ca_certs" } ] } diff --git a/tests/unittests/config/test_cc_ca_certs.py b/tests/unittests/config/test_cc_ca_certs.py index c49922e684e..20fe408f6d8 100644 --- a/tests/unittests/config/test_cc_ca_certs.py +++ b/tests/unittests/config/test_cc_ca_certs.py @@ -1,14 +1,22 @@ # This file is part of cloud-init. See LICENSE file for license information. import logging +import re import shutil import tempfile import unittest from contextlib import ExitStack from unittest import mock +import pytest + from cloudinit import distros, helpers, subp, util from cloudinit.config import cc_ca_certs -from tests.unittests.helpers import TestCase +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import TestCase, skipUnlessJsonSchema from tests.unittests.util import get_cloud @@ -406,4 +414,54 @@ def test_commands(self): ) +class TestCACertsSchema: + """Directly test schema rather than through handle.""" + + @pytest.mark.parametrize( + "config, error_msg", + ( + # Invalid schemas + ( + {"ca_certs": 1}, + "ca_certs: 1 is not of type 'object'", + ), + ( + {"ca_certs": {}}, + re.escape("ca_certs: {} does not have enough properties"), + ), + ( + {"ca_certs": {"boguskey": 1}}, + re.escape( + "ca_certs: Additional properties are not allowed" + " ('boguskey' was unexpected)" + ), + ), + ( + {"ca_certs": {"remove_defaults": 1}}, + "ca_certs.remove_defaults: 1 is not of type 'boolean'", + ), + ( + {"ca_certs": {"trusted": [1]}}, + "ca_certs.trusted.0: 1 is not of type 'string'", + ), + ( + {"ca_certs": {"trusted": []}}, + re.escape("ca_certs.trusted: [] is too short"), + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + """Assert expected schema validation and error messages.""" + # New-style schema $defs exist in config/cloud-init-schema*.json + schema = get_schema() + if error_msg is None: + validate_cloudconfig_schema(config, schema, strict=True) + else: + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) + + +# vi: ts=4 expandtab + # vi: ts=4 expandtab diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 0f0b1fdbb87..c5b50d66411 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -92,6 +92,7 @@ def test_get_schema_coalesces_known_schema(self): "cc_apt_pipelining", "cc_bootcmd", "cc_byobu", + "cc_ca_certs", "cc_keyboard", "cc_locale", "cc_ntp", @@ -117,6 +118,7 @@ def test_get_schema_coalesces_known_schema(self): {"$ref": "#/$defs/cc_apt_pipelining"}, {"$ref": "#/$defs/cc_bootcmd"}, {"$ref": "#/$defs/cc_byobu"}, + {"$ref": "#/$defs/cc_ca_certs"}, ] found_subschema_defs = [] legacy_schema_keys = [] From 0e0f207d954350aea2177f11596c5fd046878b53 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 24 Jan 2022 20:24:28 -0700 Subject: [PATCH 05/13] schema: migrate legacy cc_chef schema to cloud-init-schema Migrate legacy chef schema to new cloud-init-schea.json. Add more strict schema definition disallowing additionalProperties. Add extensive unittests for invalid schemas. --- cloudinit/config/cc_chef.py | 5 +- cloudinit/config/cloud-init-schema.json | 151 ++++++++++++++++++++- tests/unittests/config/test_cc_chef.py | 172 ++++++++++++++++++++++++ tests/unittests/config/test_schema.py | 3 +- 4 files changed, 326 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 67889683373..ba8b119301a 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -14,7 +14,7 @@ from textwrap import dedent from cloudinit import subp, temp_utils, templater, url_helper, util -from cloudinit.config.schema import get_meta_doc, validate_cloudconfig_schema +from cloudinit.config.schema import get_meta_doc from cloudinit.settings import PER_ALWAYS RUBY_VERSION_DEFAULT = "1.8" @@ -433,7 +433,7 @@ }, } -__doc__ = get_meta_doc(meta, schema) +__doc__ = get_meta_doc(meta) def post_run_chef(chef_cfg, log): @@ -489,7 +489,6 @@ def handle(name, cfg, cloud, log, _args): ) return - validate_cloudconfig_schema(cfg, schema) chef_cfg = cfg["chef"] # Ensure the chef directories we use exist diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index ce8b1213a3b..311e7a0f4a3 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -244,6 +244,154 @@ "minProperties": 1 } } + }, + "cc_chef": { + "type": "object", + "properties": { + "chef": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "directories": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + "uniqueItems": true, + "description": "Create the necessary directories for chef to run. By default, it creates the following directories:\n\n - ``/etc/chef``\n - ``/var/log/chef``\n - ``/var/lib/chef``\n - ``/var/cache/chef``\n - ``/var/backups/chef``\n - ``/var/run/chef``" + }, + "validation_cert": { + "type": "string", + "description": "Optional string to be written to file validation_key. Special value ``system`` means set use existing file." + }, + "validation_key": { + "type": "string", + "default": "/etc/chef/validation.pem", + "description": "Optional path for validation_cert. default to ``/etc/chef/validation.pem``" + }, + "firstboot_path": { + "type": "string", + "default": "/etc/chef/firstboot.json", + "description": "Path to write run_list and initial_attributes keys that should also be present in this configuration, defaults to ``/etc/chef/firstboot.json``" + }, + "exec": { + "type": "boolean", + "default": false, + "description": "Set true if we should run or not run chef (defaults to false, unless a gem installed is requested where this will then default to true)." + }, + "client_key": { + "type": "string", + "default": "/etc/chef/client.pem", + "description": "Optional path for client_cert. Default to ``/etc/chef/client.pem``." + }, + "encrypted_data_bag_secret": { + "type": "string", + "default": null, + "description": "Specifies the location of the secret key used by chef to encrypt data items. By default, this path is set to null, meaning that chef will have to look at the path ``/etc/chef/encrypted_data_bag_secret`` for it." + }, + "environment": { + "type": "string", + "default": "_default", + "description": "Specifies which environment chef will use. By default, it will use the ``_default`` configuration." + }, + "file_backup_path": { + "type": "string", + "default": "/var/backups/chef", + "description": "Specifies the location in which backup files are stored. By default, it uses the ``/var/backups/chef`` location." + }, + "file_cache_path": { + "type": "string", + "default": "/var/cache/chef", + "description": "Specifies the location in which chef cache files will be saved. By default, it uses the ``/var/cache/chef`` location." + }, + "json_attribs": { + "type": "string", + "default": "/etc/chef/firstboot.json", + "description": "Specifies the location in which some chef json data is stored. By default, it uses the ``/etc/chef/firstboot.json`` location." + }, + "log_level": { + "type": "string", + "default": ":info", + "description": "Defines the level of logging to be stored in the log file. By default this value is set to ``:info``." + }, + "log_location": { + "type": "string", + "default": "/var/log/chef/client.log", + "description": "Specifies the location of the chef lof file. By default, the location is specified at ``/var/log/chef/client.log``." + }, + "node_name": { + "type": "string", + "description": "The name of the node to run. By default, we will use th instance id as the node name." + }, + "omnibus_url": { + "type": "string", + "default": "https://www.chef.io/chef/install.sh", + "description": "Omnibus URL if chef should be installed through Omnibus. By default, it uses the ``https://www.chef.io/chef/install.sh``." + }, + "omnibus_url_retries": { + "type": "integer", + "default": 5, + "description": "The number of retries that will be attempted to reach the Omnibus URL. Default is 5." + }, + "omnibus_version": { + "type": "string", + "description": "Optional version string to require for omnibus install." + }, + "pid_file": { + "type": "string", + "default": "/var/run/chef/client.pid", + "description": "The location in which a process identification number (pid) is saved. By default, it saves in the ``/var/run/chef/client.pid`` location." + }, + "server_url": { + "type": "string", + "description": "The URL for the chef server" + }, + "show_time": { + "type": "boolean", + "default": true, + "description": "Show time in chef logs" + }, + "ssl_verify_mode": { + "type": "string", + "default": ":verify_none", + "description": "Set the verify mode for HTTPS requests. We can have two possible values for this parameter:\n\n - ``:verify_none``: No validation of SSL certificates.\n - ``:verify_peer``: Validate all SSL certificates.\n\nBy default, the parameter is set as ``:verify_none``." + }, + "validation_name": { + "type": "string", + "description": "The name of the chef-validator key that Chef Infra Client uses to access the Chef Infra Server during the initial Chef Infra Client run." + }, + "force_install": { + "type": "boolean", + "default": false, + "description": "If set to ``true``, forces chef installation, even if it is already installed." + }, + "initial_attributes": { + "type": "object", + "items": {"type": "string"}, + "description": "Specify a list of initial attributes used by the cookbooks." + }, + "install_type": { + "type": "string", + "default": "packages", + "enum": [ + "packages", + "gems", + "omnibus" + ], + "description": "The type of installation for chef. It can be one of the following values:\n\n - ``packages``\n - ``gems``\n - ``omnibus``" + }, + "run_list": { + "type": "array", + "items": {"type": "string"}, + "description": "A run list for a first boot json." + }, + "chef_license": { + "type": "string", + "description": "string that indicates if user accepts or not license related to some of chef products" + } + } + } + } } }, "allOf": [ @@ -252,6 +400,7 @@ { "$ref": "#/$defs/cc_apt_pipelining" }, { "$ref": "#/$defs/cc_bootcmd" }, { "$ref": "#/$defs/cc_byobu" }, - { "$ref": "#/$defs/cc_ca_certs" } + { "$ref": "#/$defs/cc_ca_certs" }, + { "$ref": "#/$defs/cc_chef" } ] } diff --git a/tests/unittests/config/test_cc_chef.py b/tests/unittests/config/test_cc_chef.py index 835974e52f8..f86be293b91 100644 --- a/tests/unittests/config/test_cc_chef.py +++ b/tests/unittests/config/test_cc_chef.py @@ -3,17 +3,25 @@ import json import logging import os +import re import httpretty +import pytest from cloudinit import util from cloudinit.config import cc_chef +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) from tests.unittests.helpers import ( FilesystemMockingTestCase, HttprettyTestCase, cloud_init_project_dir, mock, skipIf, + skipUnlessJsonSchema, ) from tests.unittests.util import get_cloud @@ -289,4 +297,168 @@ def test_validation_cert_with_system(self): self.assertEqual(expected_cert, util.load_file(v_path)) +@skipUnlessJsonSchema() +class TestBootCMDSchema: + """Directly test schema rather than through handle.""" + + @pytest.mark.parametrize( + "config, error_msg", + ( + # Valid schemas tested by meta.examples in test_schema + # Invalid schemas + ( + {"chef": 1}, + "chef: 1 is not of type 'object'", + ), + ( + {"chef": {}}, + re.escape(" chef: {} does not have enough properties"), + ), + ( + {"chef": {"boguskey": True}}, + re.escape( + "chef: Additional properties are not allowed" + " ('boguskey' was unexpected)" + ), + ), + ( + {"chef": {"directories": 1}}, + "chef.directories: 1 is not of type 'array'", + ), + ( + {"chef": {"directories": []}}, + re.escape("chef.directories: [] is too short"), + ), + ( + {"chef": {"directories": [1]}}, + "chef.directories.0: 1 is not of type 'string'", + ), + ( + {"chef": {"directories": ["a", "a"]}}, + re.escape( + "chef.directories: ['a', 'a'] has non-unique elements" + ), + ), + ( + {"chef": {"validation_cert": 1}}, + "chef.validation_cert: 1 is not of type 'string'", + ), + ( + {"chef": {"validation_key": 1}}, + "chef.validation_key: 1 is not of type 'string'", + ), + ( + {"chef": {"firstboot_path": 1}}, + "chef.firstboot_path: 1 is not of type 'string'", + ), + ( + {"chef": {"client_key": 1}}, + "chef.client_key: 1 is not of type 'string'", + ), + ( + {"chef": {"encrypted_data_bag_secret": 1}}, + "chef.encrypted_data_bag_secret: 1 is not of type 'string'", + ), + ( + {"chef": {"environment": 1}}, + "chef.environment: 1 is not of type 'string'", + ), + ( + {"chef": {"file_backup_path": 1}}, + "chef.file_backup_path: 1 is not of type 'string'", + ), + ( + {"chef": {"file_cache_path": 1}}, + "chef.file_cache_path: 1 is not of type 'string'", + ), + ( + {"chef": {"json_attribs": 1}}, + "chef.json_attribs: 1 is not of type 'string'", + ), + ( + {"chef": {"log_level": 1}}, + "chef.log_level: 1 is not of type 'string'", + ), + ( + {"chef": {"log_location": 1}}, + "chef.log_location: 1 is not of type 'string'", + ), + ( + {"chef": {"node_name": 1}}, + "chef.node_name: 1 is not of type 'string'", + ), + ( + {"chef": {"omnibus_url": 1}}, + "chef.omnibus_url: 1 is not of type 'string'", + ), + ( + {"chef": {"omnibus_url_retries": "one"}}, + "chef.omnibus_url_retries: 'one' is not of type 'integer'", + ), + ( + {"chef": {"omnibus_version": 1}}, + "chef.omnibus_version: 1 is not of type 'string'", + ), + ( + {"chef": {"omnibus_version": 1}}, + "chef.omnibus_version: 1 is not of type 'string'", + ), + ( + {"chef": {"pid_file": 1}}, + "chef.pid_file: 1 is not of type 'string'", + ), + ( + {"chef": {"server_url": 1}}, + "chef.server_url: 1 is not of type 'string'", + ), + ( + {"chef": {"show_time": 1}}, + "chef.show_time: 1 is not of type 'boolean'", + ), + ( + {"chef": {"ssl_verify_mode": 1}}, + "chef.ssl_verify_mode: 1 is not of type 'string'", + ), + ( + {"chef": {"validation_name": 1}}, + "chef.validation_name: 1 is not of type 'string'", + ), + ( + {"chef": {"force_install": 1}}, + "chef.force_install: 1 is not of type 'boolean'", + ), + ( + {"chef": {"initial_attributes": 1}}, + "chef.initial_attributes: 1 is not of type 'object'", + ), + ( + {"chef": {"install_type": 1}}, + "chef.install_type: 1 is not of type 'string'", + ), + ( + {"chef": {"install_type": "bogusenum"}}, + re.escape( + "chef.install_type: 'bogusenum' is not one of" + " ['packages', 'gems', 'omnibus']" + ), + ), + ( + {"chef": {"run_list": 1}}, + "chef.run_list: 1 is not of type 'array'", + ), + ( + {"chef": {"chef_license": 1}}, + "chef.chef_license: 1 is not of type 'string'", + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + """Assert expected schema validation and error messages.""" + # New-style schema $defs exist in config/cloud-init-schema*.json + schema = get_schema() + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) + + # vi: ts=4 expandtab diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index c5b50d66411..008969e6b71 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -93,6 +93,7 @@ def test_get_schema_coalesces_known_schema(self): "cc_bootcmd", "cc_byobu", "cc_ca_certs", + "cc_chef", "cc_keyboard", "cc_locale", "cc_ntp", @@ -103,7 +104,6 @@ def test_get_schema_coalesces_known_schema(self): "cc_ubuntu_drivers", "cc_write_files", "cc_zypper_add_repo", - "cc_chef", "cc_install_hotplug", ] ) == sorted( @@ -119,6 +119,7 @@ def test_get_schema_coalesces_known_schema(self): {"$ref": "#/$defs/cc_bootcmd"}, {"$ref": "#/$defs/cc_byobu"}, {"$ref": "#/$defs/cc_ca_certs"}, + {"$ref": "#/$defs/cc_chef"}, ] found_subschema_defs = [] legacy_schema_keys = [] From ac02327626a45b0c6ee2454fd0b09138a88a56d0 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 24 Jan 2022 20:58:17 -0700 Subject: [PATCH 06/13] schema: import James cc_debug schema to cloud-init-schema migrate to statis cloud-init-schema.json and add unittests --- cloudinit/config/cc_debug.py | 55 ++++++++++++++----------- cloudinit/config/cloud-init-schema.json | 23 ++++++++++- tests/unittests/config/test_cc_debug.py | 54 +++++++++++++++++++++++- tests/unittests/config/test_schema.py | 2 + 4 files changed, 109 insertions(+), 25 deletions(-) diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index d09fc129a1d..1a3c9346abd 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -2,37 +2,47 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Debug ------ -**Summary:** helper to debug cloud-init *internal* datastructures. +"""Debug: Helper to debug cloud-init *internal* datastructures.""" + +import copy +from io import StringIO +from textwrap import dedent + +from cloudinit import safeyaml, type_utils, util +from cloudinit.config.schema import get_meta_doc +from cloudinit.distros import ALL_DISTROS +from cloudinit.settings import PER_INSTANCE + +SKIP_KEYS = frozenset(["log_cfgs"]) +MODULE_DESCRIPTION = """\ This module will enable for outputting various internal information that cloud-init sources provide to either a file or to the output console/log location that this cloud-init has been configured with when running. .. note:: Log configurations are not output. - -**Internal name:** ``cc_debug`` - -**Module frequency:** per instance - -**Supported distros:** all - -**Config keys**:: - - debug: - verbose: true/false (defaulting to true) - output: (location to write output, defaulting to console + log) """ -import copy -from io import StringIO - -from cloudinit import safeyaml, type_utils, util - -SKIP_KEYS = frozenset(["log_cfgs"]) +meta = { + "id": "cc_debug", + "name": "Debug", + "title": "Helper to debug cloud-init *internal* datastructures", + "description": MODULE_DESCRIPTION, + "distros": [ALL_DISTROS], + "frequency": PER_INSTANCE, + "examples": [ + dedent( + """\ + debug: + verbose: true + output: /tmp/my_debug.log + """ + ) + ], +} + +__doc__ = get_meta_doc(meta) def _make_header(text): @@ -53,7 +63,6 @@ def _dumps(obj): def handle(name, cfg, cloud, log, args): """Handler method activated by cloud-init.""" - verbose = util.get_cfg_by_path(cfg, ("debug", "verbose"), default=True) if args: # if args are provided (from cmdline) then explicitly set verbose diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index 311e7a0f4a3..3511e4316e2 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -392,6 +392,26 @@ } } } + }, + "cc_debug": { + "type": "object", + "properties": { + "debug": { + "additionalProperties": false, + "minProperties": 1, + "type": "object", + "properties": { + "verbose": { + "description": "Should always be true for this module", + "type": "boolean" + }, + "output": { + "description": "Location to write output. Defaults to console + log", + "type": "string" + } + } + } + } } }, "allOf": [ @@ -401,6 +421,7 @@ { "$ref": "#/$defs/cc_bootcmd" }, { "$ref": "#/$defs/cc_byobu" }, { "$ref": "#/$defs/cc_ca_certs" }, - { "$ref": "#/$defs/cc_chef" } + { "$ref": "#/$defs/cc_chef" }, + { "$ref": "#/$defs/cc_debug" } ] } diff --git a/tests/unittests/config/test_cc_debug.py b/tests/unittests/config/test_cc_debug.py index 79a88561983..fc8d43dc240 100644 --- a/tests/unittests/config/test_cc_debug.py +++ b/tests/unittests/config/test_cc_debug.py @@ -2,12 +2,24 @@ # # This file is part of cloud-init. See LICENSE file for license information. import logging +import re import shutil import tempfile +import pytest + from cloudinit import util from cloudinit.config import cc_debug -from tests.unittests.helpers import FilesystemMockingTestCase, mock +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import ( + FilesystemMockingTestCase, + mock, + skipUnlessJsonSchema, +) from tests.unittests.util import get_cloud LOG = logging.getLogger(__name__) @@ -57,4 +69,44 @@ def test_debug_no_write(self, m_locale): ) +@skipUnlessJsonSchema() +class TestDebugSchema: + """Directly test schema rather than through handle.""" + + @pytest.mark.parametrize( + "config, error_msg", + ( + # Valid schemas tested by meta.examples in test_schema + # Invalid schemas + ({"debug": 1}, "debug: 1 is not of type 'object'"), + ( + {"debug": {}}, + re.escape("debug: {} does not have enough properties"), + ), + ( + {"debug": {"boguskey": True}}, + re.escape( + "Additional properties are not allowed ('boguskey' was" + " unexpected)" + ), + ), + ( + {"debug": {"verbose": 1}}, + "debug.verbose: 1 is not of type 'boolean'", + ), + ( + {"debug": {"output": 1}}, + "debug.output: 1 is not of type 'string'", + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + """Assert expected schema validation and error messages.""" + # New-style schema $defs exist in config/cloud-init-schema*.json + schema = get_schema() + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) + + # vi: ts=4 expandtab diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 008969e6b71..e66de93eefb 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -94,6 +94,7 @@ def test_get_schema_coalesces_known_schema(self): "cc_byobu", "cc_ca_certs", "cc_chef", + "cc_debug", "cc_keyboard", "cc_locale", "cc_ntp", @@ -120,6 +121,7 @@ def test_get_schema_coalesces_known_schema(self): {"$ref": "#/$defs/cc_byobu"}, {"$ref": "#/$defs/cc_ca_certs"}, {"$ref": "#/$defs/cc_chef"}, + {"$ref": "#/$defs/cc_debug"}, ] found_subschema_defs = [] legacy_schema_keys = [] From 7da4f29c11ed6f0a06ee7c622156321901efda8e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 25 Jan 2022 12:51:53 -0700 Subject: [PATCH 07/13] schema: migrate James cc_disable_ec2_metadata schema to static schema Migrate to static cloud-init-schema.json and add unittests --- cloudinit/config/cc_disable_ec2_metadata.py | 41 ++++++++++--------- cloudinit/config/cloud-init-schema.json | 13 +++++- .../config/test_cc_disable_ec2_metadata.py | 33 ++++++++++++++- tests/unittests/config/test_schema.py | 4 +- 4 files changed, 68 insertions(+), 23 deletions(-) diff --git a/cloudinit/config/cc_disable_ec2_metadata.py b/cloudinit/config/cc_disable_ec2_metadata.py index 5e528e810f0..6a5e7eda302 100644 --- a/cloudinit/config/cc_disable_ec2_metadata.py +++ b/cloudinit/config/cc_disable_ec2_metadata.py @@ -6,34 +6,35 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Disable EC2 Metadata --------------------- -**Summary:** disable aws ec2 metadata +"""Disable EC2 Metadata: Disable AWS EC2 metadata.""" -This module can disable the ec2 datasource by rejecting the route to -``169.254.169.254``, the usual route to the datasource. This module is disabled -by default. - -**Internal name:** ``cc_disable_ec2_metadata`` - -**Module frequency:** always - -**Supported distros:** all - -**Config keys**:: - - disable_ec2_metadata: -""" +from textwrap import dedent from cloudinit import subp, util +from cloudinit.config.schema import get_meta_doc +from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_ALWAYS -frequency = PER_ALWAYS - REJECT_CMD_IF = ["route", "add", "-host", "169.254.169.254", "reject"] REJECT_CMD_IP = ["ip", "route", "add", "prohibit", "169.254.169.254"] +meta = { + "id": "cc_disable_ec2_metadata", + "name": "Disable EC2 Metadata", + "title": "Disable AWS EC2 Metadata", + "description": dedent( + """\ + This module can disable the ec2 datasource by rejecting the route to + ``169.254.169.254``, the usual route to the datasource. This module + is disabled by default.""" + ), + "distros": [ALL_DISTROS], + "frequency": PER_ALWAYS, + "examples": ["disable_ec2_metadata: true"], +} + +__doc__ = get_meta_doc(meta) + def handle(name, cfg, _cloud, log, _args): disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False) diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index 3511e4316e2..aecd7542eb3 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -412,6 +412,16 @@ } } } + }, + "cc_disable_ec2_metadata": { + "type": "object", + "properties": { + "disable_ec2_metadata": { + "default": false, + "description": "Set true to disable IPv4 routes to EC2 metadata. Default: false.", + "type": "boolean" + } + } } }, "allOf": [ @@ -422,6 +432,7 @@ { "$ref": "#/$defs/cc_byobu" }, { "$ref": "#/$defs/cc_ca_certs" }, { "$ref": "#/$defs/cc_chef" }, - { "$ref": "#/$defs/cc_debug" } + { "$ref": "#/$defs/cc_debug" }, + { "$ref": "#/$defs/cc_disable_ec2_metadata" } ] } diff --git a/tests/unittests/config/test_cc_disable_ec2_metadata.py b/tests/unittests/config/test_cc_disable_ec2_metadata.py index 3c3313a7014..5755e29e42e 100644 --- a/tests/unittests/config/test_cc_disable_ec2_metadata.py +++ b/tests/unittests/config/test_cc_disable_ec2_metadata.py @@ -4,8 +4,15 @@ import logging +import pytest + import cloudinit.config.cc_disable_ec2_metadata as ec2_meta -from tests.unittests.helpers import CiTestCase, mock +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema LOG = logging.getLogger(__name__) @@ -47,4 +54,28 @@ def test_disable_no_tool(self, m_subp, m_which): m_subp.assert_not_called() +@skipUnlessJsonSchema() +class TestDisableEc2MetadataSchema: + """Directly test schema rather than through handle.""" + + @pytest.mark.parametrize( + "config, error_msg", + ( + # Valid schemas tested by meta.examples in test_schema + # Invalid schemas + ( + {"disable_ec2_metadata": 1}, + "disable_ec2_metadata: 1 is not of type 'boolean'", + ), + ), + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + """Assert expected schema validation and error messages.""" + # New-style schema $defs exist in config/cloud-init-schema*.json + schema = get_schema() + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, schema, strict=True) + + # vi: ts=4 expandtab diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index e66de93eefb..337d6f7d95c 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -95,6 +95,8 @@ def test_get_schema_coalesces_known_schema(self): "cc_ca_certs", "cc_chef", "cc_debug", + "cc_disable_ec2_metadata", + "cc_install_hotplug", "cc_keyboard", "cc_locale", "cc_ntp", @@ -105,7 +107,6 @@ def test_get_schema_coalesces_known_schema(self): "cc_ubuntu_drivers", "cc_write_files", "cc_zypper_add_repo", - "cc_install_hotplug", ] ) == sorted( [meta["id"] for meta in get_metas().values() if meta is not None] @@ -122,6 +123,7 @@ def test_get_schema_coalesces_known_schema(self): {"$ref": "#/$defs/cc_ca_certs"}, {"$ref": "#/$defs/cc_chef"}, {"$ref": "#/$defs/cc_debug"}, + {"$ref": "#/$defs/cc_disable_ec2_metadata"}, ] found_subschema_defs = [] legacy_schema_keys = [] From fb219cd748335c36d61919516079baa0bf5bee5a Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 25 Jan 2022 16:50:42 -0700 Subject: [PATCH 08/13] schema: forgot to drop cc_chef.schema --- cloudinit/config/cc_chef.py | 296 ------------------------------------ 1 file changed, 296 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index ba8b119301a..aaf7eaf10e3 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -137,302 +137,6 @@ "frequency": frequency, } -schema = { - "type": "object", - "properties": { - "chef": { - "type": "object", - "additionalProperties": False, - "properties": { - "directories": { - "type": "array", - "items": {"type": "string"}, - "uniqueItems": True, - "description": dedent( - """\ - Create the necessary directories for chef to run. By - default, it creates the following directories: - - {chef_dirs}""" - ).format( - chef_dirs="\n".join( - [" - ``{}``".format(d) for d in CHEF_DIRS] - ) - ), - }, - "validation_cert": { - "type": "string", - "description": dedent( - """\ - Optional string to be written to file validation_key. - Special value ``system`` means set use existing file. - """ - ), - }, - "validation_key": { - "type": "string", - "default": CHEF_VALIDATION_PEM_PATH, - "description": dedent( - """\ - Optional path for validation_cert. default to - ``{}``.""".format( - CHEF_VALIDATION_PEM_PATH - ) - ), - }, - "firstboot_path": { - "type": "string", - "default": CHEF_FB_PATH, - "description": dedent( - """\ - Path to write run_list and initial_attributes keys that - should also be present in this configuration, defaults - to ``{}``.""".format( - CHEF_FB_PATH - ) - ), - }, - "exec": { - "type": "boolean", - "default": False, - "description": dedent( - """\ - define if we should run or not run chef (defaults to - false, unless a gem installed is requested where this - will then default to true).""" - ), - }, - "client_key": { - "type": "string", - "default": CHEF_RB_TPL_DEFAULTS["client_key"], - "description": dedent( - """\ - Optional path for client_cert. default to - ``{}``.""".format( - CHEF_RB_TPL_DEFAULTS["client_key"] - ) - ), - }, - "encrypted_data_bag_secret": { - "type": "string", - "default": None, - "description": dedent( - """\ - Specifies the location of the secret key used by chef - to encrypt data items. By default, this path is set - to None, meaning that chef will have to look at the - path ``{}`` for it. - """.format( - CHEF_ENCRYPTED_DATA_BAG_PATH - ) - ), - }, - "environment": { - "type": "string", - "default": CHEF_ENVIRONMENT, - "description": dedent( - """\ - Specifies which environment chef will use. By default, - it will use the ``{}`` configuration. - """.format( - CHEF_ENVIRONMENT - ) - ), - }, - "file_backup_path": { - "type": "string", - "default": CHEF_RB_TPL_DEFAULTS["file_backup_path"], - "description": dedent( - """\ - Specifies the location in which backup files are - stored. By default, it uses the - ``{}`` location.""".format( - CHEF_RB_TPL_DEFAULTS["file_backup_path"] - ) - ), - }, - "file_cache_path": { - "type": "string", - "default": CHEF_RB_TPL_DEFAULTS["file_cache_path"], - "description": dedent( - """\ - Specifies the location in which chef cache files will - be saved. By default, it uses the ``{}`` - location.""".format( - CHEF_RB_TPL_DEFAULTS["file_cache_path"] - ) - ), - }, - "json_attribs": { - "type": "string", - "default": CHEF_FB_PATH, - "description": dedent( - """\ - Specifies the location in which some chef json data is - stored. By default, it uses the - ``{}`` location.""".format( - CHEF_FB_PATH - ) - ), - }, - "log_level": { - "type": "string", - "default": CHEF_RB_TPL_DEFAULTS["log_level"], - "description": dedent( - """\ - Defines the level of logging to be stored in the log - file. By default this value is set to ``{}``. - """.format( - CHEF_RB_TPL_DEFAULTS["log_level"] - ) - ), - }, - "log_location": { - "type": "string", - "default": CHEF_RB_TPL_DEFAULTS["log_location"], - "description": dedent( - """\ - Specifies the location of the chef lof file. By - default, the location is specified at - ``{}``.""".format( - CHEF_RB_TPL_DEFAULTS["log_location"] - ) - ), - }, - "node_name": { - "type": "string", - "description": dedent( - """\ - The name of the node to run. By default, we will - use th instance id as the node name.""" - ), - }, - "omnibus_url": { - "type": "string", - "default": OMNIBUS_URL, - "description": dedent( - """\ - Omnibus URL if chef should be installed through - Omnibus. By default, it uses the - ``{}``.""".format( - OMNIBUS_URL - ) - ), - }, - "omnibus_url_retries": { - "type": "integer", - "default": OMNIBUS_URL_RETRIES, - "description": dedent( - """\ - The number of retries that will be attempted to reach - the Omnibus URL""" - ), - }, - "omnibus_version": { - "type": "string", - "description": dedent( - """\ - Optional version string to require for omnibus - install.""" - ), - }, - "pid_file": { - "type": "string", - "default": CHEF_RB_TPL_DEFAULTS["pid_file"], - "description": dedent( - """\ - The location in which a process identification - number (pid) is saved. By default, it saves - in the ``{}`` location.""".format( - CHEF_RB_TPL_DEFAULTS["pid_file"] - ) - ), - }, - "server_url": { - "type": "string", - "description": "The URL for the chef server", - }, - "show_time": { - "type": "boolean", - "default": True, - "description": "Show time in chef logs", - }, - "ssl_verify_mode": { - "type": "string", - "default": CHEF_RB_TPL_DEFAULTS["ssl_verify_mode"], - "description": dedent( - """\ - Set the verify mode for HTTPS requests. We can have - two possible values for this parameter: - - - ``:verify_none``: No validation of SSL \ - certificates. - - ``:verify_peer``: Validate all SSL certificates. - - By default, the parameter is set as ``{}``. - """.format( - CHEF_RB_TPL_DEFAULTS["ssl_verify_mode"] - ) - ), - }, - "validation_name": { - "type": "string", - "description": dedent( - """\ - The name of the chef-validator key that Chef Infra - Client uses to access the Chef Infra Server during - the initial Chef Infra Client run.""" - ), - }, - "force_install": { - "type": "boolean", - "default": False, - "description": dedent( - """\ - If set to ``True``, forces chef installation, even - if it is already installed.""" - ), - }, - "initial_attributes": { - "type": "object", - "items": {"type": "string"}, - "description": dedent( - """\ - Specify a list of initial attributes used by the - cookbooks.""" - ), - }, - "install_type": { - "type": "string", - "default": "packages", - "description": dedent( - """\ - The type of installation for chef. It can be one of - the following values: - - - ``packages`` - - ``gems`` - - ``omnibus``""" - ), - }, - "run_list": { - "type": "array", - "items": {"type": "string"}, - "description": "A run list for a first boot json.", - }, - "chef_license": { - "type": "string", - "description": dedent( - """\ - string that indicates if user accepts or not license - related to some of chef products""" - ), - }, - }, - } - }, -} - __doc__ = get_meta_doc(meta) From c1f38f89bddfafb7642c636bfab8bc66ceedd3af Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 25 Jan 2022 16:50:55 -0700 Subject: [PATCH 09/13] schema: migrate James cc_disk_setup schema to static schema Migrate to static cloud-init-schema.json and add unittests Needs more unittests for failure cases --- cloudinit/config/cc_disk_setup.py | 159 +++++++------------ cloudinit/config/cloud-init-schema.json | 114 ++++++++++++- tests/unittests/config/test_cc_disk_setup.py | 50 +++++- tests/unittests/config/test_schema.py | 3 +- 4 files changed, 224 insertions(+), 102 deletions(-) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 4d527c7a5b7..c59d00cd1fa 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -5,110 +5,18 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Disk Setup ----------- -**Summary:** configure partitions and filesystems - -This module is able to configure simple partition tables and filesystems. - -.. note:: - for more detail about configuration options for disk setup, see the disk - setup example - -For convenience, aliases can be specified for disks using the -``device_aliases`` config key, which takes a dictionary of alias: path -mappings. There are automatic aliases for ``swap`` and ``ephemeral``, where -``swap`` will always refer to the active swap partition and ``ephemeral`` -will refer to the block device of the ephemeral image. - -Disk partitioning is done using the ``disk_setup`` directive. This config -directive accepts a dictionary where each key is either a path to a block -device or an alias specified in ``device_aliases``, and each value is the -configuration options for the device. The ``table_type`` option specifies the -partition table type, either ``mbr`` or ``gpt``. The ``layout`` option -specifies how partitions on the device are to be arranged. If ``layout`` is set -to ``true``, a single partition using all the space on the device will be -created. If set to ``false``, no partitions will be created. Partitions can be -specified by providing a list to ``layout``, where each entry in the list is -either a size or a list containing a size and the numerical value for a -partition type. The size for partitions is specified in **percentage** of disk -space, not in bytes (e.g. a size of 33 would take up 1/3 of the disk space). -The ``overwrite`` option controls whether this module tries to be safe about -writing partition tables or not. If ``overwrite: false`` is set, the device -will be checked for a partition table and for a file system and if either is -found, the operation will be skipped. If ``overwrite: true`` is set, no checks -will be performed. - -.. note:: - Using ``overwrite: true`` is dangerous and can lead to data loss, so double - check that the correct device has been specified if using this option. - -File system configuration is done using the ``fs_setup`` directive. This config -directive accepts a list of filesystem configs. The device to create the -filesystem on may be specified either as a path or as an alias in the format -``.`` where ```` denotes the partition number on the device. -The partition can also be specified by setting ``partition`` to the desired -partition number. The ``partition`` option may also be set to ``auto``, in -which this module will search for the existence of a filesystem matching the -``label``, ``type`` and ``device`` of the ``fs_setup`` entry and will skip -creating the filesystem if one is found. The ``partition`` option may also be -set to ``any``, in which case any file system that matches ``type`` and -``device`` will cause this module to skip filesystem creation for the -``fs_setup`` entry, regardless of ``label`` matching or not. To write a -filesystem directly to a device, use ``partition: none``. ``partition: none`` -will **always** write the filesystem, even when the ``label`` and -``filesystem`` are matched, and ``overwrite`` is ``false``. - -A label can be specified for the filesystem using -``label``, and the filesystem type can be specified using ``filesystem``. - -.. note:: - If specifying device using the ``.`` format, - the value of ``partition`` will be overwritten. - -.. note:: - Using ``overwrite: true`` for filesystems is dangerous and can lead to data - loss, so double check the entry in ``fs_setup``. - -.. note:: - ``replace_fs`` is ignored unless ``partition`` is ``auto`` or ``any``. - -**Internal name:** ``cc_disk_setup`` - -**Module frequency:** per instance - -**Supported distros:** all - -**Config keys**:: - - device_aliases: - : - disk_setup: - : - table_type: <'mbr'/'gpt'> - layout: - - [33,82] - - 66 - overwrite: - fs_setup: - - label: