From 3b5567bec45a0e3d221a0480670d8b658e24efac Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 13 Apr 2022 14:49:34 -0600 Subject: [PATCH 01/12] schema: add json defs for modules cc_users_groups --- cloudinit/config/cc_users_groups.py | 190 ++++++++---------- cloudinit/config/cloud-init-schema.json | 144 +++++++++++++ cloudinit/config/schema.py | 25 ++- doc/examples/cloud-config-user-groups.txt | 11 +- .../unittests/config/test_cc_users_groups.py | 34 +++- tests/unittests/config/test_schema.py | 4 +- 6 files changed, 283 insertions(+), 125 deletions(-) diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index ef77a7998a9..2fd512f8bd9 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -4,72 +4,32 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Users and Groups ----------------- -**Summary:** configure users and groups +"Users and Groups: Configure users and groups" + +from textwrap import dedent + +from cloudinit import log as logging +from cloudinit.config.schema import MetaSchema, get_meta_doc + +# Ensure this is aliased to a name not 'distros' +# since the module attribute 'distros' +# is a list of distros that are supported, not a sub-module +from cloudinit.distros import ug_util +from cloudinit.settings import PER_INSTANCE +MODULE_DESCRIPTION = """\ This module configures users and groups. For more detailed information on user -options, see the ``Including users and groups`` config example. +options, see the :ref:`Including users and groups` config +example. Groups to add to the system can be specified as a list under the ``groups`` key. Each entry in the list should either contain a the group name as a string, or a dictionary with the group name as the key and a list of users who should -be members of the group as the value. **Note**: Groups are added before users, -so any users in a group list must already exist on the system. - -The ``users`` config key takes a list of users to configure. The first entry in -this list is used as the default user for the system. To preserve the standard -default user for the distro, the string ``default`` may be used as the first -entry of the ``users`` list. Each entry in the ``users`` list, other than a -``default`` entry, should be a dictionary of options for the user. Supported -config keys for an entry in ``users`` are as follows: - - - ``name``: The user's login name - - ``expiredate``: Optional. Date on which the user's account will be - disabled. Default: none - - ``gecos``: Optional. Comment about the user, usually a comma-separated - string of real name and contact information. Default: none - - ``groups``: Optional. Additional groups to add the user to. Default: none - - ``homedir``: Optional. Home dir for user. Default is ``/home/`` - - ``inactive``: Optional. Number of days after a password expires until - the account is permanently disabled. Default: none - - ``lock_passwd``: Optional. Disable password login. Default: true - - ``no_create_home``: Optional. Do not create home directory. Default: - false - - ``no_log_init``: Optional. Do not initialize lastlog and faillog for - user. Default: false - - ``no_user_group``: Optional. Do not create group named after user. - Default: false - - ``passwd``: Hash of user password - - ``primary_group``: Optional. Primary group for user. Default to new group - named after user. - - ``selinux_user``: Optional. SELinux user for user's login. Default to - default SELinux user. - - ``shell``: Optional. The user's login shell. The default is to set no - shell, which results in a system-specific default being used. - - ``snapuser``: Optional. Specify an email address to create the user as - a Snappy user through ``snap create-user``. If an Ubuntu SSO account is - associated with the address, username and SSH keys will be requested from - there. Default: none - - ``ssh_authorized_keys``: Optional. List of SSH keys to add to user's - authkeys file. Default: none. This key can not be combined with - ``ssh_redirect_user``. - - ``ssh_import_id``: Optional. SSH id to import for user. Default: none. - This key can not be combined with ``ssh_redirect_user``. - - ``ssh_redirect_user``: Optional. Boolean set to true to disable SSH - logins for this user. When specified, all cloud meta-data public SSH - keys will be set up in a disabled state for this username. Any SSH login - as this username will timeout and prompt with a message to login instead - as the configured for this instance. Default: false. - This key can not be combined with ``ssh_import_id`` or - ``ssh_authorized_keys``. - - ``sudo``: Optional. Sudo rule to use, list of sudo rules to use or False. - Default: none. An absence of sudo key, or a value of none or false - will result in no sudo rules being written for the user. - - ``system``: Optional. Create user as system user with no home directory. - Default: false - - ``uid``: Optional. The user's ID. Default: The next available value. +be members of the group as the value. + +.. note:: + Groups are added before users, so any users in a group list must + already exist on the system. .. note:: Specifying a hash of a user's password with ``passwd`` is a security risk @@ -84,61 +44,73 @@ already exists. The following options are the exceptions; they are applied to already-existing users: ``plain_text_passwd``, ``hashed_passwd``, ``lock_passwd``, ``sudo``, ``ssh_authorized_keys``, ``ssh_redirect_user``. - -**Internal name:** ``cc_users_groups`` - -**Module frequency:** per instance - -**Supported distros:** all - -**Config keys**:: - - groups: - - : [, ] - - - - users: - - default - # User explicitly omitted from sudo permission; also default behavior. - - name: - sudo: false - - name: - expiredate: '' - gecos: - groups: - homedir: - inactive: '' - lock_passwd: - no_create_home: - no_log_init: - no_user_group: - passwd: - primary_group: - selinux_user: - shell: - snapuser: - ssh_redirect_user: - ssh_authorized_keys: - - - - - ssh_import_id: - sudo: - system: - uid: """ -from cloudinit import log as logging +meta: MetaSchema = { + "id": "cc_users_groups", + "name": "Users and Groups", + "title": "Configure users and groups", + "description": MODULE_DESCRIPTION, + "distros": ["all"], + "examples": [ + dedent( + """\ + # Add the 'admingroup' with members 'root' and 'sys' and an empty + # group cloud-users. + groups: + - admingroup: [root,sys] + - cloud-users + """ + ), + dedent( + """\ + # Skip creation of the user and only create newsuper. + # Password-based login is rejected, but the github user TheRealFalcon + # and the launchpad user falcojr can SSH as newsuper. The default + # shell for newsuper is bash instead of system default. + users: + - name: newsuper + gecos: Big Stuff + groups: users, admin + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + lock_passwd: true + ssh_import_id: + - lp:falcojr + - gh:TheRealFalcon + """ + ), + dedent( + """\ + # On a system with SELinux enabled, add youruser and set the + # SELinux user to 'staff_u'. When omitted on SELinux, the system will + # select the configured default SELinux user. + users: + - default + - name: youruser + selinux_user: staff_u + """ + ), + dedent( + """\ + # To redirect a legacy username to the user for a + # distribution, ssh_redirect_user will accept an SSH connection and + # emit a message telling the client to ssh as the user. + # SSH clients will get the message: + users: + - default + - name: nosshlogins + ssh_redirect_user: true + """ + ), + ], + "frequency": PER_INSTANCE, +} -# Ensure this is aliased to a name not 'distros' -# since the module attribute 'distros' -# is a list of distros that are supported, not a sub-module -from cloudinit.distros import ug_util -from cloudinit.settings import PER_INSTANCE +__doc__ = get_meta_doc(meta) LOG = logging.getLogger(__name__) -frequency = PER_INSTANCE - def handle(name, cfg, cloud, _log, _args): (users, groups) = ug_util.normalize_users_groups(cfg, cloud.distro) diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index 02a73c0615b..d4a8020f867 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -1,6 +1,127 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "$defs": { + "users_groups.group_with_users": { + "type": ["string", "object"], + "patternProperties": { + "^.+$": { + "label": "", + "description": "Optional list of usernames to add to the group", + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + } + } + }, + "users_groups.user": { + "type": "object", + "oneOf": [ + {"required": ["name"]}, + {"required": ["snapuser"]} + ], + "properties": { + "name": { + "description": "The user's login name. Required otherwise user creation will be skipped for this user.", + "type": "string" + }, + "expiredate": { + "default": null, + "description": "Optional. Date on which the user's account will be disabled. Default: ``null``", + "type": "string" + }, + "gecos": { + "description": "Optional comment about the user, usually a comma-separated string of real name and contact information", + "type": "string" + }, + "groups": { + "description": "Optional comma-separated list of groups to add the user to.", + "type": "string" + }, + "homedir": { + "description": "Optional home dir for user. Default: ``/home/``", + "default": "``/home/``", + "type": "string" + }, + "inactive": { + "description": "Optional string representing the number of days until the user is disabled. ", + "type": "string" + }, + "lock_passwd": { + "default": true, + "description": "Disable password login. Default: ``true``", + "type": "boolean" + }, + "no_create_home": { + "default": false, + "description": "Do not create home directory. Default: ``false``", + "type": "boolean" + }, + "no_log_init": { + "default": false, + "description": "Do not initialize lastlog and faillog for user. Default: ``false``", + "type": "boolean" + }, + "no_user_group": { + "default": false, + "description": " Do not create group named after user. Default: ``false``", + "type": "boolean" + }, + "passwd": { + "description": "Hash of user password. To generate this hash, run: mkpasswd --method=SHA-512 --rounds=4096. **Note:** While hashed password is better than plain text, using passwd in user-data represents a security risk as user-data could be accessible by third-parties depending on your cloud platform.", + "type": "string" + }, + "primary_group": { + "default": "````", + "description": "Primary group for user. Default: ````", + "type": "string" + }, + "selinux_user": { + "description": "SELinux user for user's login. Default to default SELinux user.", + "type": "string" + }, + "shell": { + "description": "Path to the user's login shell. The default is to set no shell, which results in a system-specific default being used.", + "type": "string" + }, + "snapuser": { + "description": " Specify an email address to create the user as a Snappy user through ``snap create-user``. If an Ubuntu SSO account is associated with the address, username and SSH keys will be requested from there.", + "type": "string" + }, + "ssh_authorized_keys": { + "description": "List of SSH keys to add to user's authkeys file. Can not be combined with ``ssh_redirect_user``", + "type": "array", + "items": {"type": "string"}, + "minItems": 1 + }, + "ssh_import_id": { + "description": "List of SSH IDs to import for user. Can not be combined with ``ssh_redirect_user``.", + "type": "array", + "items": {"type": "string"}, + "minItems": 1 + }, + "ssh_redirect_user": { + "type": "boolean", + "default": false, + "description": "Boolean set to true to disable SSH logins for this user. When specified, all cloud meta-data public SSH keys will be set up in a disabled state for this username. Any SSH login as this username will timeout and prompt with a message to login instead as the ``default_username`` for this instance. Default: ``false``. This key can not be combined with ``ssh_import_id`` or ``ssh_authorized_keys``." + }, + "system": { + "description": "Optional. Create user as system user with no home directory. Default: ``false``.", + "type": "boolean", + "default": false + }, + "sudo": { + "type": ["boolean", "string"], + "description": "Sudo rule to use or false. Absence of a sudo value or ``false`` will result in no sudo rules added for this user. DEPRECATED: the value ``false`` will be deprecated in the future release. Use ``null`` or no ``sudo`` key instead." + }, + "uid": { + "description": "The user's ID. Default is next available value.", + "type": "integer" + } + }, + "additionalProperties": false + }, "apt_configure.mirror": { "type": "array", "items": { @@ -1845,6 +1966,28 @@ } } }, + "cc_users_groups": { + "type": "object", + "properties": { + "groups": { + "type": "array", + "items": { + "$ref": "#/$defs/users_groups.group_with_users" + }, + "minItems": 1 + }, + "users": { + "type": "array", + "items": { + "oneOf": [ + {"enum": ["default"]}, + {"$ref": "#/$defs/users_groups.user"} + ] + }, + "minItems": 1 + } + } + }, "cc_write_files": { "type": "object", "properties": { @@ -2034,6 +2177,7 @@ { "$ref": "#/$defs/cc_ubuntu_drivers"}, { "$ref": "#/$defs/cc_update_etc_hosts"}, { "$ref": "#/$defs/cc_update_hostname"}, + { "$ref": "#/$defs/cc_users_groups"}, { "$ref": "#/$defs/cc_write_files"}, { "$ref": "#/$defs/cc_yum_add_repo"}, { "$ref": "#/$defs/cc_zypper_add_repo"} diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 40647f9f693..1bfb89fcca5 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -435,10 +435,11 @@ def _schemapath_for_cloudconfig(config, original_content): return schema_line_numbers -def _get_property_type(property_dict: dict) -> str: +def _get_property_type(property_dict: dict, defs: dict) -> str: """Return a string representing a property type from a given jsonschema. """ + _flatten_schema_refs(property_dict, defs) property_type = property_dict.get("type") if property_type is None: if property_dict.get("enum"): @@ -459,9 +460,11 @@ def _get_property_type(property_dict: dict) -> str: for sub_item in items.get("oneOf", {}): if sub_property_type: sub_property_type += "/" - sub_property_type += "(" + _get_property_type(sub_item) + ")" + sub_property_type += _get_property_type(sub_item, defs) if sub_property_type: - return "{0} of {1}".format(property_type, sub_property_type) + if "/" in sub_property_type: + sub_property_type = f"({sub_property_type})" + return f"{property_type} of {sub_property_type}" return property_type or "UNDEFINED" @@ -487,6 +490,15 @@ def _parse_description(description, prefix) -> str: return description +def _flatten_schema_refs(src_cfg: dict, defs: dict): + """Flatten schema: replace $refs in src_cfg with definitions from $defs.""" + if "$ref" not in src_cfg: + return + # Update the defined references in subschema for doc rendering + ref = defs[src_cfg["$ref"].replace("#/$defs/", "")] + src_cfg.update(ref) + + def _get_property_doc(schema: dict, defs: dict, prefix=" ") -> str: """Return restructured text describing the supported schema properties.""" new_prefix = prefix + " " @@ -498,10 +510,7 @@ def _get_property_doc(schema: dict, defs: dict, prefix=" ") -> str: for props in property_keys: for prop_key, prop_config in props.items(): - if "$ref" in prop_config: - # Update the defined references in subschema for doc rendering - ref = defs[prop_config["$ref"].replace("#/$defs/", "")] - prop_config.update(ref) + _flatten_schema_refs(prop_config, defs) # Define prop_name and description for SCHEMA_PROPERTY_TMPL description = prop_config.get("description", "") @@ -512,7 +521,7 @@ def _get_property_doc(schema: dict, defs: dict, prefix=" ") -> str: prefix=prefix, prop_name=label, description=_parse_description(description, prefix), - prop_type=_get_property_type(prop_config), + prop_type=_get_property_type(prop_config, defs), ) ) items = prop_config.get("items") diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt index f8fb3f9265a..987d9508231 100644 --- a/doc/examples/cloud-config-user-groups.txt +++ b/doc/examples/cloud-config-user-groups.txt @@ -3,7 +3,7 @@ # The following example adds the ubuntu group with members 'root' and 'sys' # and the empty group cloud-users. groups: - - ubuntu: [root,sys] + - admingroup: [root,sys] - cloud-users # Add users to the system. Users are added after groups are added. @@ -34,17 +34,16 @@ users: - gh:TheRealFalcon lock_passwd: true ssh_authorized_keys: - - - - + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSL7uWGj8cgWyIOaspgKdVy0cKJ+UTjfv7jBOjG2H/GN8bJVXy72XAvnhM0dUM+CCs8FOf0YlPX+Frvz2hKInrmRhZVwRSL129PasD12MlI3l44u6IwS1o/W86Q+tkQYEljtqDOo0a+cOsaZkvUNzUyEXUwz/lmYa6G4hMKZH4NBj7nbAAF96wsMCoyNwbWryBnDYUr6wMbjRR1J9Pw7Xh7WRC73wy4Va2YuOgbD3V/5ZrFPLbWZW/7TFXVrql04QVbyei4aiFR5n//GvoqwQDNe58LmbzX/xvxyKJYdny2zXmdAhMxbrpFQsfpkJ9E/H5w0yOdSvnWbUoG5xNGoOB csmith@fringe - name: cloudy gecos: Magic Cloud App Daemon User inactive: '5' system: true - name: fizzbuzz - sudo: False + sudo: false + shell: /bin/bash ssh_authorized_keys: - - - - + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSL7uWGj8cgWyIOaspgKdVy0cKJ+UTjfv7jBOjG2H/GN8bJVXy72XAvnhM0dUM+CCs8FOf0YlPX+Frvz2hKInrmRhZVwRSL129PasD12MlI3l44u6IwS1o/W86Q+tkQYEljtqDOo0a+cOsaZkvUNzUyEXUwz/lmYa6G4hMKZH4NBj7nbAAF96wsMCoyNwbWryBnDYUr6wMbjRR1J9Pw7Xh7WRC73wy4Va2YuOgbD3V/5ZrFPLbWZW/7TFXVrql04QVbyei4aiFR5n//GvoqwQDNe58LmbzX/xvxyKJYdny2zXmdAhMxbrpFQsfpkJ9E/H5w0yOdSvnWbUoG5xNGoOB csmith@fringe - snapuser: joe@joeuser.io - name: nosshlogins ssh_redirect_user: true diff --git a/tests/unittests/config/test_cc_users_groups.py b/tests/unittests/config/test_cc_users_groups.py index 0bd3c98091d..9eaea27a8d4 100644 --- a/tests/unittests/config/test_cc_users_groups.py +++ b/tests/unittests/config/test_cc_users_groups.py @@ -1,8 +1,15 @@ # This file is part of cloud-init. See LICENSE file for license information. +import re +import pytest from cloudinit.config import cc_users_groups -from tests.unittests.helpers import CiTestCase, mock +from cloudinit.config.schema import ( + SchemaValidationError, + get_schema, + validate_cloudconfig_schema, +) +from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema MODPATH = "cloudinit.config.cc_users_groups" @@ -266,3 +273,28 @@ def test_users_ssh_redirect_user_and_no_default(self, m_user, m_group): " cloud configuration users: [default, ..].\n", self.logs.getvalue(), ) + + +class TestUsersGroupsSchema: + @pytest.mark.parametrize( + "config, error_msg", + [ + # Validate default settings + ({"groups": ["anygrp"]}, None), + ({"users": ["default"]}, None), + ({"users": [{"name": "bbsw"}]}, None), + # minItems >= 1 for opaque-key + ( + {"groups": [{"needitems": []}]}, + re.escape("groups.0.needitems: [] is too short"), + ), + ({"groups": [{"yep": ["user1"]}]}, None), + ], + ) + @skipUnlessJsonSchema() + def test_schema_validation(self, config, error_msg): + if error_msg is None: + validate_cloudconfig_schema(config, get_schema(), strict=True) + else: + with pytest.raises(SchemaValidationError, match=error_msg): + validate_cloudconfig_schema(config, get_schema(), strict=True) diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 7630290a852..538cb511016 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -148,6 +148,7 @@ def test_get_schema_coalesces_known_schema(self): "cc_ubuntu_drivers", "cc_update_etc_hosts", "cc_update_hostname", + "cc_users_groups", "cc_write_files", "cc_yum_add_repo", "cc_zypper_add_repo", @@ -206,6 +207,7 @@ def test_get_schema_coalesces_known_schema(self): {"$ref": "#/$defs/cc_ubuntu_drivers"}, {"$ref": "#/$defs/cc_update_etc_hosts"}, {"$ref": "#/$defs/cc_update_hostname"}, + {"$ref": "#/$defs/cc_users_groups"}, {"$ref": "#/$defs/cc_write_files"}, {"$ref": "#/$defs/cc_yum_add_repo"}, {"$ref": "#/$defs/cc_zypper_add_repo"}, @@ -576,7 +578,7 @@ def test_get_meta_doc_handles_nested_oneof_property_types(self): } } self.assertIn( - "**prop1:** (array of (string)/(integer))", + "**prop1:** (array of (string/integer))", get_meta_doc(self.meta, schema), ) From f84c7b21ae76cc309dc45a28947f5d5444d257f9 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 13 Apr 2022 14:43:29 -0600 Subject: [PATCH 02/12] schema: markdown rendering updates for nested $refs - Support types as list for doc rendering versus single {"type": "string"} - Add _sort_property_order so 'array' and 'object' types are sorted last when multiple types are listed via oneOf or type lists - Decorate enum values with double back-ticks and replace "string" type - Add _flatten_schema_refs support for extracting $defs into schema for references nested under "items" and "oneOf" keys - Update unit tests for markdown doc coverage --- cloudinit/config/schema.py | 109 +++++++++---- tests/unittests/config/test_schema.py | 219 +++++++++++++++++--------- tests/unittests/test_cli.py | 2 +- 3 files changed, 220 insertions(+), 110 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 1bfb89fcca5..63095bd38df 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -42,9 +42,9 @@ {examples} """ SCHEMA_PROPERTY_HEADER = "**Config schema**:" -SCHEMA_PROPERTY_TMPL = "{prefix}**{prop_name}:** ({prop_type}) {description}" +SCHEMA_PROPERTY_TMPL = "{prefix}**{prop_name}:** ({prop_type}){description}" SCHEMA_LIST_ITEM_TMPL = ( - "{prefix}Each item in **{prop_name}** list supports the following keys:" + "{prefix}Each object in **{prop_name}** list supports the following keys:" ) SCHEMA_EXAMPLES_HEADER = "**Examples**::\n\n" SCHEMA_EXAMPLES_SPACER_TEMPLATE = "\n # --- Example{0} ---" @@ -435,36 +435,57 @@ def _schemapath_for_cloudconfig(config, original_content): return schema_line_numbers +def _sort_property_order(value): + """Provide a sorting weight for documentation of property types. + + Weight values ensure 'array' sorted after 'object' which is sorted + after anything else which remains unsorted. + """ + if value == "array": + return 2 + elif value == "object": + return 1 + return 0 + + def _get_property_type(property_dict: dict, defs: dict) -> str: """Return a string representing a property type from a given jsonschema. """ _flatten_schema_refs(property_dict, defs) - property_type = property_dict.get("type") - if property_type is None: - if property_dict.get("enum"): - property_type = [ - str(_YAML_MAP.get(k, k)) for k in property_dict["enum"] - ] - elif property_dict.get("oneOf"): - property_type = [ + property_types = property_dict.get("type", []) + if not isinstance(property_types, list): + property_types = [property_types] + if property_dict.get("enum"): + property_types = [ + f"``{_YAML_MAP.get(k, k)}``" for k in property_dict["enum"] + ] + elif property_dict.get("oneOf"): + property_types.extend( + [ subschema["type"] for subschema in property_dict.get("oneOf") if subschema.get("type") ] - if isinstance(property_type, list): - property_type = "/".join(property_type) + ) + if len(property_types) == 1: + property_type = property_types[0] + else: + property_types.sort(key=_sort_property_order) + property_type = "/".join(property_types) items = property_dict.get("items", {}) - sub_property_type = items.get("type", "") + sub_property_types = items.get("type", []) + if not isinstance(sub_property_types, list): + sub_property_types = [sub_property_types] # Collect each item type for sub_item in items.get("oneOf", {}): - if sub_property_type: - sub_property_type += "/" - sub_property_type += _get_property_type(sub_item, defs) - if sub_property_type: - if "/" in sub_property_type: - sub_property_type = f"({sub_property_type})" - return f"{property_type} of {sub_property_type}" + sub_property_types.append(_get_property_type(sub_item, defs)) + if sub_property_types: + if len(sub_property_types) == 1: + return f"{property_type} of {sub_property_types[0]}" + sub_property_types.sort(key=_sort_property_order) + sub_property_doc = f"({'/'.join(sub_property_types)})" + return f"{property_type} of {sub_property_doc}" return property_type or "UNDEFINED" @@ -492,11 +513,20 @@ def _parse_description(description, prefix) -> str: def _flatten_schema_refs(src_cfg: dict, defs: dict): """Flatten schema: replace $refs in src_cfg with definitions from $defs.""" - if "$ref" not in src_cfg: - return - # Update the defined references in subschema for doc rendering - ref = defs[src_cfg["$ref"].replace("#/$defs/", "")] - src_cfg.update(ref) + if "$ref" in src_cfg: + reference = src_cfg.pop("$ref").replace("#/$defs/", "") + # Update the defined references in subschema for doc rendering + src_cfg.update(defs[reference]) + if "items" in src_cfg: + if "$ref" in src_cfg["items"]: + reference = src_cfg["items"].pop("$ref").replace("#/$defs/", "") + # Update the references in subschema for doc rendering + src_cfg["items"].update(defs[reference]) + if "oneOf" in src_cfg["items"]: + for alt_schema in src_cfg["items"]["oneOf"]: + if "$ref" in alt_schema: + reference = alt_schema.pop("$ref").replace("#/$defs/", "") + alt_schema.update(defs[reference]) def _get_property_doc(schema: dict, defs: dict, prefix=" ") -> str: @@ -513,6 +543,8 @@ def _get_property_doc(schema: dict, defs: dict, prefix=" ") -> str: _flatten_schema_refs(prop_config, defs) # Define prop_name and description for SCHEMA_PROPERTY_TMPL description = prop_config.get("description", "") + if description: + description = " " + description # Define prop_name and description for SCHEMA_PROPERTY_TMPL label = prop_config.get("label", prop_key) @@ -526,16 +558,8 @@ def _get_property_doc(schema: dict, defs: dict, prefix=" ") -> str: ) items = prop_config.get("items") if items: - if isinstance(items, list): - for item in items: - properties.append( - _get_property_doc( - item, defs=defs, prefix=new_prefix - ) - ) - elif isinstance(items, dict) and ( - items.get("properties") or items.get("patternProperties") - ): + _flatten_schema_refs(items, defs) + if items.get("properties") or items.get("patternProperties"): properties.append( SCHEMA_LIST_ITEM_TMPL.format( prefix=new_prefix, prop_name=label @@ -545,6 +569,21 @@ def _get_property_doc(schema: dict, defs: dict, prefix=" ") -> str: properties.append( _get_property_doc(items, defs=defs, prefix=new_prefix) ) + for alt_schema in items.get("oneOf", []): + if alt_schema.get("properties") or alt_schema.get( + "patternProperties" + ): + properties.append( + SCHEMA_LIST_ITEM_TMPL.format( + prefix=new_prefix, prop_name=label + ) + ) + new_prefix += " " + properties.append( + _get_property_doc( + alt_schema, defs=defs, prefix=new_prefix + ) + ) if ( "properties" in prop_config or "patternProperties" in prop_config diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 538cb511016..a6bb5cff021 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -480,31 +480,29 @@ def test_validateconfig_file_sctrictly_validates_schema(self): ) -class GetSchemaDocTest(CiTestCase): +class TestSchemaDocMarkdown: """Tests for get_meta_doc.""" - def setUp(self): - super(GetSchemaDocTest, self).setUp() - self.required_schema = { - "title": "title", - "description": "description", - "id": "id", - "name": "name", - "frequency": "frequency", - "distros": ["debian", "rhel"], - } - self.meta: MetaSchema = { - "title": "title", - "description": "description", - "id": "id", - "name": "name", - "frequency": "frequency", - "distros": ["debian", "rhel"], - "examples": [ - 'ex1:\n [don\'t, expand, "this"]', - "ex2: true", - ], - } + required_schema = { + "title": "title", + "description": "description", + "id": "id", + "name": "name", + "frequency": "frequency", + "distros": ["debian", "rhel"], + } + meta: MetaSchema = { + "title": "title", + "description": "description", + "id": "id", + "name": "name", + "frequency": "frequency", + "distros": ["debian", "rhel"], + "examples": [ + 'ex1:\n [don\'t, expand, "this"]', + "ex2: true", + ], + } def test_get_meta_doc_returns_restructured_text(self): """get_meta_doc returns restructured text for a cloudinit schema.""" @@ -522,48 +520,89 @@ def test_get_meta_doc_returns_restructured_text(self): ) doc = get_meta_doc(self.meta, full_schema) - self.assertEqual( + assert ( dedent( """ - name - ---- - **Summary:** title + name + ---- + **Summary:** title - description + description - **Internal name:** ``id`` + **Internal name:** ``id`` - **Module frequency:** frequency + **Module frequency:** frequency - **Supported distros:** debian, rhel + **Supported distros:** debian, rhel - **Config schema**: - **prop1:** (array of integer) prop-description + **Config schema**: + **prop1:** (array of integer) prop-description - **Examples**:: + **Examples**:: - ex1: - [don't, expand, "this"] - # --- Example2 --- - ex2: true - """ - ), - doc, + ex1: + [don't, expand, "this"] + # --- Example2 --- + ex2: true + """ + ) + == doc ) def test_get_meta_doc_handles_multiple_types(self): """get_meta_doc delimits multiple property types with a '/'.""" schema = {"properties": {"prop1": {"type": ["string", "integer"]}}} - self.assertIn( - "**prop1:** (string/integer)", get_meta_doc(self.meta, schema) + assert "**prop1:** (string/integer)" in get_meta_doc(self.meta, schema) + + def test_references_are_flattened_in_schema_docs(self): + """get_meta_doc flattens and renders full schema definitions.""" + schema = { + "$defs": { + "flattenit": { + "type": ["object", "string"], + "description": "Objects support the following keys:", + "patternProperties": { + "^.+$": { + "label": "", + "description": "List of cool strings", + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + } + }, + } + }, + "properties": {"prop1": {"$ref": "#/$defs/flattenit"}}, + } + assert ( + dedent( + """\ + **prop1:** (string/object) Objects support the following keys: + + **:** (array of string) List of cool strings + """ + ) + in get_meta_doc(self.meta, schema) ) - def test_get_meta_doc_handles_enum_types(self): + @pytest.mark.parametrize( + "sub_schema,expected", + ( + ( + {"enum": [True, False, "stuff"]}, + "**prop1:** (``true``/``false``/``stuff``)", + ), + # When type: string and enum, document enum values + ( + {"type": "string", "enum": ["a", "b"]}, + "**prop1:** (``a``/``b``)", + ), + ), + ) + def test_get_meta_doc_handles_enum_types(self, sub_schema, expected): """get_meta_doc converts enum types to yaml and delimits with '/'.""" - schema = {"properties": {"prop1": {"enum": [True, False, "stuff"]}}} - self.assertIn( - "**prop1:** (true/false/stuff)", get_meta_doc(self.meta, schema) - ) + schema = {"properties": {"prop1": sub_schema}} + assert expected in get_meta_doc(self.meta, schema) def test_get_meta_doc_handles_nested_oneof_property_types(self): """get_meta_doc describes array items oneOf declarations in type.""" @@ -577,9 +616,41 @@ def test_get_meta_doc_handles_nested_oneof_property_types(self): } } } - self.assertIn( - "**prop1:** (array of (string/integer))", - get_meta_doc(self.meta, schema), + assert "**prop1:** (array of (string/integer))" in get_meta_doc( + self.meta, schema + ) + + def test_get_meta_doc_handles_types_as_list(self): + """get_meta_doc renders types which have a list value.""" + schema = { + "properties": { + "prop1": { + "type": ["boolean", "array"], + "items": { + "oneOf": [{"type": "string"}, {"type": "integer"}] + }, + } + } + } + assert ( + "**prop1:** (boolean/array of (string/integer))" + in get_meta_doc(self.meta, schema) + ) + + def test_get_meta_doc_handles_flattening_defs(self): + """get_meta_doc renders $defs.""" + schema = { + "$defs": { + "prop1object": { + "type": "object", + "properties": {"subprop": {"type": "string"}}, + } + }, + "properties": {"prop1": {"$ref": "#/$defs/prop1object"}}, + } + assert ( + "**prop1:** (object)\n\n **subprop:** (string)\n" + in get_meta_doc(self.meta, schema) ) def test_get_meta_doc_handles_string_examples(self): @@ -600,21 +671,21 @@ def test_get_meta_doc_handles_string_examples(self): }, } ) - self.assertIn( + assert ( dedent( """ - **Config schema**: - **prop1:** (array of integer) prop-description + **Config schema**: + **prop1:** (array of integer) prop-description - **Examples**:: + **Examples**:: - ex1: - [don't, expand, "this"] - # --- Example2 --- - ex2: true + ex1: + [don't, expand, "this"] + # --- Example2 --- + ex2: true """ - ), - get_meta_doc(self.meta, full_schema), + ) + in get_meta_doc(self.meta, full_schema) ) def test_get_meta_doc_properly_parse_description(self): @@ -640,21 +711,21 @@ def test_get_meta_doc_properly_parse_description(self): } } - self.assertIn( + assert ( dedent( """ - **Config schema**: - **p1:** (string) This item has the following options: + **Config schema**: + **p1:** (string) This item has the following options: - - option1 - - option2 - - option3 + - option1 + - option2 + - option3 - The default value is option1 + The default value is option1 - """ - ), - get_meta_doc(self.meta, schema), + """ + ) + in get_meta_doc(self.meta, schema) ) def test_get_meta_doc_raises_key_errors(self): @@ -672,9 +743,9 @@ def test_get_meta_doc_raises_key_errors(self): for key in self.meta: invalid_meta = copy(self.meta) invalid_meta.pop(key) - with self.assertRaises(KeyError) as context_mgr: + with pytest.raises(KeyError) as context_mgr: get_meta_doc(invalid_meta, schema) - self.assertIn(key, str(context_mgr.exception)) + assert key in str(context_mgr.value) def test_label_overrides_property_name(self): """get_meta_doc overrides property name with label.""" @@ -709,7 +780,7 @@ def test_label_overrides_property_name(self): assert "**label1:** (string)" in meta_doc assert "**label2:** (string" in meta_doc assert "**prop_no_label:** (string)" in meta_doc - assert "Each item in **array_label** list" in meta_doc + assert "Each object in **array_label** list" in meta_doc assert "prop1" not in meta_doc assert ".*" not in meta_doc diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index bed73a93aab..d2de9c8776a 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -260,7 +260,7 @@ def test_wb_devel_schema_subcommand_doc_all_spot_check(self): "openEuler, opensuse, photon, rhel, rocky, sles, ubuntu, " "virtuozzo", "**Config schema**:\n **resize_rootfs:** " - "(true/false/noblock)", + "(``true``/``false``/``noblock``)", "**Examples**::\n\n runcmd:\n - [ ls, -l, / ]\n", ] stdout = stdout.getvalue() From fb2ec569731ea769445fb6d33f9ba4e0bc00569e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 14 Apr 2022 10:56:46 -0600 Subject: [PATCH 03/12] schema: inline $defs/users_groups.group_with_users --- cloudinit/config/cloud-init-schema.json | 27 +++++++++++-------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index d4a8020f867..679a9f7256b 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -1,20 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "$defs": { - "users_groups.group_with_users": { - "type": ["string", "object"], - "patternProperties": { - "^.+$": { - "label": "", - "description": "Optional list of usernames to add to the group", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1 - } - } - }, "users_groups.user": { "type": "object", "oneOf": [ @@ -1972,7 +1958,18 @@ "groups": { "type": "array", "items": { - "$ref": "#/$defs/users_groups.group_with_users" + "type": ["string", "object"], + "patternProperties": { + "^.+$": { + "label": "", + "description": "Optional list of usernames to add to the group", + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + } + } }, "minItems": 1 }, From 10b98b38f1e32ca2c51d1ad72183793cf10273d4 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 14 Apr 2022 11:11:19 -0600 Subject: [PATCH 04/12] schema: disk_setup drop supplemental integer type --- cloudinit/config/cloud-init-schema.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index 679a9f7256b..f1a8c9f34f9 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -624,8 +624,7 @@ { "type": "string", "enum": ["auto", "any", "none"] - }, - {"type": "integer"} + } ], "description": "The partition can be specified by setting ``partition`` to the desired partition number. The ``partition`` option may also be set to ``auto``, in which this module will search for the existence of a filesystem matching the ``label``, ``type`` and ``device`` of the ``fs_setup`` entry and will skip creating the filesystem if one is found. The ``partition`` option may also be set to ``any``, in which case any file system that matches ``type`` and ``device`` will cause this module to skip filesystem creation for the ``fs_setup`` entry, regardless of ``label`` matching or not. To write a filesystem directly to a device, use ``partition: none``. ``partition: none`` will **always** write the filesystem, even when the ``label`` and ``filesystem`` are matched, and ``overwrite`` is ``false``." }, From 134cd6d94399da1149c6b0610486160342fd604d Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 14 Apr 2022 12:08:14 -0600 Subject: [PATCH 05/12] schema: users add hashed_passwd plain_text_passwd and create_groups defs --- cloudinit/config/cloud-init-schema.json | 17 +++++++++++++++-- cloudinit/distros/ug_util.py | 4 ++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index f1a8c9f34f9..c7bb4d6befc 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -51,13 +51,26 @@ }, "no_user_group": { "default": false, - "description": " Do not create group named after user. Default: ``false``", + "description": "Do not create group named after user. Default: ``false``", "type": "boolean" }, "passwd": { - "description": "Hash of user password. To generate this hash, run: mkpasswd --method=SHA-512 --rounds=4096. **Note:** While hashed password is better than plain text, using passwd in user-data represents a security risk as user-data could be accessible by third-parties depending on your cloud platform.", + "description": "Hash of user password applied when user does not exist. To generate this hash, run: mkpasswd --method=SHA-512 --rounds=4096. **Note:** While hashed password is better than plain text, using ``passwd`` in user-data represents a security risk as user-data could be accessible by third-parties depending on your cloud platform.", "type": "string" }, + "hashed_passwd": { + "description": "Hash of user password applied to new or existing users. To generate this hash, run: mkpasswd --method=SHA-512 --rounds=4096. **Note:** While ``hashed_password`` is better than ``plain_text_passwd``, using ``passwd`` in user-data represents a security risk as user-data could be accessible by third-parties depending on your cloud platform.", + "type": "string" + }, + "plain_text_passwd": { + "description": "Clear text of user password applied to new or existing users. There are many more secure options than using plain text passwords, such as ``ssh_import_id`` or ``hashed_passwd``. Do not use this in production as user-data and your password can be exposed.", + "type": "string" + }, + "create_groups": { + "default": true, + "description": "Boolean set ``false`` to disable creation of specified user ``groups``. Default: ``true``.", + "type": "boolean" + }, "primary_group": { "default": "````", "description": "Primary group for user. Default: ````", diff --git a/cloudinit/distros/ug_util.py b/cloudinit/distros/ug_util.py index 727663927e5..31f9b07e877 100755 --- a/cloudinit/distros/ug_util.py +++ b/cloudinit/distros/ug_util.py @@ -170,6 +170,10 @@ def normalize_users_groups(cfg, distro): # format. old_user = {} if "user" in cfg and cfg["user"]: + LOG.warning( + f"DEPRECATED: 'user' key is deprecated and will be removed in a" + " future release. Use 'users' instead." + ) old_user = cfg["user"] # Translate it into a format that will be more useful going forward if isinstance(old_user, str): From 18a9e9cb47f12c61c87027a6e0901ef34b0a5280 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 14 Apr 2022 15:31:00 -0600 Subject: [PATCH 06/12] schema: add user support and users/groups to support strings - Add deprecated 'user' schema support pointing at 'users'. - groups key can support comma-separated strings of groups instead of list of dicts - users can be string of comma-separated usernames instead of list --- cloudinit/config/cc_users_groups.py | 16 ++++++++++++++-- cloudinit/config/cloud-init-schema.json | 13 ++++++++++--- cloudinit/config/schema.py | 4 ++++ cloudinit/distros/ug_util.py | 4 ++-- tests/unittests/config/test_cc_users_groups.py | 9 ++++++++- 5 files changed, 38 insertions(+), 8 deletions(-) diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index 2fd512f8bd9..2afad059464 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -22,8 +22,9 @@ options, see the :ref:`Including users and groups` config example. -Groups to add to the system can be specified as a list under the ``groups`` -key. Each entry in the list should either contain a the group name as a string, +Groups to add to the system can be specified under the ``groups`` key as +a string of comma-separated groups to create, or a list. Each item in +the list should either contain a string of a single group to create, or a dictionary with the group name as the key and a list of users who should be members of the group as the value. @@ -31,6 +32,17 @@ Groups are added before users, so any users in a group list must already exist on the system. +Users to add can be specified as a list under the ``users`` key. Each entry in +the list should either be a string or a dictionary. When a string +is specified, that string can be comma-separated usernames to create or the +reserved value ``default``. The string ``default`` will setup the +``system_info:default_user`` as configured in ``/etc/cloud/cloud.cfg``. +Omission of ``default`` as the first item in the ``users`` list skips creation +the default_user. + +Each User dictionary must contain either a ``name`` or ``snapuser`` key, +otherwise it will be ignored. + .. note:: Specifying a hash of a user's password with ``passwd`` is a security risk if the cloud-config can be intercepted. SSH authentication is preferred. diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index c7bb4d6befc..a735ce3f265 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -22,7 +22,7 @@ "type": "string" }, "groups": { - "description": "Optional comma-separated list of groups to add the user to.", + "description": "Optional comma-separated string of groups to add the user to.", "type": "string" }, "homedir": { @@ -1968,7 +1968,7 @@ "type": "object", "properties": { "groups": { - "type": "array", + "type": ["string", "array"], "items": { "type": ["string", "object"], "patternProperties": { @@ -1985,11 +1985,18 @@ }, "minItems": 1 }, + "user": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/$defs/users_groups.user"} + ], + "description": "DEPRECATED, will be removed in a future release. Use ``users`` instead. This option creates a single default user. It overrides the ``default_user`` settings from /etc/cloud/cloud.cfg for a distribution. It accepts either a string of the default username to create or an object with any keys supported by the ``users`` schema." + }, "users": { "type": "array", "items": { "oneOf": [ - {"enum": ["default"]}, + {"type": "string"}, {"$ref": "#/$defs/users_groups.user"} ] }, diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 63095bd38df..bfdddf14041 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -527,6 +527,10 @@ def _flatten_schema_refs(src_cfg: dict, defs: dict): if "$ref" in alt_schema: reference = alt_schema.pop("$ref").replace("#/$defs/", "") alt_schema.update(defs[reference]) + for alt_schema in src_cfg.get("oneOf", []): + if "$ref" in alt_schema: + reference = alt_schema.pop("$ref").replace("#/$defs/", "") + alt_schema.update(defs[reference]) def _get_property_doc(schema: dict, defs: dict, prefix=" ") -> str: diff --git a/cloudinit/distros/ug_util.py b/cloudinit/distros/ug_util.py index 31f9b07e877..dd81e8c627b 100755 --- a/cloudinit/distros/ug_util.py +++ b/cloudinit/distros/ug_util.py @@ -171,8 +171,8 @@ def normalize_users_groups(cfg, distro): old_user = {} if "user" in cfg and cfg["user"]: LOG.warning( - f"DEPRECATED: 'user' key is deprecated and will be removed in a" - " future release. Use 'users' instead." + "DEPRECATED: 'user' key is deprecated and will be removed in a" + " future release. Use 'users' instead." ) old_user = cfg["user"] # Translate it into a format that will be more useful going forward diff --git a/tests/unittests/config/test_cc_users_groups.py b/tests/unittests/config/test_cc_users_groups.py index 9eaea27a8d4..3c72072927f 100644 --- a/tests/unittests/config/test_cc_users_groups.py +++ b/tests/unittests/config/test_cc_users_groups.py @@ -279,9 +279,12 @@ class TestUsersGroupsSchema: @pytest.mark.parametrize( "config, error_msg", [ - # Validate default settings + # Validate default settings not covered by examples ({"groups": ["anygrp"]}, None), + ({"groups": "anygrp,anyothergroup"}, None), + ({"user": "olddefault"}, None), ({"users": ["default"]}, None), + ({"users": ["foobar"]}, None), # non default user creation ({"users": [{"name": "bbsw"}]}, None), # minItems >= 1 for opaque-key ( @@ -289,6 +292,10 @@ class TestUsersGroupsSchema: re.escape("groups.0.needitems: [] is too short"), ), ({"groups": [{"yep": ["user1"]}]}, None), + ( + {"user": ["no_list_allowed"]}, + re.escape("user: ['no_list_allowed'] is not valid "), + ), ], ) @skipUnlessJsonSchema() From 6eeb9f01c43ebd4568a3c857ec16f55794c6cbe7 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 14 Apr 2022 15:49:41 -0600 Subject: [PATCH 07/12] schema: group object value can be a string of a single username or list --- cloudinit/config/cc_users_groups.py | 4 ++-- cloudinit/config/cloud-init-schema.json | 4 ++-- tests/unittests/config/test_cc_users_groups.py | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index 2afad059464..79a6d88130d 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -25,8 +25,8 @@ Groups to add to the system can be specified under the ``groups`` key as a string of comma-separated groups to create, or a list. Each item in the list should either contain a string of a single group to create, -or a dictionary with the group name as the key and a list of users who should -be members of the group as the value. +or a dictionary with the group name as the key and string of a single user as +a member of that group or a list of users who should be members of the group. .. note:: Groups are added before users, so any users in a group list must diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index a735ce3f265..a40588ba399 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -1974,8 +1974,8 @@ "patternProperties": { "^.+$": { "label": "", - "description": "Optional list of usernames to add to the group", - "type": "array", + "description": "Optional string of single username or a list of usernames to add to the group", + "type": ["string", "array"], "items": { "type": "string" }, diff --git a/tests/unittests/config/test_cc_users_groups.py b/tests/unittests/config/test_cc_users_groups.py index 3c72072927f..72784f297c6 100644 --- a/tests/unittests/config/test_cc_users_groups.py +++ b/tests/unittests/config/test_cc_users_groups.py @@ -282,9 +282,11 @@ class TestUsersGroupsSchema: # Validate default settings not covered by examples ({"groups": ["anygrp"]}, None), ({"groups": "anygrp,anyothergroup"}, None), + ({"groups": [{"anygrp": "user1"}]}, None), + ({"groups": [{"anygrp": ["user1", "user2"]}]}, None), ({"user": "olddefault"}, None), ({"users": ["default"]}, None), - ({"users": ["foobar"]}, None), # non default user creation + ({"users": ["foobar"]}, None), # no default user creation ({"users": [{"name": "bbsw"}]}, None), # minItems >= 1 for opaque-key ( From 172884e5a6b17800eaec494ea0770279ef851c2e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 14 Apr 2022 17:05:08 -0600 Subject: [PATCH 08/12] schema: groups allow object type. Will not validate this schema yet I don't want to add schema validation yet as is makes duplicated markdown confusing at the moment. --- cloudinit/config/cloud-init-schema.json | 2 +- tests/unittests/config/test_cc_users_groups.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index a40588ba399..9658c470276 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -1968,7 +1968,7 @@ "type": "object", "properties": { "groups": { - "type": ["string", "array"], + "type": ["string", "object", "array"], "items": { "type": ["string", "object"], "patternProperties": { diff --git a/tests/unittests/config/test_cc_users_groups.py b/tests/unittests/config/test_cc_users_groups.py index 72784f297c6..776d8fb0d95 100644 --- a/tests/unittests/config/test_cc_users_groups.py +++ b/tests/unittests/config/test_cc_users_groups.py @@ -282,7 +282,12 @@ class TestUsersGroupsSchema: # Validate default settings not covered by examples ({"groups": ["anygrp"]}, None), ({"groups": "anygrp,anyothergroup"}, None), + # Create anygrp with user1 as member ({"groups": [{"anygrp": "user1"}]}, None), + # Create anygrp with user1 as member using object/string syntax + ({"groups": {"anygrp": "user1"}}, None), + # Create anygrp with user1 as member using object/list syntax + ({"groups": {"anygrp": ["user1"]}}, None), ({"groups": [{"anygrp": ["user1", "user2"]}]}, None), ({"user": "olddefault"}, None), ({"users": ["default"]}, None), From 7e3cef0cb9f074dd78e1b9d795847abb645e88dd Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 14 Apr 2022 19:26:58 -0600 Subject: [PATCH 09/12] schema: groups as dict honor same schema as list of dicts --- cloudinit/config/cloud-init-schema.json | 26 +++++++++++-------- .../unittests/config/test_cc_users_groups.py | 4 +++ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index 9658c470276..a235b5827d5 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -1,6 +1,19 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "$defs": { + "users_groups.groups_by_groupname": { + "patternProperties": { + "^.+$": { + "label": "", + "description": "Optional string of single username or a list of usernames to add to the group", + "type": ["string", "array"], + "items": { + "type": "string" + }, + "minItems": 1 + } + } + }, "users_groups.user": { "type": "object", "oneOf": [ @@ -1969,19 +1982,10 @@ "properties": { "groups": { "type": ["string", "object", "array"], + "$ref": "#/$defs/users_groups.groups_by_groupname", "items": { "type": ["string", "object"], - "patternProperties": { - "^.+$": { - "label": "", - "description": "Optional string of single username or a list of usernames to add to the group", - "type": ["string", "array"], - "items": { - "type": "string" - }, - "minItems": 1 - } - } + "$ref": "#/$defs/users_groups.groups_by_groupname" }, "minItems": 1 }, diff --git a/tests/unittests/config/test_cc_users_groups.py b/tests/unittests/config/test_cc_users_groups.py index 776d8fb0d95..2d28f7c473e 100644 --- a/tests/unittests/config/test_cc_users_groups.py +++ b/tests/unittests/config/test_cc_users_groups.py @@ -303,6 +303,10 @@ class TestUsersGroupsSchema: {"user": ["no_list_allowed"]}, re.escape("user: ['no_list_allowed'] is not valid "), ), + ( + {"groups": {"anygrp": 1}}, + "groups.anygrp: 1 is not of type 'string', 'array'", + ), ], ) @skipUnlessJsonSchema() From 84951b4f8b8b01e604a43020de7d054fd654d804 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 14 Apr 2022 20:07:52 -0600 Subject: [PATCH 10/12] schema: add hidden property to hide docs When hidden is True, hide all docs for that object. When hidden is a list hide only docs for matching top-level keys. Fixes: SC-897 --- cloudinit/config/cloud-init-schema.json | 1 + cloudinit/config/schema.py | 14 +++++-- tests/unittests/config/test_schema.py | 49 +++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index a235b5827d5..16ccd5a141c 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -1982,6 +1982,7 @@ "properties": { "groups": { "type": ["string", "object", "array"], + "hidden": ["patternProperties"], "$ref": "#/$defs/users_groups.groups_by_groupname", "items": { "type": ["string", "object"], diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index bfdddf14041..774532c8e85 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -537,14 +537,20 @@ def _get_property_doc(schema: dict, defs: dict, prefix=" ") -> str: """Return restructured text describing the supported schema properties.""" new_prefix = prefix + " " properties = [] + if schema.get("hidden") is True: + return "" # no docs for this schema property_keys = [ - schema.get("properties", {}), - schema.get("patternProperties", {}), + key + for key in ("properties", "patternProperties") + if "hidden" not in schema or key not in schema["hidden"] ] + property_schemas = [schema.get(key, {}) for key in property_keys] - for props in property_keys: - for prop_key, prop_config in props.items(): + for prop_schema in property_schemas: + for prop_key, prop_config in prop_schema.items(): _flatten_schema_refs(prop_config, defs) + if prop_config.get("hidden") is True: + continue # document nothing for this property # Define prop_name and description for SCHEMA_PROPERTY_TMPL description = prop_config.get("description", "") if description: diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index a6bb5cff021..19ef0f484be 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -604,6 +604,55 @@ def test_get_meta_doc_handles_enum_types(self, sub_schema, expected): schema = {"properties": {"prop1": sub_schema}} assert expected in get_meta_doc(self.meta, schema) + @pytest.mark.parametrize( + "schema,expected", + ( + ( # Hide top-level keys like 'properties' + { + "hidden": ["properties"], + "properties": { + "p1": {"type": "string"}, + "p2": {"type": "boolean"}, + }, + "patternProperties": { + "^.*$": { + "type": "string", + "label": "label2", + } + }, + }, + dedent( + """ + **Config schema**: + **label2:** (string) + """ + ), + ), + ( # Hide nested individual keys with a bool + { + "properties": { + "p1": {"type": "string", "hidden": True}, + "p2": {"type": "boolean"}, + } + }, + dedent( + """ + **Config schema**: + **p2:** (boolean) + """ + ), + ), + ), + ) + def test_get_meta_doc_hidden_hides_specific_properties_from_docs( + self, schema, expected + ): + """Docs are hidden for any property in the hidden list. + + Useful for hiding deprecated key schema. + """ + assert expected in get_meta_doc(self.meta, schema) + def test_get_meta_doc_handles_nested_oneof_property_types(self): """get_meta_doc describes array items oneOf declarations in type.""" schema = { From 64088454442d4e4db6bb21fd399c215f466956eb Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 15 Apr 2022 10:13:02 -0600 Subject: [PATCH 11/12] schema: users can be a string update docs/tests --- cloudinit/config/cc_users_groups.py | 45 +++++++++++++++---- cloudinit/config/cloud-init-schema.json | 2 +- cloudinit/distros/ug_util.py | 18 +++++--- .../unittests/config/test_cc_users_groups.py | 5 ++- 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index 79a6d88130d..d3c945e6ca2 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -32,16 +32,20 @@ Groups are added before users, so any users in a group list must already exist on the system. -Users to add can be specified as a list under the ``users`` key. Each entry in -the list should either be a string or a dictionary. When a string +Users to add can be specified as a string or list under the ``users`` key. +Each entry in the list should either be a string or a dictionary. If a string is specified, that string can be comma-separated usernames to create or the -reserved value ``default``. The string ``default`` will setup the -``system_info:default_user`` as configured in ``/etc/cloud/cloud.cfg``. -Omission of ``default`` as the first item in the ``users`` list skips creation -the default_user. +reserved string ``default`` which represents the primary admin user used to +access the system. The ``default`` user varies per distribution and is +generally configured in ``/etc/cloud/cloud.cfg`` by the ``default_user`` key. -Each User dictionary must contain either a ``name`` or ``snapuser`` key, -otherwise it will be ignored. +Each ``users`` dictionary item must contain either a ``name`` or ``snapuser`` +key, otherwise it will be ignored. Omission of ``default`` as the first item +in the ``users`` list skips creation the default user. If no ``users`` key is +provided the default behavior is to create the default user via this config:: + + users: + - default .. note:: Specifying a hash of a user's password with ``passwd`` is a security risk @@ -56,6 +60,10 @@ already exists. The following options are the exceptions; they are applied to already-existing users: ``plain_text_passwd``, ``hashed_passwd``, ``lock_passwd``, ``sudo``, ``ssh_authorized_keys``, ``ssh_redirect_user``. + +The ``user`` key can be used to override the ``default_user`` configuration +defined in ``/etc/cloud/cloud.cfg``. The ``user`` value should be a dictionary +which supports the same config keys as the ``users`` dictionary items. """ meta: MetaSchema = { @@ -65,6 +73,15 @@ "description": MODULE_DESCRIPTION, "distros": ["all"], "examples": [ + dedent( + """\ + # Add the ``default_user`` from /etc/cloud/cloud.cfg. + # This is also the default behavior of cloud-init when no `users` key + # is provided. + users: + - default + """ + ), dedent( """\ # Add the 'admingroup' with members 'root' and 'sys' and an empty @@ -115,6 +132,18 @@ ssh_redirect_user: true """ ), + dedent( + """\ + # Override any ``default_user`` config in /etc/cloud/cloud.cfg with + # supplemental config options. + # This config will make the default user to mynewdefault and change + # the user to not have sudo rights. + ssh_import_id: [chad.smith] + user: + name: mynewdefault + sudo: false + """ + ), ], "frequency": PER_INSTANCE, } diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index 16ccd5a141c..2d357e96ea9 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -1998,7 +1998,7 @@ "description": "DEPRECATED, will be removed in a future release. Use ``users`` instead. This option creates a single default user. It overrides the ``default_user`` settings from /etc/cloud/cloud.cfg for a distribution. It accepts either a string of the default username to create or an object with any keys supported by the ``users`` schema." }, "users": { - "type": "array", + "type": ["string", "array"], "items": { "oneOf": [ {"type": "string"}, diff --git a/cloudinit/distros/ug_util.py b/cloudinit/distros/ug_util.py index dd81e8c627b..e0a4d0682c9 100755 --- a/cloudinit/distros/ug_util.py +++ b/cloudinit/distros/ug_util.py @@ -170,14 +170,14 @@ def normalize_users_groups(cfg, distro): # format. old_user = {} if "user" in cfg and cfg["user"]: - LOG.warning( - "DEPRECATED: 'user' key is deprecated and will be removed in a" - " future release. Use 'users' instead." - ) old_user = cfg["user"] # Translate it into a format that will be more useful going forward if isinstance(old_user, str): old_user = {"name": old_user} + LOG.warning( + "DEPRECATED: 'user' of type string is deprecated and will" + " be removed in a future release. Use 'users' list instead." + ) elif not isinstance(old_user, dict): LOG.warning( "Format for 'user' key must be a string or dictionary" @@ -205,9 +205,15 @@ def normalize_users_groups(cfg, distro): default_user_config = util.mergemanydict([old_user, distro_user_config]) base_users = cfg.get("users", []) - if not isinstance(base_users, (list, dict, str)): + if isinstance(base_users, (dict, str)): + LOG.warning( + "DEPRECATED: 'users' of type %s is deprecated and will be removed" + " in a future release. Use 'users' as a list.", + type(base_users), + ) + elif not isinstance(base_users, (list)): LOG.warning( - "Format for 'users' key must be a comma separated string" + "Format for 'users' key must be a comma-separated string" " or a dictionary or a list but found %s", type_utils.obj_name(base_users), ) diff --git a/tests/unittests/config/test_cc_users_groups.py b/tests/unittests/config/test_cc_users_groups.py index 2d28f7c473e..db8615b8d27 100644 --- a/tests/unittests/config/test_cc_users_groups.py +++ b/tests/unittests/config/test_cc_users_groups.py @@ -281,7 +281,7 @@ class TestUsersGroupsSchema: [ # Validate default settings not covered by examples ({"groups": ["anygrp"]}, None), - ({"groups": "anygrp,anyothergroup"}, None), + ({"groups": "anygrp,anyothergroup"}, None), # DEPRECATED # Create anygrp with user1 as member ({"groups": [{"anygrp": "user1"}]}, None), # Create anygrp with user1 as member using object/string syntax @@ -289,7 +289,10 @@ class TestUsersGroupsSchema: # Create anygrp with user1 as member using object/list syntax ({"groups": {"anygrp": ["user1"]}}, None), ({"groups": [{"anygrp": ["user1", "user2"]}]}, None), + # Make default username "olddefault": DEPRECATED ({"user": "olddefault"}, None), + # Create multiple users, and include default user. DEPRECATED + ({"users": "oldstyle,default"}, None), ({"users": ["default"]}, None), ({"users": ["foobar"]}, None), # no default user creation ({"users": [{"name": "bbsw"}]}, None), From 3aa046923bfbda3da07d9d93b496e9505842dae8 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 15 Apr 2022 11:08:58 -0600 Subject: [PATCH 12/12] schema: users can also be an object and list of lists --- cloudinit/config/cloud-init-schema.json | 10 +++++----- tests/unittests/config/test_cc_users_groups.py | 6 +----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/cloudinit/config/cloud-init-schema.json b/cloudinit/config/cloud-init-schema.json index 2d357e96ea9..7eb05919c82 100644 --- a/cloudinit/config/cloud-init-schema.json +++ b/cloudinit/config/cloud-init-schema.json @@ -15,7 +15,6 @@ } }, "users_groups.user": { - "type": "object", "oneOf": [ {"required": ["name"]}, {"required": ["snapuser"]} @@ -1993,16 +1992,17 @@ "user": { "oneOf": [ {"type": "string"}, - {"$ref": "#/$defs/users_groups.user"} + {"type": "object", "$ref": "#/$defs/users_groups.user"} ], - "description": "DEPRECATED, will be removed in a future release. Use ``users`` instead. This option creates a single default user. It overrides the ``default_user`` settings from /etc/cloud/cloud.cfg for a distribution. It accepts either a string of the default username to create or an object with any keys supported by the ``users`` schema." + "description": "The ``user`` dictionary values override the ``default_user`` configuration from ``/etc/cloud/cloud.cfg``. The `user` dictionary keys supported for the default_user are the same as the ``users`` schema. DEPRECATED: string and types will be removed in a future release. Use ``users`` instead." }, "users": { - "type": ["string", "array"], + "type": ["string", "array", "object"], "items": { "oneOf": [ {"type": "string"}, - {"$ref": "#/$defs/users_groups.user"} + {"type": "array", "items": {"type": "string"}}, + {"type": "object", "$ref": "#/$defs/users_groups.user"} ] }, "minItems": 1 diff --git a/tests/unittests/config/test_cc_users_groups.py b/tests/unittests/config/test_cc_users_groups.py index db8615b8d27..9210c1f6bd6 100644 --- a/tests/unittests/config/test_cc_users_groups.py +++ b/tests/unittests/config/test_cc_users_groups.py @@ -294,13 +294,9 @@ class TestUsersGroupsSchema: # Create multiple users, and include default user. DEPRECATED ({"users": "oldstyle,default"}, None), ({"users": ["default"]}, None), + ({"users": ["default", ["aaa", "bbb"]]}, None), ({"users": ["foobar"]}, None), # no default user creation ({"users": [{"name": "bbsw"}]}, None), - # minItems >= 1 for opaque-key - ( - {"groups": [{"needitems": []}]}, - re.escape("groups.0.needitems: [] is too short"), - ), ({"groups": [{"yep": ["user1"]}]}, None), ( {"user": ["no_list_allowed"]},