From 99faf212266ff0de5193818b2fac04268ee44590 Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Wed, 6 May 2020 18:57:05 -0300 Subject: [PATCH 01/13] Add initial version of apt_configure schema --- cloudinit/config/cc_apt_configure.py | 369 ++++++++++++++++++ .../examples/install_run_chef_recipes.yaml | 73 ++-- tests/unittests/test_handler/test_schema.py | 1 + 3 files changed, 407 insertions(+), 36 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ff58c0623c1..cdafaa99b42 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -218,14 +218,383 @@ import glob import os import re +from textwrap import dedent from cloudinit import gpg from cloudinit import log as logging from cloudinit import templater from cloudinit import util +from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) +frequency = PER_INSTANCE +distros = ["ubuntu", "debian"] +schema = { + 'id': 'cc_apt_configure', + 'name': 'Apt Configure', + 'title': 'Configure apt for the user', # IMPROVE + 'description': dedent("""\ + This module handles both configuration of apt options and adding + source lists. There are configuration options such as + ``apt_get_wrapper`` and ``apt_get_command`` that control how + cloud-init invokes apt-get. These configuration options are + handled on a per-distro basis, so consult documentation for + cloud-init's distro support for instructions on using + these config options. + + .. note:: + To ensure that apt configuration is valid yaml, any strings + containing special characters, especially ``:`` should be quoted. + + .. note:: + For more information about apt configuration, see the + ``Additional apt configuration`` example."""), + 'distros': distros, + 'examples': [dedent("""\ + apt: + preserve_sources_list: false + disable_suites: + - $RELEASE-updates + - backports + - $RELEASE + - mysuite + primary: + - arches: + - amd64 + - i386 + - default + uri: 'http://us.archive.ubuntu.com/ubuntu' + search: + - 'http://cool.but-sometimes-unreachable.com/ubuntu' + - 'http://us.archive.ubuntu.com/ubuntu' + search_dns: + - arches: + - s390x + - arm64 + uri: 'http://archive-to-use-for-arm64.example.com/ubuntu' + security: + - arches: + - default + search_dns: true + sources_list: | + deb $MIRROR $RELEASE main restricted + deb-src $MIRROR $RELEASE main restricted + deb $PRIMARY $RELEASE universe restricted + deb $SECURITY $RELEASE-security multiverse + debconf_selections: + set1: the-package the-package/some-flag boolean true + conf: | + APT { + Get { + Assume-Yes 'true'; + Fix-Broken 'true'; + } + } + proxy: 'http://[[user][:pass]@]host[:port]/' + http_proxy: 'http://[[user][:pass]@]host[:port]/' + ftp_proxy: 'ftp://[[user][:pass]@]host[:port]/' + https_proxy: 'https://[[user][:pass]@]host[:port]/' + sources: + source1: + keyid: 'keyid' + keyserver: 'keyserverurl' + source: 'deb http:/// xenial main' + source2: + source: 'ppa:' + source3: + source: 'deb $MIRROR $RELEASE multiverse' + key: | + ------BEGIN PGP PUBLIC KEY BLOCK------- + + ------END PGP PUBLIC KEY BLOCK-------""")], + 'frequency': frequency, + 'type': 'object', + 'properties': { + 'apt': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'preserve_sources_list': { + 'type': 'boolean', + 'description': dedent("""\ + By default, cloud-init will generate a new + sources list in ``/etc/apt/sources.list.d`` + based on any changes specified in cloud config. + To disable this behavior and preserve the sources + list from the pristine image, set + ``preserve_sources_list`` to ``true``. + + .. note:: + The ``preserve_sources_list`` option overrides + all other config keys that would alter + ``sources.list`` or ``sources.list.d``, + **except** for additional sources to be added + to ``sources.list.d``.""") + }, + 'disable_suites': { + 'type': 'array', + 'items': { + 'type': 'string' + }, + 'uniqueItems': True, + 'description': dedent("""\ + Entries in the sources list can be disabled using + ``disable_suites``, which takes a list of suites + to be disabled. If the string ``$RELEASE`` is + present in a suite in the ``disable_suites`` list, + it will be replaced with the release name. If a + suite specified in ``disable_suites`` is not + present in ``sources.list`` it will be ignored. + For convenience, several aliases are provided for + ``disable_suites``: + + - ``updates`` => ``$RELEASE-updates`` + - ``backports`` => ``$RELEASE-backports`` + - ``security`` => ``$RELEASE-security`` + - ``proposed`` => ``$RELEASE-proposed`` + - ``release`` => ``$RELEASE`` + + .. note:: + When a suite is disabled using + ``disable_suites``, its entry in + ``sources.list`` is not deleted; it is just + commented out.""") + }, + 'primary': { + 'type': 'array', + 'item': { + 'type': 'object', + 'additionalProperties': False, + 'required': ['arches'], + 'properties': { + 'arches': { + 'type': 'array', + 'item': { + 'type': 'string' + }, + 'minItems': 1 + }, + 'uri': { + 'type': 'string', + 'format': 'uri' + }, + 'search': { + 'type': 'array', + 'item': { + 'type': 'string', + 'format': 'uri' + }, + 'minItems': 1 + }, + 'search_dns': { + 'type': 'boolean', + } + } + }, + 'description': dedent("""\ + The primary and security archive mirrors can be + specified using the ``primary`` and ``security` + keys, respectively Both the ``primary`` and + ``security`` keys take a list of configs, + allowing mirrors to be specified on a + per-architecture basis. Each config is a + dictionary which must have an entry for + ``arches``, specifying which architectures + that config entry is for. The keyword + ``default`` applies to any architecture not + explicitly listed. The mirror url can be specified + with the ``uri`` key, or a list of mirrors to + check can be provided in order, with the first + mirror that can be resolved being selected. This + allows the same configuration to be used in + different environment, with different hosts used + for a local apt mirror. If no mirror is provided + by ``uri`` or ``search``, ``search_dns`` may be + used to search for dns names in the format + ``-mirror`` in each of the following: + + - fqdn of this host per cloud metadata + - localdomain + - domains listed in ``/etc/resolv.conf`` + + If there is a dns entry for ``-mirror``, + then it is assumed that there is a distro mirror + at ``http://-mirror./``. + If the ``primary`` key is defined, but not the + ``security`` key, then then configuration for + ``primary`` is also used for ``security``. + If ``search_dns`` is used for the ``security`` + key, the search pattern will be + ``-security-mirror``. + + If no mirrors are specified, or all lookups fail, + then default mirrors defined in the datasource + are used. If none are present in the datasource + either the following defaults are used: + + - primary: ``http://archive.ubuntu.com/ubuntu`` + - security: ``http://security.ubuntu.com/ubuntu`` + """) + }, + 'security': { + 'type': 'array', + 'item': { + 'type': 'object', + 'additionalProperties': False, + 'required': ['arches'], + 'properties': { + 'arches': { + 'type': 'array', + 'uniqueItems': True, + 'item': { + 'type': 'string' + }, + 'minItems': 1 + }, + 'uri': { + 'type': 'string', + 'format': 'uri' + }, + 'search': { + 'type': 'array', + 'uniqueItems': True, + 'item': { + 'type': 'string', + 'format': 'uri' + }, + 'minItems': 1 + }, + 'search_dns': { + 'type': 'boolean', + } + } + } + }, + 'add_apt_repo_match': { + 'type': 'string', + 'description': dedent("""\ + All source entries in ``apt-sources`` that match + regex in ``add_apt_repo_match`` will be added to + the system using ``add-apt-repository``. If + ``add_apt_repo_match`` is not specified, it + defaults to ``^[\\w-]+:\\w``""") + }, + 'debconf_selections': { + 'type': 'object', + 'items': {'type': 'string'}, + 'description': dedent("""\ + Debconf additioanl configurations can be specified as a + dictionary under the ``debconf_selections`` config + key, with each key in the dict representing a + different set of configurations. The value of each key + must be a string containing all the debconf + configurations that must be applied. We will bundle + all of the values and pass them to + ``debconf-set-selections``. Therefore, each value line + must be a valid entry for ``debconf-set-selections``, + meaning that they must possess for distinct fields: + + pkgname question type answer + + Where: + + - pkgname is the name of the package + - question the name of the questions + - type is the type of question + - answer is the value used to ansert the question + + For example: + + - ippackage ippackage/ip string 127.0.01 + """) + }, + 'sources_list': { + 'type': 'string', + 'description': dedent("""\ + Specifies a custom template for rendering + ``sources.list`` . If no ``sources_list`` template + is given, cloud-init will use sane default. Within + this template, the following strings will be + replaced with the appropriate values: + + - ``$MIRROR`` + - ``$RELEASE`` + - ``$PRIMARY`` + - ``$SECURITY``""") + }, + 'conf': { + 'type': 'string', + 'description': dedent("""\ + Specify proxy configuration for apt""") + }, + 'https_proxy': { + 'type': 'string', + 'description': dedent("""\ + More convinient way to specify https apt proxy. + https proxy url is specified in the format + ``https://[[user][:pass]@]host[:port]/``.""") + }, + 'http_proxy': { + 'type': 'string', + 'description': dedent("""\ + More convinient way to specify http apt proxy. + http proxy url is specified in the format + ``http://[[user][:pass]@]host[:port]/``.""") + }, + 'proxy': { + 'type': 'string', + 'description': dedent("""\ + Alias for defining a http apt proxy.""") + }, + 'ftp_proxy': { + 'type': 'string', + 'description': dedent("""\ + More convinient way to specify ftp apt proxy. + ftp proxy url is specified in the format + ``ftp://[[user][:pass]@]host[:port]/``.""") + }, + 'sources': { + 'type': 'object', + 'items': {'type': 'string'}, + 'description': dedent("""\ + Source list entries can be specified as a + dictionary under the ``sources`` config key, with + key in the dict representing a different source + file. The key of each source entry will be used + as an id that can be referenced in other config + entries, as well as the filename for the source's + configuration under ``/etc/apt/sources.list.d``. + If the name does not end with ``.list``, it will + be appended. If there is no configuration for a + key in ``sources``, no file will be written, but + the key may still be referred to as an id in other + ``sources`` entries. + + Each entry under ``sources`` is a dictionary which + may contain any of the following optional keys: + + - ``source``: a sources.list entry \ + (some variable replacements apply) + - ``keyid``: a key to import via shortid or \ + fingerprint + - ``key``: a raw PGP key + - ``keyserver``: alternate keyserver to pull \ + ``keyid`` key from + + The ``source`` key supports variable + replacements for the following strings: + + - ``$MIRROR`` + - ``$PRIMARY`` + - ``$SECURITY`` + - ``$RELEASE``""") + } + } + } + } +} + # this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') ADD_APT_REPO_MATCH = r"^[\w-]+:\w" diff --git a/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml b/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml index 0bec305e589..68ca95b59f3 100644 --- a/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml +++ b/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml @@ -8,43 +8,44 @@ cloud_config: | #cloud-config # Key from https://packages.chef.io/chef.asc apt: - source1: - source: "deb http://packages.chef.io/repos/apt/stable $RELEASE main" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v1.4.12 (Darwin) - Comment: GPGTools - http://gpgtools.org + sources: + source1: + source: "deb http://packages.chef.io/repos/apt/stable $RELEASE main" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1.4.12 (Darwin) + Comment: GPGTools - http://gpgtools.org - mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu - twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 - dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC - JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W - ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I - XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe - DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm - sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO - Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ - YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG - CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K - +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu0IENIRUYgUGFja2FnZXMg - PHBhY2thZ2VzQGNoZWYuaW8+iGIEExECACIFAlQwYFECGwMGCwkIBwMCBhUIAgkK - CwQWAgMBAh4BAheAAAoJEClAq6mD74JqX94An26z99XOHWpLN8ahzm7cp13t4Xid - AJ9wVcgoUBzvgg91lKfv/34cmemZn7kCDQRKaQu0EAgAg7ZLCVGVTmLqBM6njZEd - Zbv+mZbvwLBSomdiqddE6u3eH0X3GuwaQfQWHUVG2yedyDMiG+EMtCdEeeRebTCz - SNXQ8Xvi22hRPoEsBSwWLZI8/XNg0n0f1+GEr+mOKO0BxDB2DG7DA0nnEISxwFkK - OFJFebR3fRsrWjj0KjDxkhse2ddU/jVz1BY7Nf8toZmwpBmdozETMOTx3LJy1HZ/ - Te9FJXJMUaB2lRyluv15MVWCKQJro4MQG/7QGcIfrIZNfAGJ32DDSjV7/YO+IpRY - IL4CUBQ65suY4gYUG4jhRH6u7H1p99sdwsg5OIpBe/v2Vbc/tbwAB+eJJAp89Zeu - twADBQf/ZcGoPhTGFuzbkcNRSIz+boaeWPoSxK2DyfScyCAuG41CY9+g0HIw9Sq8 - DuxQvJ+vrEJjNvNE3EAEdKl/zkXMZDb1EXjGwDi845TxEMhhD1dDw2qpHqnJ2mtE - WpZ7juGwA3sGhi6FapO04tIGacCfNNHmlRGipyq5ZiKIRq9mLEndlECr8cwaKgkS - 0wWu+xmMZe7N5/t/TK19HXNh4tVacv0F3fYK54GUjt2FjCQV75USnmNY4KPTYLXA - dzC364hEMlXpN21siIFgB04w+TXn5UF3B4FfAy5hevvr4DtV4MvMiGLu0oWjpaLC - MpmrR3Ny2wkmO0h+vgri9uIP06ODWIhJBBgRAgAJBQJKaQu0AhsMAAoJEClAq6mD - 74Jq4hIAoJ5KrYS8kCwj26SAGzglwggpvt3CAJ0bekyky56vNqoegB+y4PQVDv4K - zA== - =IxPr - -----END PGP PUBLIC KEY BLOCK----- + mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu + twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 + dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC + JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W + ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I + XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe + DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm + sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO + Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ + YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG + CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K + +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu0IENIRUYgUGFja2FnZXMg + PHBhY2thZ2VzQGNoZWYuaW8+iGIEExECACIFAlQwYFECGwMGCwkIBwMCBhUIAgkK + CwQWAgMBAh4BAheAAAoJEClAq6mD74JqX94An26z99XOHWpLN8ahzm7cp13t4Xid + AJ9wVcgoUBzvgg91lKfv/34cmemZn7kCDQRKaQu0EAgAg7ZLCVGVTmLqBM6njZEd + Zbv+mZbvwLBSomdiqddE6u3eH0X3GuwaQfQWHUVG2yedyDMiG+EMtCdEeeRebTCz + SNXQ8Xvi22hRPoEsBSwWLZI8/XNg0n0f1+GEr+mOKO0BxDB2DG7DA0nnEISxwFkK + OFJFebR3fRsrWjj0KjDxkhse2ddU/jVz1BY7Nf8toZmwpBmdozETMOTx3LJy1HZ/ + Te9FJXJMUaB2lRyluv15MVWCKQJro4MQG/7QGcIfrIZNfAGJ32DDSjV7/YO+IpRY + IL4CUBQ65suY4gYUG4jhRH6u7H1p99sdwsg5OIpBe/v2Vbc/tbwAB+eJJAp89Zeu + twADBQf/ZcGoPhTGFuzbkcNRSIz+boaeWPoSxK2DyfScyCAuG41CY9+g0HIw9Sq8 + DuxQvJ+vrEJjNvNE3EAEdKl/zkXMZDb1EXjGwDi845TxEMhhD1dDw2qpHqnJ2mtE + WpZ7juGwA3sGhi6FapO04tIGacCfNNHmlRGipyq5ZiKIRq9mLEndlECr8cwaKgkS + 0wWu+xmMZe7N5/t/TK19HXNh4tVacv0F3fYK54GUjt2FjCQV75USnmNY4KPTYLXA + dzC364hEMlXpN21siIFgB04w+TXn5UF3B4FfAy5hevvr4DtV4MvMiGLu0oWjpaLC + MpmrR3Ny2wkmO0h+vgri9uIP06ODWIhJBBgRAgAJBQJKaQu0AhsMAAoJEClAq6mD + 74Jq4hIAoJ5KrYS8kCwj26SAGzglwggpvt3CAJ0bekyky56vNqoegB+y4PQVDv4K + zA== + =IxPr + -----END PGP PUBLIC KEY BLOCK----- chef: diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 071b98bc8bc..bd25e4e62c9 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -24,6 +24,7 @@ def test_get_schema_coalesces_known_schema(self): schema = get_schema() self.assertCountEqual( [ + 'cc_apt_configure', 'cc_bootcmd', 'cc_locale', 'cc_ntp', From a20121349d4f15089d5506556ce693b1b09f4461 Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Thu, 7 May 2020 10:17:51 -0300 Subject: [PATCH 02/13] Better format schema for documentation --- cloudinit/config/cc_apt_configure.py | 316 +++++---------------------- 1 file changed, 56 insertions(+), 260 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index cdafaa99b42..50126018c35 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -6,220 +6,14 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Apt Configure -------------- -**Summary:** configure apt - -This module handles both configuration of apt options and adding source lists. -There are configuration options such as ``apt_get_wrapper`` and -``apt_get_command`` that control how cloud-init invokes apt-get. -These configuration options are handled on a per-distro basis, so consult -documentation for cloud-init's distro support for instructions on using -these config options. - -.. note:: - To ensure that apt configuration is valid yaml, any strings containing - special characters, especially ``:`` should be quoted. - -.. note:: - For more information about apt configuration, see the - ``Additional apt configuration`` example. - -**Preserve sources.list:** - -By default, cloud-init will generate a new sources list in -``/etc/apt/sources.list.d`` based on any changes specified in cloud config. -To disable this behavior and preserve the sources list from the pristine image, -set ``preserve_sources_list`` to ``true``. - -.. note:: - The ``preserve_sources_list`` option overrides all other config keys that - would alter ``sources.list`` or ``sources.list.d``, **except** for - additional sources to be added to ``sources.list.d``. - -**Disable source suites:** - -Entries in the sources list can be disabled using ``disable_suites``, which -takes a list of suites to be disabled. If the string ``$RELEASE`` is present in -a suite in the ``disable_suites`` list, it will be replaced with the release -name. If a suite specified in ``disable_suites`` is not present in -``sources.list`` it will be ignored. For convenience, several aliases are -provided for ``disable_suites``: - - - ``updates`` => ``$RELEASE-updates`` - - ``backports`` => ``$RELEASE-backports`` - - ``security`` => ``$RELEASE-security`` - - ``proposed`` => ``$RELEASE-proposed`` - - ``release`` => ``$RELEASE`` - -.. note:: - When a suite is disabled using ``disable_suites``, its entry in - ``sources.list`` is not deleted; it is just commented out. - -**Configure primary and security mirrors:** - -The primary and security archive mirrors can be specified using the ``primary`` -and ``security`` keys, respectively. Both the ``primary`` and ``security`` keys -take a list of configs, allowing mirrors to be specified on a per-architecture -basis. Each config is a dictionary which must have an entry for ``arches``, -specifying which architectures that config entry is for. The keyword -``default`` applies to any architecture not explicitly listed. The mirror url -can be specified with the ``uri`` key, or a list of mirrors to check can be -provided in order, with the first mirror that can be resolved being selected. -This allows the same configuration to be used in different environment, with -different hosts used for a local apt mirror. If no mirror is provided by -``uri`` or ``search``, ``search_dns`` may be used to search for dns names in -the format ``-mirror`` in each of the following: - - - fqdn of this host per cloud metadata - - localdomain - - domains listed in ``/etc/resolv.conf`` - -If there is a dns entry for ``-mirror``, then it is assumed that there -is a distro mirror at ``http://-mirror./``. If the -``primary`` key is defined, but not the ``security`` key, then then -configuration for ``primary`` is also used for ``security``. If ``search_dns`` -is used for the ``security`` key, the search pattern will be. -``-security-mirror``. - -If no mirrors are specified, or all lookups fail, then default mirrors defined -in the datasource are used. If none are present in the datasource either the -following defaults are used: - - - primary: ``http://archive.ubuntu.com/ubuntu`` - - security: ``http://security.ubuntu.com/ubuntu`` - -**Specify sources.list template:** - -A custom template for rendering ``sources.list`` can be specefied with -``sources_list``. If no ``sources_list`` template is given, cloud-init will -use sane default. Within this template, the following strings will be replaced -with the appropriate values: - - - ``$MIRROR`` - - ``$RELEASE`` - - ``$PRIMARY`` - - ``$SECURITY`` - -**Pass configuration to apt:** - -Apt configuration can be specified using ``conf``. Configuration is specified -as a string. For multiline apt configuration, make sure to follow yaml syntax. - -**Configure apt proxy:** - -Proxy configuration for apt can be specified using ``conf``, but proxy config -keys also exist for convenience. The proxy config keys, ``http_proxy``, -``ftp_proxy``, and ``https_proxy`` may be used to specify a proxy for http, ftp -and https protocols respectively. The ``proxy`` key also exists as an alias for -``http_proxy``. Proxy url is specified in the format -``://[[user][:pass]@]host[:port]/``. - -**Add apt repos by regex:** - -All source entries in ``apt-sources`` that match regex in -``add_apt_repo_match`` will be added to the system using -``add-apt-repository``. If ``add_apt_repo_match`` is not specified, it defaults -to ``^[\\w-]+:\\w`` - -**Add source list entries:** - -Source list entries can be specified as a dictionary under the ``sources`` -config key, with key in the dict representing a different source file. The key -of each source entry will be used as an id that can be referenced in -other config entries, as well as the filename for the source's configuration -under ``/etc/apt/sources.list.d``. If the name does not end with ``.list``, -it will be appended. If there is no configuration for a key in ``sources``, no -file will be written, but the key may still be referred to as an id in other -``sources`` entries. - -Each entry under ``sources`` is a dictionary which may contain any of the -following optional keys: - - - ``source``: a sources.list entry (some variable replacements apply) - - ``keyid``: a key to import via shortid or fingerprint - - ``key``: a raw PGP key - - ``keyserver``: alternate keyserver to pull ``keyid`` key from - -The ``source`` key supports variable replacements for the following strings: - - - ``$MIRROR`` - - ``$PRIMARY`` - - ``$SECURITY`` - - ``$RELEASE`` - -**Internal name:** ``cc_apt_configure`` - -**Module frequency:** per instance - -**Supported distros:** ubuntu, debian - -**Config keys**:: - - apt: - preserve_sources_list: - disable_suites: - - $RELEASE-updates - - backports - - $RELEASE - - mysuite - primary: - - arches: - - amd64 - - i386 - - default - uri: "http://us.archive.ubuntu.com/ubuntu" - search: - - "http://cool.but-sometimes-unreachable.com/ubuntu" - - "http://us.archive.ubuntu.com/ubuntu" - search_dns: - - arches: - - s390x - - arm64 - uri: "http://archive-to-use-for-arm64.example.com/ubuntu" - security: - - arches: - - default - search_dns: true - sources_list: | - deb $MIRROR $RELEASE main restricted - deb-src $MIRROR $RELEASE main restricted - deb $PRIMARY $RELEASE universe restricted - deb $SECURITY $RELEASE-security multiverse - debconf_selections: - set1: the-package the-package/some-flag boolean true - conf: | - APT { - Get { - Assume-Yes "true"; - Fix-Broken "true"; - } - } - proxy: "http://[[user][:pass]@]host[:port]/" - http_proxy: "http://[[user][:pass]@]host[:port]/" - ftp_proxy: "ftp://[[user][:pass]@]host[:port]/" - https_proxy: "https://[[user][:pass]@]host[:port]/" - sources: - source1: - keyid: "keyid" - keyserver: "keyserverurl" - source: "deb http:/// xenial main" - source2: - source: "ppa:" - source3: - source: "deb $MIRROR $RELEASE multiverse" - key: | - ------BEGIN PGP PUBLIC KEY BLOCK------- - - ------END PGP PUBLIC KEY BLOCK------- -""" +"""Apt Configure: Configure apt for the user.""" import glob import os import re from textwrap import dedent +from cloudinit.config.schema import get_schema_doc from cloudinit import gpg from cloudinit import log as logging from cloudinit import templater @@ -318,19 +112,18 @@ 'preserve_sources_list': { 'type': 'boolean', 'description': dedent("""\ - By default, cloud-init will generate a new - sources list in ``/etc/apt/sources.list.d`` - based on any changes specified in cloud config. - To disable this behavior and preserve the sources - list from the pristine image, set - ``preserve_sources_list`` to ``true``. - - .. note:: - The ``preserve_sources_list`` option overrides - all other config keys that would alter - ``sources.list`` or ``sources.list.d``, - **except** for additional sources to be added - to ``sources.list.d``.""") + By default, cloud-init will generate a new sources + list in ``/etc/apt/sources.list.d`` based on any + changes specified in cloud config. To disable this + behavior and preserve the sources list from the + pristine image, set ``preserve_sources_list`` + to ``true``. + + The ``preserve_sources_list`` option overrides + all other config keys that would alter + ``sources.list`` or ``sources.list.d``, + **except** for additional sources to be added + to ``sources.list.d``.""") }, 'disable_suites': { 'type': 'array', @@ -349,17 +142,15 @@ For convenience, several aliases are provided for ``disable_suites``: - - ``updates`` => ``$RELEASE-updates`` - - ``backports`` => ``$RELEASE-backports`` - - ``security`` => ``$RELEASE-security`` - - ``proposed`` => ``$RELEASE-proposed`` - - ``release`` => ``$RELEASE`` - - .. note:: - When a suite is disabled using - ``disable_suites``, its entry in - ``sources.list`` is not deleted; it is just - commented out.""") + ``updates`` => ``$RELEASE-updates``, + ``backports`` => ``$RELEASE-backports``. + ``security`` => ``$RELEASE-security``, + ``proposed`` => ``$RELEASE-proposed``, + ``release`` => ``$RELEASE``. + + When a suite is disabled using ``disable_suites``, + its entry in ``sources.list`` is not deleted; it + is just commented out.""") }, 'primary': { 'type': 'array', @@ -414,9 +205,9 @@ used to search for dns names in the format ``-mirror`` in each of the following: - - fqdn of this host per cloud metadata - - localdomain - - domains listed in ``/etc/resolv.conf`` + fqdn of this host per cloud metadata, + localdomain, + domains listed in ``/etc/resolv.conf``. If there is a dns entry for ``-mirror``, then it is assumed that there is a distro mirror @@ -433,8 +224,9 @@ are used. If none are present in the datasource either the following defaults are used: - - primary: ``http://archive.ubuntu.com/ubuntu`` - - security: ``http://security.ubuntu.com/ubuntu`` + ``primary`` => ``http://archive.ubuntu.com/ubuntu``, + ``security`` => + ``http://security.ubuntu.com/ubuntu`` """) }, 'security': { @@ -499,14 +291,13 @@ Where: - - pkgname is the name of the package - - question the name of the questions - - type is the type of question - - answer is the value used to ansert the question - - For example: + ``pkgname`` is the name of the package, + ``question`` the name of the questions, + ``type`` is the type of question, + ``answer`` is the value used to ansert the + question. - - ippackage ippackage/ip string 127.0.01 + For example: ippackage ippackage/ip string 127.0.01 """) }, 'sources_list': { @@ -518,15 +309,18 @@ this template, the following strings will be replaced with the appropriate values: - - ``$MIRROR`` - - ``$RELEASE`` - - ``$PRIMARY`` - - ``$SECURITY``""") + ``$MIRROR``, + ``$RELEASE``, + ``$PRIMARY``, + ``$SECURITY``""") }, 'conf': { 'type': 'string', 'description': dedent("""\ - Specify proxy configuration for apt""") + Specify configuration for apt, such as proxy + configuratiun. This configuration is specified as a + string. For multiline apt configuration, make sure + to follow yaml syntax.""") }, 'https_proxy': { 'type': 'string', @@ -574,27 +368,29 @@ Each entry under ``sources`` is a dictionary which may contain any of the following optional keys: - - ``source``: a sources.list entry \ - (some variable replacements apply) - - ``keyid``: a key to import via shortid or \ - fingerprint - - ``key``: a raw PGP key - - ``keyserver``: alternate keyserver to pull \ - ``keyid`` key from + ``source``: a sources.list entry \ + (some variable replacements apply). + ``keyid``: a key to import via shortid or \ + fingerprin. + ``key``: a raw PGP key. + ``keyserver``: alternate keyserver to pull \ + ``keyid`` key from. The ``source`` key supports variable replacements for the following strings: - - ``$MIRROR`` - - ``$PRIMARY`` - - ``$SECURITY`` - - ``$RELEASE``""") + ``$MIRROR``, + ``$PRIMARY``, + ``$SECURITY``, + ``$RELEASE``""") } } } } } +__doc__ = get_schema_doc(schema) + # this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') ADD_APT_REPO_MATCH = r"^[\w-]+:\w" From e6e0a6cfb0ace37d71389eda4cabf73a11905dc4 Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Thu, 7 May 2020 10:22:40 -0300 Subject: [PATCH 03/13] Validate schema in apt configure handle --- cloudinit/config/cc_apt_configure.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 50126018c35..2eef26d6bb8 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -13,7 +13,8 @@ import re from textwrap import dedent -from cloudinit.config.schema import get_schema_doc +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit import gpg from cloudinit import log as logging from cloudinit import templater @@ -444,6 +445,7 @@ def handle(name, ocfg, cloud, log, _): "Expected dictionary for 'apt' config, found {config_type}".format( config_type=type(cfg))) + validate_cloudconfig_schema(cfg, schema) apply_debconf_selections(cfg, target) apply_apt(cfg, cloud, target) From b5986aa6e12af4c3b02b150f7bbbb76080776d88 Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Thu, 7 May 2020 10:25:32 -0300 Subject: [PATCH 04/13] Update debconf_selections in cloud-config example --- doc/examples/cloud-config.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 8c1e4bb0534..20a0ce0d280 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -203,13 +203,14 @@ ssh_import_id: [smoser] # # Default: none # -debconf_selections: | # Need to preserve newlines +debconf_selections: # Force debconf priority to critical. - debconf debconf/priority select critical + set1: debconf debconf/priority select critical # Override default frontend to readline, but allow user to select. - debconf debconf/frontend select readline - debconf debconf/frontend seen false + set2: | + debconf debconf/frontend select readline + debconf debconf/frontend seen false # manage byobu defaults # byobu_by_default: From c8d8dcca708ca60788b5c382f8ac74ee5f216945 Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Thu, 7 May 2020 10:27:52 -0300 Subject: [PATCH 05/13] Keep apt_configure title as it is --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 2eef26d6bb8..4399c09feb4 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -28,7 +28,7 @@ schema = { 'id': 'cc_apt_configure', 'name': 'Apt Configure', - 'title': 'Configure apt for the user', # IMPROVE + 'title': 'Configure apt for the user', 'description': dedent("""\ This module handles both configuration of apt options and adding source lists. There are configuration options such as From 915685ae1dce2e1a3b3cb3df80b2d79e001d06bd Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Thu, 7 May 2020 10:59:28 -0300 Subject: [PATCH 06/13] Better format schema --- cloudinit/config/cc_apt_configure.py | 175 +++++++++++---------------- 1 file changed, 74 insertions(+), 101 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 4399c09feb4..147a8516947 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -25,6 +25,38 @@ frequency = PER_INSTANCE distros = ["ubuntu", "debian"] +mirror_property = { + 'type': 'array', + 'item': { + 'type': 'object', + 'additionalProperties': False, + 'required': ['arches'], + 'properties': { + 'arches': { + 'type': 'array', + 'item': { + 'type': 'string' + }, + 'minItems': 1 + }, + 'uri': { + 'type': 'string', + 'format': 'uri' + }, + 'search': { + 'type': 'array', + 'item': { + 'type': 'string', + 'format': 'uri' + }, + 'minItems': 1 + }, + 'search_dns': { + 'type': 'boolean', + } + } + } +} schema = { 'id': 'cc_apt_configure', 'name': 'Apt Configure', @@ -154,115 +186,56 @@ is just commented out.""") }, 'primary': { - 'type': 'array', - 'item': { - 'type': 'object', - 'additionalProperties': False, - 'required': ['arches'], - 'properties': { - 'arches': { - 'type': 'array', - 'item': { - 'type': 'string' - }, - 'minItems': 1 - }, - 'uri': { - 'type': 'string', - 'format': 'uri' - }, - 'search': { - 'type': 'array', - 'item': { - 'type': 'string', - 'format': 'uri' - }, - 'minItems': 1 - }, - 'search_dns': { - 'type': 'boolean', - } - } - }, + **mirror_property, 'description': dedent("""\ - The primary and security archive mirrors can be - specified using the ``primary`` and ``security` - keys, respectively Both the ``primary`` and - ``security`` keys take a list of configs, - allowing mirrors to be specified on a - per-architecture basis. Each config is a - dictionary which must have an entry for + The primary and security archive mirrors can + be specified using the ``primary`` and + ``security`` keys, respectively Both the + ``primary`` and ``security`` keys take a list + of configs, allowing mirrors to be specified + on a per-architecture basis. Each config is a + dictionary which must have an entry for ``arches``, specifying which architectures - that config entry is for. The keyword + that config entry is for. The keyword ``default`` applies to any architecture not - explicitly listed. The mirror url can be specified - with the ``uri`` key, or a list of mirrors to - check can be provided in order, with the first - mirror that can be resolved being selected. This - allows the same configuration to be used in - different environment, with different hosts used - for a local apt mirror. If no mirror is provided - by ``uri`` or ``search``, ``search_dns`` may be - used to search for dns names in the format + explicitly listed. The mirror url can be specified + with the ``uri`` key, or a list of mirrors to + check can be provided in order, with the first + mirror that can be resolved being selected. This + allows the same configuration to be used in + different environment, with different hosts used + for a local apt mirror. If no mirror is provided + by ``uri`` or ``search``, ``search_dns`` may be + used to search for dns names in the format ``-mirror`` in each of the following: - fqdn of this host per cloud metadata, - localdomain, - domains listed in ``/etc/resolv.conf``. + fqdn of this host per cloud metadata, + localdomain, + domains listed in ``/etc/resolv.conf``. - If there is a dns entry for ``-mirror``, - then it is assumed that there is a distro mirror - at ``http://-mirror./``. - If the ``primary`` key is defined, but not the + If there is a dns entry for ``-mirror``, + then it is assumed that there is a distro mirror + at ``http://-mirror./``. + If the ``primary`` key is defined, but not the ``security`` key, then then configuration for ``primary`` is also used for ``security``. - If ``search_dns`` is used for the ``security`` + If ``search_dns`` is used for the ``security`` key, the search pattern will be ``-security-mirror``. - If no mirrors are specified, or all lookups fail, - then default mirrors defined in the datasource - are used. If none are present in the datasource - either the following defaults are used: + If no mirrors are specified, or all lookups fail, + then default mirrors defined in the datasource + are used. If none are present in the datasource + either the following defaults are used: - ``primary`` => ``http://archive.ubuntu.com/ubuntu``, - ``security`` => - ``http://security.ubuntu.com/ubuntu`` - """) - }, + ``primary`` => ``http://archive.ubuntu.com/ubuntu``, + ``security`` => + ``http://security.ubuntu.com/ubuntu`` + """)}, 'security': { - 'type': 'array', - 'item': { - 'type': 'object', - 'additionalProperties': False, - 'required': ['arches'], - 'properties': { - 'arches': { - 'type': 'array', - 'uniqueItems': True, - 'item': { - 'type': 'string' - }, - 'minItems': 1 - }, - 'uri': { - 'type': 'string', - 'format': 'uri' - }, - 'search': { - 'type': 'array', - 'uniqueItems': True, - 'item': { - 'type': 'string', - 'format': 'uri' - }, - 'minItems': 1 - }, - 'search_dns': { - 'type': 'boolean', - } - } - } + **mirror_property, + 'description': dedent("""\ + Please refer to the primary config documentation""") }, 'add_apt_repo_match': { 'type': 'string', @@ -270,7 +243,7 @@ All source entries in ``apt-sources`` that match regex in ``add_apt_repo_match`` will be added to the system using ``add-apt-repository``. If - ``add_apt_repo_match`` is not specified, it + ``add_apt_repo_match`` is not specified, it defaults to ``^[\\w-]+:\\w``""") }, 'debconf_selections': { @@ -328,14 +301,14 @@ 'description': dedent("""\ More convinient way to specify https apt proxy. https proxy url is specified in the format - ``https://[[user][:pass]@]host[:port]/``.""") + ``https://[[user][:pass]@]host[:port]/``.""") }, 'http_proxy': { 'type': 'string', 'description': dedent("""\ More convinient way to specify http apt proxy. http proxy url is specified in the format - ``http://[[user][:pass]@]host[:port]/``.""") + ``http://[[user][:pass]@]host[:port]/``.""") }, 'proxy': { 'type': 'string', @@ -347,7 +320,7 @@ 'description': dedent("""\ More convinient way to specify ftp apt proxy. ftp proxy url is specified in the format - ``ftp://[[user][:pass]@]host[:port]/``.""") + ``ftp://[[user][:pass]@]host[:port]/``.""") }, 'sources': { 'type': 'object', @@ -364,7 +337,7 @@ be appended. If there is no configuration for a key in ``sources``, no file will be written, but the key may still be referred to as an id in other - ``sources`` entries. + ``sources`` entries. Each entry under ``sources`` is a dictionary which may contain any of the following optional keys: From 7b5966008cad4524c8444fa1dbc34773332b6d7a Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Mon, 11 May 2020 10:24:42 -0300 Subject: [PATCH 07/13] Fix apt schema examples --- doc/examples/cloud-config-apt.txt | 2 +- doc/examples/cloud-config-chef-oneiric.txt | 63 +++++++++++----------- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/doc/examples/cloud-config-apt.txt b/doc/examples/cloud-config-apt.txt index 5a56cd1b6c2..004894b778f 100644 --- a/doc/examples/cloud-config-apt.txt +++ b/doc/examples/cloud-config-apt.txt @@ -142,7 +142,7 @@ apt: # as above, allowing to have one config for different per arch mirrors # security is optional, if not defined it is set to the same value as primary security: - uri: http://security.ubuntu.com/ubuntu + - uri: http://security.ubuntu.com/ubuntu # If search_dns is set for security the searched pattern is: # -security-mirror diff --git a/doc/examples/cloud-config-chef-oneiric.txt b/doc/examples/cloud-config-chef-oneiric.txt index ab2e2a33377..241fbf9b460 100644 --- a/doc/examples/cloud-config-chef-oneiric.txt +++ b/doc/examples/cloud-config-chef-oneiric.txt @@ -13,38 +13,39 @@ # Key from http://apt.opscode.com/packages@opscode.com.gpg.key apt: sources: - - source: "deb http://apt.opscode.com/ $RELEASE-0.10 main" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v1.4.9 (GNU/Linux) + source1: + source: "deb http://apt.opscode.com/ $RELEASE-0.10 main" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1.4.9 (GNU/Linux) - mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu - twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 - dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC - JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W - ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I - XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe - DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm - sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO - Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ - YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG - CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K - +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR - lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh - DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu - wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx - EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g - w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8 - AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN - QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X - Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ - 3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V - Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL - zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb - DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG - 0GLl8EkfA8uhluM= - =zKAm - -----END PGP PUBLIC KEY BLOCK----- + mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu + twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 + dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC + JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W + ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I + XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe + DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm + sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO + Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ + YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG + CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K + +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR + lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh + DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu + wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx + EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g + w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8 + AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN + QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X + Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ + 3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V + Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL + zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb + DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG + 0GLl8EkfA8uhluM= + =zKAm + -----END PGP PUBLIC KEY BLOCK----- chef: From 87e59bc89d517887e8cbacfb5e194ab39bbe1d84 Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Mon, 11 May 2020 18:48:45 -0300 Subject: [PATCH 08/13] Add default variable to apt schema --- cloudinit/config/cc_apt_configure.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 147a8516947..02e2dce92e1 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -144,6 +144,7 @@ 'properties': { 'preserve_sources_list': { 'type': 'boolean', + 'default': False, 'description': dedent("""\ By default, cloud-init will generate a new sources list in ``/etc/apt/sources.list.d`` based on any @@ -239,6 +240,7 @@ }, 'add_apt_repo_match': { 'type': 'string', + 'default': '^[\\w-]+:\\w', 'description': dedent("""\ All source entries in ``apt-sources`` that match regex in ``add_apt_repo_match`` will be added to @@ -250,7 +252,7 @@ 'type': 'object', 'items': {'type': 'string'}, 'description': dedent("""\ - Debconf additioanl configurations can be specified as a + Debconf additional configurations can be specified as a dictionary under the ``debconf_selections`` config key, with each key in the dict representing a different set of configurations. The value of each key From e6539ab2697c0b9847f273b8ec4fd6fdfb2d36bc Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Tue, 12 May 2020 09:46:58 -0300 Subject: [PATCH 09/13] Use ADD_APT_REPO_MATCH in apt_configure schema --- cloudinit/config/cc_apt_configure.py | 141 ++++++++++++++------------- 1 file changed, 71 insertions(+), 70 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 02e2dce92e1..ee4d33571d3 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -23,6 +23,9 @@ LOG = logging.getLogger(__name__) +# this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') +ADD_APT_REPO_MATCH = r"^[\w-]+:\w" + frequency = PER_INSTANCE distros = ["ubuntu", "debian"] mirror_property = { @@ -63,20 +66,20 @@ 'title': 'Configure apt for the user', 'description': dedent("""\ This module handles both configuration of apt options and adding - source lists. There are configuration options such as - ``apt_get_wrapper`` and ``apt_get_command`` that control how - cloud-init invokes apt-get. These configuration options are - handled on a per-distro basis, so consult documentation for - cloud-init's distro support for instructions on using - these config options. - - .. note:: - To ensure that apt configuration is valid yaml, any strings - containing special characters, especially ``:`` should be quoted. - - .. note:: - For more information about apt configuration, see the - ``Additional apt configuration`` example."""), + source lists. There are configuration options such as + ``apt_get_wrapper`` and ``apt_get_command`` that control how + cloud-init invokes apt-get. These configuration options are + handled on a per-distro basis, so consult documentation for + cloud-init's distro support for instructions on using + these config options. + + .. note:: + To ensure that apt configuration is valid yaml, any strings + containing special characters, especially ``:`` should be quoted. + + .. note:: + For more information about apt configuration, see the + ``Additional apt configuration`` example."""), 'distros': distros, 'examples': [dedent("""\ apt: @@ -190,48 +193,48 @@ **mirror_property, 'description': dedent("""\ The primary and security archive mirrors can - be specified using the ``primary`` and - ``security`` keys, respectively Both the - ``primary`` and ``security`` keys take a list - of configs, allowing mirrors to be specified - on a per-architecture basis. Each config is a - dictionary which must have an entry for - ``arches``, specifying which architectures - that config entry is for. The keyword - ``default`` applies to any architecture not - explicitly listed. The mirror url can be specified - with the ``uri`` key, or a list of mirrors to - check can be provided in order, with the first - mirror that can be resolved being selected. This - allows the same configuration to be used in - different environment, with different hosts used - for a local apt mirror. If no mirror is provided - by ``uri`` or ``search``, ``search_dns`` may be - used to search for dns names in the format - ``-mirror`` in each of the following: - - fqdn of this host per cloud metadata, - localdomain, - domains listed in ``/etc/resolv.conf``. - - If there is a dns entry for ``-mirror``, - then it is assumed that there is a distro mirror - at ``http://-mirror./``. - If the ``primary`` key is defined, but not the - ``security`` key, then then configuration for - ``primary`` is also used for ``security``. - If ``search_dns`` is used for the ``security`` - key, the search pattern will be - ``-security-mirror``. - - If no mirrors are specified, or all lookups fail, - then default mirrors defined in the datasource - are used. If none are present in the datasource - either the following defaults are used: - - ``primary`` => ``http://archive.ubuntu.com/ubuntu``, - ``security`` => - ``http://security.ubuntu.com/ubuntu`` + be specified using the ``primary`` and + ``security`` keys, respectively. Both the + ``primary`` and ``security`` keys take a list + of configs, allowing mirrors to be specified + on a per-architecture basis. Each config is a + dictionary which must have an entry for + ``arches``, specifying which architectures + that config entry is for. The keyword + ``default`` applies to any architecture not + explicitly listed. The mirror url can be specified + with the ``uri`` key, or a list of mirrors to + check can be provided in order, with the first + mirror that can be resolved being selected. This + allows the same configuration to be used in + different environment, with different hosts used + for a local apt mirror. If no mirror is provided + by ``uri`` or ``search``, ``search_dns`` may be + used to search for dns names in the format + ``-mirror`` in each of the following: + + fqdn of this host per cloud metadata, + localdomain, + domains listed in ``/etc/resolv.conf``. + + If there is a dns entry for ``-mirror``, + then it is assumed that there is a distro mirror + at ``http://-mirror./``. + If the ``primary`` key is defined, but not the + ``security`` key, then then configuration for + ``primary`` is also used for ``security``. + If ``search_dns`` is used for the ``security`` + key, the search pattern will be + ``-security-mirror``. + + If no mirrors are specified, or all lookups fail, + then default mirrors defined in the datasource + are used. If none are present in the datasource + either the following defaults are used: + + ``primary`` => ``http://archive.ubuntu.com/ubuntu``, + ``security`` => + ``http://security.ubuntu.com/ubuntu`` """)}, 'security': { **mirror_property, @@ -240,13 +243,13 @@ }, 'add_apt_repo_match': { 'type': 'string', - 'default': '^[\\w-]+:\\w', + 'default': ADD_APT_REPO_MATCH, 'description': dedent("""\ All source entries in ``apt-sources`` that match regex in ``add_apt_repo_match`` will be added to the system using ``add-apt-repository``. If ``add_apt_repo_match`` is not specified, it - defaults to ``^[\\w-]+:\\w``""") + defaults to ``{}``""".format(ADD_APT_REPO_MATCH)) }, 'debconf_selections': { 'type': 'object', @@ -294,23 +297,23 @@ 'type': 'string', 'description': dedent("""\ Specify configuration for apt, such as proxy - configuratiun. This configuration is specified as a - string. For multiline apt configuration, make sure - to follow yaml syntax.""") + configuratiun. This configuration is specified as a + string. For multiline apt configuration, make sure + to follow yaml syntax.""") }, 'https_proxy': { 'type': 'string', 'description': dedent("""\ More convinient way to specify https apt proxy. - https proxy url is specified in the format - ``https://[[user][:pass]@]host[:port]/``.""") + https proxy url is specified in the format + ``https://[[user][:pass]@]host[:port]/``.""") }, 'http_proxy': { 'type': 'string', 'description': dedent("""\ More convinient way to specify http apt proxy. - http proxy url is specified in the format - ``http://[[user][:pass]@]host[:port]/``.""") + http proxy url is specified in the format + ``http://[[user][:pass]@]host[:port]/``.""") }, 'proxy': { 'type': 'string', @@ -321,8 +324,8 @@ 'type': 'string', 'description': dedent("""\ More convinient way to specify ftp apt proxy. - ftp proxy url is specified in the format - ``ftp://[[user][:pass]@]host[:port]/``.""") + ftp proxy url is specified in the format + ``ftp://[[user][:pass]@]host[:port]/``.""") }, 'sources': { 'type': 'object', @@ -330,7 +333,7 @@ 'description': dedent("""\ Source list entries can be specified as a dictionary under the ``sources`` config key, with - key in the dict representing a different source + each key in the dict representing a different source file. The key of each source entry will be used as an id that can be referenced in other config entries, as well as the filename for the source's @@ -367,8 +370,6 @@ __doc__ = get_schema_doc(schema) -# this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') -ADD_APT_REPO_MATCH = r"^[\w-]+:\w" # place where apt stores cached repository data APT_LISTS = "/var/lib/apt/lists" From c3b6ae2507865b3b34eef79dc70f1b57485851bf Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Tue, 12 May 2020 09:55:36 -0300 Subject: [PATCH 10/13] Add bullet points on apt_configure schema --- cloudinit/config/cc_apt_configure.py | 45 ++++++++++++++-------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ee4d33571d3..d37dc52771e 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -213,9 +213,9 @@ used to search for dns names in the format ``-mirror`` in each of the following: - fqdn of this host per cloud metadata, - localdomain, - domains listed in ``/etc/resolv.conf``. + - fqdn of this host per cloud metadata, + - localdomain, + - domains listed in ``/etc/resolv.conf``. If there is a dns entry for ``-mirror``, then it is assumed that there is a distro mirror @@ -232,8 +232,8 @@ are used. If none are present in the datasource either the following defaults are used: - ``primary`` => ``http://archive.ubuntu.com/ubuntu``, - ``security`` => + - ``primary`` => ``http://archive.ubuntu.com/ubuntu``, + - ``security`` => ``http://security.ubuntu.com/ubuntu`` """)}, 'security': { @@ -270,10 +270,10 @@ Where: - ``pkgname`` is the name of the package, - ``question`` the name of the questions, - ``type`` is the type of question, - ``answer`` is the value used to ansert the + - ``pkgname`` is the name of the package, + - ``question`` the name of the questions, + - ``type`` is the type of question, + - ``answer`` is the value used to ansert the question. For example: ippackage ippackage/ip string 127.0.01 @@ -288,10 +288,10 @@ this template, the following strings will be replaced with the appropriate values: - ``$MIRROR``, - ``$RELEASE``, - ``$PRIMARY``, - ``$SECURITY``""") + - ``$MIRROR``, + - ``$RELEASE``, + - ``$PRIMARY``, + - ``$SECURITY``""") }, 'conf': { 'type': 'string', @@ -317,8 +317,7 @@ }, 'proxy': { 'type': 'string', - 'description': dedent("""\ - Alias for defining a http apt proxy.""") + 'description': 'Alias for defining a http apt proxy.' }, 'ftp_proxy': { 'type': 'string', @@ -347,21 +346,21 @@ Each entry under ``sources`` is a dictionary which may contain any of the following optional keys: - ``source``: a sources.list entry \ + - ``source``: a sources.list entry \ (some variable replacements apply). - ``keyid``: a key to import via shortid or \ + - ``keyid``: a key to import via shortid or \ fingerprin. - ``key``: a raw PGP key. - ``keyserver``: alternate keyserver to pull \ + - ``key``: a raw PGP key. + - ``keyserver``: alternate keyserver to pull \ ``keyid`` key from. The ``source`` key supports variable replacements for the following strings: - ``$MIRROR``, - ``$PRIMARY``, - ``$SECURITY``, - ``$RELEASE``""") + - ``$MIRROR``, + - ``$PRIMARY``, + - ``$SECURITY``, + - ``$RELEASE``""") } } } From 83584a2917d42ce46ed96c167bbdf73cb3cd8874 Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Tue, 12 May 2020 10:14:04 -0300 Subject: [PATCH 11/13] Better format bullet points --- cloudinit/config/cc_apt_configure.py | 31 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index d37dc52771e..2c323392520 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -179,10 +179,10 @@ For convenience, several aliases are provided for ``disable_suites``: - ``updates`` => ``$RELEASE-updates``, + ``updates`` => ``$RELEASE-updates``. ``backports`` => ``$RELEASE-backports``. - ``security`` => ``$RELEASE-security``, - ``proposed`` => ``$RELEASE-proposed``, + ``security`` => ``$RELEASE-security``. + ``proposed`` => ``$RELEASE-proposed``. ``release`` => ``$RELEASE``. When a suite is disabled using ``disable_suites``, @@ -232,7 +232,8 @@ are used. If none are present in the datasource either the following defaults are used: - - ``primary`` => ``http://archive.ubuntu.com/ubuntu``, + - ``primary`` => + ``http://archive.ubuntu.com/ubuntu``. - ``security`` => ``http://security.ubuntu.com/ubuntu`` """)}, @@ -270,11 +271,11 @@ Where: - - ``pkgname`` is the name of the package, - - ``question`` the name of the questions, - - ``type`` is the type of question, + - ``pkgname`` is the name of the package. + - ``question`` the name of the questions. + - ``type`` is the type of question. - ``answer`` is the value used to ansert the - question. + question. For example: ippackage ippackage/ip string 127.0.01 """) @@ -288,9 +289,9 @@ this template, the following strings will be replaced with the appropriate values: - - ``$MIRROR``, - - ``$RELEASE``, - - ``$PRIMARY``, + - ``$MIRROR`` + - ``$RELEASE`` + - ``$PRIMARY`` - ``$SECURITY``""") }, 'conf': { @@ -349,7 +350,7 @@ - ``source``: a sources.list entry \ (some variable replacements apply). - ``keyid``: a key to import via shortid or \ - fingerprin. + fingerprint. - ``key``: a raw PGP key. - ``keyserver``: alternate keyserver to pull \ ``keyid`` key from. @@ -357,9 +358,9 @@ The ``source`` key supports variable replacements for the following strings: - - ``$MIRROR``, - - ``$PRIMARY``, - - ``$SECURITY``, + - ``$MIRROR`` + - ``$PRIMARY`` + - ``$SECURITY`` - ``$RELEASE``""") } } From 1ff22235ab0a194a8750fb87c512688e5162c0b3 Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Wed, 13 May 2020 10:17:45 -0300 Subject: [PATCH 12/13] Update description parser --- cloudinit/config/cc_apt_configure.py | 293 ++++++++++---------- cloudinit/config/cc_ntp.py | 42 +-- cloudinit/config/cc_write_files.py | 22 +- cloudinit/config/schema.py | 24 +- tests/unittests/test_handler/test_schema.py | 35 +++ 5 files changed, 237 insertions(+), 179 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 2c323392520..b2164f4a709 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -66,20 +66,20 @@ 'title': 'Configure apt for the user', 'description': dedent("""\ This module handles both configuration of apt options and adding - source lists. There are configuration options such as - ``apt_get_wrapper`` and ``apt_get_command`` that control how - cloud-init invokes apt-get. These configuration options are - handled on a per-distro basis, so consult documentation for - cloud-init's distro support for instructions on using - these config options. - - .. note:: - To ensure that apt configuration is valid yaml, any strings - containing special characters, especially ``:`` should be quoted. - - .. note:: - For more information about apt configuration, see the - ``Additional apt configuration`` example."""), + source lists. There are configuration options such as + ``apt_get_wrapper`` and ``apt_get_command`` that control how + cloud-init invokes apt-get. These configuration options are + handled on a per-distro basis, so consult documentation for + cloud-init's distro support for instructions on using + these config options. + + .. note:: + To ensure that apt configuration is valid yaml, any strings + containing special characters, especially ``:`` should be quoted. + + .. note:: + For more information about apt configuration, see the + ``Additional apt configuration`` example."""), 'distros': distros, 'examples': [dedent("""\ apt: @@ -150,17 +150,17 @@ 'default': False, 'description': dedent("""\ By default, cloud-init will generate a new sources - list in ``/etc/apt/sources.list.d`` based on any - changes specified in cloud config. To disable this - behavior and preserve the sources list from the - pristine image, set ``preserve_sources_list`` - to ``true``. + list in ``/etc/apt/sources.list.d`` based on any + changes specified in cloud config. To disable this + behavior and preserve the sources list from the + pristine image, set ``preserve_sources_list`` + to ``true``. The ``preserve_sources_list`` option overrides - all other config keys that would alter - ``sources.list`` or ``sources.list.d``, - **except** for additional sources to be added - to ``sources.list.d``.""") + all other config keys that would alter + ``sources.list`` or ``sources.list.d``, + **except** for additional sources to be added + to ``sources.list.d``.""") }, 'disable_suites': { 'type': 'array', @@ -170,20 +170,20 @@ 'uniqueItems': True, 'description': dedent("""\ Entries in the sources list can be disabled using - ``disable_suites``, which takes a list of suites - to be disabled. If the string ``$RELEASE`` is - present in a suite in the ``disable_suites`` list, - it will be replaced with the release name. If a - suite specified in ``disable_suites`` is not - present in ``sources.list`` it will be ignored. - For convenience, several aliases are provided for - ``disable_suites``: - - ``updates`` => ``$RELEASE-updates``. - ``backports`` => ``$RELEASE-backports``. - ``security`` => ``$RELEASE-security``. - ``proposed`` => ``$RELEASE-proposed``. - ``release`` => ``$RELEASE``. + ``disable_suites``, which takes a list of suites + to be disabled. If the string ``$RELEASE`` is + present in a suite in the ``disable_suites`` list, + it will be replaced with the release name. If a + suite specified in ``disable_suites`` is not + present in ``sources.list`` it will be ignored. + For convenience, several aliases are provided for + ``disable_suites``: + + - ``updates`` => ``$RELEASE-updates`` + - ``backports`` => ``$RELEASE-backports`` + - ``security`` => ``$RELEASE-security`` + - ``proposed`` => ``$RELEASE-proposed`` + - ``release`` => ``$RELEASE``. When a suite is disabled using ``disable_suites``, its entry in ``sources.list`` is not deleted; it @@ -193,49 +193,49 @@ **mirror_property, 'description': dedent("""\ The primary and security archive mirrors can - be specified using the ``primary`` and - ``security`` keys, respectively. Both the - ``primary`` and ``security`` keys take a list - of configs, allowing mirrors to be specified - on a per-architecture basis. Each config is a - dictionary which must have an entry for - ``arches``, specifying which architectures - that config entry is for. The keyword - ``default`` applies to any architecture not - explicitly listed. The mirror url can be specified - with the ``uri`` key, or a list of mirrors to - check can be provided in order, with the first - mirror that can be resolved being selected. This - allows the same configuration to be used in - different environment, with different hosts used - for a local apt mirror. If no mirror is provided - by ``uri`` or ``search``, ``search_dns`` may be - used to search for dns names in the format - ``-mirror`` in each of the following: - - - fqdn of this host per cloud metadata, - - localdomain, - - domains listed in ``/etc/resolv.conf``. - - If there is a dns entry for ``-mirror``, - then it is assumed that there is a distro mirror - at ``http://-mirror./``. - If the ``primary`` key is defined, but not the - ``security`` key, then then configuration for - ``primary`` is also used for ``security``. - If ``search_dns`` is used for the ``security`` - key, the search pattern will be - ``-security-mirror``. - - If no mirrors are specified, or all lookups fail, - then default mirrors defined in the datasource - are used. If none are present in the datasource - either the following defaults are used: - - - ``primary`` => - ``http://archive.ubuntu.com/ubuntu``. - - ``security`` => - ``http://security.ubuntu.com/ubuntu`` + be specified using the ``primary`` and + ``security`` keys, respectively. Both the + ``primary`` and ``security`` keys take a list + of configs, allowing mirrors to be specified + on a per-architecture basis. Each config is a + dictionary which must have an entry for + ``arches``, specifying which architectures + that config entry is for. The keyword + ``default`` applies to any architecture not + explicitly listed. The mirror url can be specified + with the ``uri`` key, or a list of mirrors to + check can be provided in order, with the first + mirror that can be resolved being selected. This + allows the same configuration to be used in + different environment, with different hosts used + for a local apt mirror. If no mirror is provided + by ``uri`` or ``search``, ``search_dns`` may be + used to search for dns names in the format + ``-mirror`` in each of the following: + + - fqdn of this host per cloud metadata, + - localdomain, + - domains listed in ``/etc/resolv.conf``. + + If there is a dns entry for ``-mirror``, + then it is assumed that there is a distro mirror + at ``http://-mirror./``. + If the ``primary`` key is defined, but not the + ``security`` key, then then configuration for + ``primary`` is also used for ``security``. + If ``search_dns`` is used for the ``security`` + key, the search pattern will be + ``-security-mirror``. + + If no mirrors are specified, or all lookups fail, + then default mirrors defined in the datasource + are used. If none are present in the datasource + either the following defaults are used: + + - ``primary`` => \ + ``http://archive.ubuntu.com/ubuntu``. + - ``security`` => \ + ``http://security.ubuntu.com/ubuntu`` """)}, 'security': { **mirror_property, @@ -247,74 +247,75 @@ 'default': ADD_APT_REPO_MATCH, 'description': dedent("""\ All source entries in ``apt-sources`` that match - regex in ``add_apt_repo_match`` will be added to - the system using ``add-apt-repository``. If - ``add_apt_repo_match`` is not specified, it - defaults to ``{}``""".format(ADD_APT_REPO_MATCH)) + regex in ``add_apt_repo_match`` will be added to + the system using ``add-apt-repository``. If + ``add_apt_repo_match`` is not specified, it + defaults to ``{}``""".format(ADD_APT_REPO_MATCH)) }, 'debconf_selections': { 'type': 'object', 'items': {'type': 'string'}, 'description': dedent("""\ Debconf additional configurations can be specified as a - dictionary under the ``debconf_selections`` config - key, with each key in the dict representing a - different set of configurations. The value of each key - must be a string containing all the debconf - configurations that must be applied. We will bundle - all of the values and pass them to - ``debconf-set-selections``. Therefore, each value line - must be a valid entry for ``debconf-set-selections``, - meaning that they must possess for distinct fields: + dictionary under the ``debconf_selections`` config + key, with each key in the dict representing a + different set of configurations. The value of each key + must be a string containing all the debconf + configurations that must be applied. We will bundle + all of the values and pass them to + ``debconf-set-selections``. Therefore, each value line + must be a valid entry for ``debconf-set-selections``, + meaning that they must possess for distinct fields: - pkgname question type answer + ``pkgname question type answer`` - Where: + Where: - ``pkgname`` is the name of the package. - ``question`` the name of the questions. - ``type`` is the type of question. - - ``answer`` is the value used to ansert the - question. + - ``answer`` is the value used to ansert the \ + question. - For example: ippackage ippackage/ip string 127.0.01 + For example: \ + ``ippackage ippackage/ip string 127.0.01`` """) }, 'sources_list': { 'type': 'string', 'description': dedent("""\ Specifies a custom template for rendering - ``sources.list`` . If no ``sources_list`` template - is given, cloud-init will use sane default. Within - this template, the following strings will be - replaced with the appropriate values: - - - ``$MIRROR`` - - ``$RELEASE`` - - ``$PRIMARY`` - - ``$SECURITY``""") + ``sources.list`` . If no ``sources_list`` template + is given, cloud-init will use sane default. Within + this template, the following strings will be + replaced with the appropriate values: + + - ``$MIRROR`` + - ``$RELEASE`` + - ``$PRIMARY`` + - ``$SECURITY``""") }, 'conf': { 'type': 'string', 'description': dedent("""\ Specify configuration for apt, such as proxy - configuratiun. This configuration is specified as a - string. For multiline apt configuration, make sure - to follow yaml syntax.""") + configuratiun. This configuration is specified as a + string. For multiline apt configuration, make sure + to follow yaml syntax.""") }, 'https_proxy': { 'type': 'string', 'description': dedent("""\ More convinient way to specify https apt proxy. - https proxy url is specified in the format - ``https://[[user][:pass]@]host[:port]/``.""") + https proxy url is specified in the format + ``https://[[user][:pass]@]host[:port]/``.""") }, 'http_proxy': { 'type': 'string', 'description': dedent("""\ More convinient way to specify http apt proxy. - http proxy url is specified in the format - ``http://[[user][:pass]@]host[:port]/``.""") + http proxy url is specified in the format + ``http://[[user][:pass]@]host[:port]/``.""") }, 'proxy': { 'type': 'string', @@ -324,44 +325,44 @@ 'type': 'string', 'description': dedent("""\ More convinient way to specify ftp apt proxy. - ftp proxy url is specified in the format - ``ftp://[[user][:pass]@]host[:port]/``.""") + ftp proxy url is specified in the format + ``ftp://[[user][:pass]@]host[:port]/``.""") }, 'sources': { 'type': 'object', 'items': {'type': 'string'}, 'description': dedent("""\ Source list entries can be specified as a - dictionary under the ``sources`` config key, with - each key in the dict representing a different source - file. The key of each source entry will be used - as an id that can be referenced in other config - entries, as well as the filename for the source's - configuration under ``/etc/apt/sources.list.d``. - If the name does not end with ``.list``, it will - be appended. If there is no configuration for a - key in ``sources``, no file will be written, but - the key may still be referred to as an id in other - ``sources`` entries. - - Each entry under ``sources`` is a dictionary which - may contain any of the following optional keys: - - - ``source``: a sources.list entry \ - (some variable replacements apply). - - ``keyid``: a key to import via shortid or \ - fingerprint. - - ``key``: a raw PGP key. - - ``keyserver``: alternate keyserver to pull \ - ``keyid`` key from. - - The ``source`` key supports variable - replacements for the following strings: - - - ``$MIRROR`` - - ``$PRIMARY`` - - ``$SECURITY`` - - ``$RELEASE``""") + dictionary under the ``sources`` config key, with + each key in the dict representing a different source + file. The key of each source entry will be used + as an id that can be referenced in other config + entries, as well as the filename for the source's + configuration under ``/etc/apt/sources.list.d``. + If the name does not end with ``.list``, it will + be appended. If there is no configuration for a + key in ``sources``, no file will be written, but + the key may still be referred to as an id in other + ``sources`` entries. + + Each entry under ``sources`` is a dictionary which + may contain any of the following optional keys: + + - ``source``: a sources.list entry \ + (some variable replacements apply). + - ``keyid``: a key to import via shortid or \ + fingerprint. + - ``key``: a raw PGP key. + - ``keyserver``: alternate keyserver to pull \ + ``keyid`` key from. + + The ``source`` key supports variable + replacements for the following strings: + + - ``$MIRROR`` + - ``$PRIMARY`` + - ``$SECURITY`` + - ``$RELEASE``""") } } } diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 5498bbaa864..3b2c2020605 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -169,8 +169,8 @@ 'uniqueItems': True, 'description': dedent("""\ List of ntp pools. If both pools and servers are - empty, 4 default pool servers will be provided of - the format ``{0-3}.{distro}.pool.ntp.org``.""") + empty, 4 default pool servers will be provided of + the format ``{0-3}.{distro}.pool.ntp.org``.""") }, 'servers': { 'type': 'array', @@ -181,46 +181,46 @@ 'uniqueItems': True, 'description': dedent("""\ List of ntp servers. If both pools and servers are - empty, 4 default pool servers will be provided with - the format ``{0-3}.{distro}.pool.ntp.org``.""") + empty, 4 default pool servers will be provided with + the format ``{0-3}.{distro}.pool.ntp.org``.""") }, 'ntp_client': { 'type': 'string', 'default': 'auto', 'description': dedent("""\ Name of an NTP client to use to configure system NTP. - When unprovided or 'auto' the default client preferred - by the distribution will be used. The following - built-in client names can be used to override existing - configuration defaults: chrony, ntp, ntpdate, - systemd-timesyncd."""), + When unprovided or 'auto' the default client preferred + by the distribution will be used. The following + built-in client names can be used to override existing + configuration defaults: chrony, ntp, ntpdate, + systemd-timesyncd."""), }, 'enabled': { 'type': 'boolean', 'default': True, 'description': dedent("""\ Attempt to enable ntp clients if set to True. If set - to False, ntp client will not be configured or - installed"""), + to False, ntp client will not be configured or + installed"""), }, 'config': { 'description': dedent("""\ Configuration settings or overrides for the - ``ntp_client`` specified."""), + ``ntp_client`` specified."""), 'type': ['object'], 'properties': { 'confpath': { 'type': 'string', 'description': dedent("""\ The path to where the ``ntp_client`` - configuration is written."""), + configuration is written."""), }, 'check_exe': { 'type': 'string', 'description': dedent("""\ The executable name for the ``ntp_client``. - For example, ntp service ``check_exe`` is - 'ntpd' because it runs the ntpd binary."""), + For example, ntp service ``check_exe`` is + 'ntpd' because it runs the ntpd binary."""), }, 'packages': { 'type': 'array', @@ -230,22 +230,22 @@ 'uniqueItems': True, 'description': dedent("""\ List of packages needed to be installed for the - selected ``ntp_client``."""), + selected ``ntp_client``."""), }, 'service_name': { 'type': 'string', 'description': dedent("""\ The systemd or sysvinit service name used to - start and stop the ``ntp_client`` - service."""), + start and stop the ``ntp_client`` + service."""), }, 'template': { 'type': 'string', 'description': dedent("""\ Inline template allowing users to define their - own ``ntp_client`` configuration template. - The value must start with '## template:jinja' - to enable use of templating support. + own ``ntp_client`` configuration template. + The value must start with '## template:jinja' + to enable use of templating support. """), }, }, diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 204cbfd674d..8601e70799f 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -103,7 +103,7 @@ 'type': 'string', 'description': dedent("""\ Path of the file to which ``content`` is decoded - and written + and written """), }, 'content': { @@ -111,9 +111,9 @@ 'default': '', 'description': dedent("""\ Optional content to write to the provided ``path``. - When content is present and encoding is not '%s', - decode the content prior to writing. Default: - **''** + When content is present and encoding is not '%s', + decode the content prior to writing. Default: + **''** """ % UNKNOWN_ENC), }, 'owner': { @@ -121,7 +121,7 @@ 'default': DEFAULT_OWNER, 'description': dedent("""\ Optional owner:group to chown on the file. Default: - **{owner}** + **{owner}** """.format(owner=DEFAULT_OWNER)), }, 'permissions': { @@ -129,8 +129,8 @@ 'default': oct(DEFAULT_PERMS).replace('o', ''), 'description': dedent("""\ Optional file permissions to set on ``path`` - represented as an octal string '0###'. Default: - **'{perms}'** + represented as an octal string '0###'. Default: + **'{perms}'** """.format(perms=oct(DEFAULT_PERMS).replace('o', ''))), }, 'encoding': { @@ -139,16 +139,16 @@ 'enum': supported_encoding_types, 'description': dedent("""\ Optional encoding type of the content. Default is - **text/plain** and no content decoding is - performed. Supported encoding types are: - %s.""" % ", ".join(supported_encoding_types)), + **text/plain** and no content decoding is + performed. Supported encoding types are: + %s.""" % ", ".join(supported_encoding_types)), }, 'append': { 'type': 'boolean', 'default': False, 'description': dedent("""\ Whether to append ``content`` to existing file if - ``path`` exists. Default: **false**. + ``path`` exists. Default: **false**. """), }, }, diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index a295d63d3ba..da00060aa5d 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -306,6 +306,28 @@ def _get_property_type(property_dict): return property_type +def _parse_description(description, prefix): + """Parse description from the schema in a format that we can better + display in our docs. This parser does three things: + + - Guarantee that a paragraph will be in a single line + - Guarantee that each new paragraph will be aligned with + the first paragraph + - Proper align lists of items + + @param description: The original description in the schema. + @param prefix: The number of spaces used to align the current description + """ + list_paragraph = prefix * 3 + description = re.sub(r"(\S)\n(\S)", r"\1 \2", description) + description = re.sub( + r"\n\n", r"\n\n{}".format(prefix), description) + description = re.sub( + r"\n( +)-", r"\n{}-".format(list_paragraph), description) + + return description + + def _get_property_doc(schema, prefix=' '): """Return restructured text describing the supported schema properties.""" new_prefix = prefix + ' ' @@ -318,7 +340,7 @@ def _get_property_doc(schema, prefix=' '): prefix=prefix, prop_name=prop_key, type=_get_property_type(prop_config), - description=description.replace('\n', ''))) + description=_parse_description(description, prefix))) items = prop_config.get('items') if items: if isinstance(items, list): diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index bd25e4e62c9..e19d13b8d6a 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -290,6 +290,41 @@ def test_get_schema_doc_handles_string_examples(self): """), get_schema_doc(full_schema)) + def test_get_schema_doc_properly_parse_description(self): + """get_schema_doc description properly formatted""" + full_schema = copy(self.required_schema) + full_schema.update( + {'properties': { + 'p1': { + 'type': 'string', + 'description': dedent("""\ + This item + has the + following options: + + - option1 + - option2 + - option3 + + The default value is + option1""") + } + }} + ) + + self.assertIn( + dedent(""" + **Config schema**: + **p1:** (string) This item has the following options: + + - option1 + - option2 + - option3 + + The default value is option1 + """), + get_schema_doc(full_schema)) + def test_get_schema_doc_raises_key_errors(self): """get_schema_doc raises KeyErrors on missing keys.""" for key in self.required_schema: From 6c4513aba1faa862ab659ecfde6b856d40305e93 Mon Sep 17 00:00:00 2001 From: Lucas Moura Date: Wed, 13 May 2020 16:48:13 -0300 Subject: [PATCH 13/13] Fix typos in cc_apt_configure schema --- cloudinit/config/cc_apt_configure.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index b2164f4a709..9a33451df19 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -299,21 +299,21 @@ 'type': 'string', 'description': dedent("""\ Specify configuration for apt, such as proxy - configuratiun. This configuration is specified as a + configuration. This configuration is specified as a string. For multiline apt configuration, make sure to follow yaml syntax.""") }, 'https_proxy': { 'type': 'string', 'description': dedent("""\ - More convinient way to specify https apt proxy. + More convenient way to specify https apt proxy. https proxy url is specified in the format ``https://[[user][:pass]@]host[:port]/``.""") }, 'http_proxy': { 'type': 'string', 'description': dedent("""\ - More convinient way to specify http apt proxy. + More convenient way to specify http apt proxy. http proxy url is specified in the format ``http://[[user][:pass]@]host[:port]/``.""") }, @@ -324,7 +324,7 @@ 'ftp_proxy': { 'type': 'string', 'description': dedent("""\ - More convinient way to specify ftp apt proxy. + More convenient way to specify ftp apt proxy. ftp proxy url is specified in the format ``ftp://[[user][:pass]@]host[:port]/``.""") },