Promote cloud-init schema (SC-953)

Makefile

@@ -108,7 +108,8 @@ _CHECK_SPELLING := find doc -type f -exec spellintian {} + | \
        grep -v -e 'doc/rtd/topics/cli.rst: modules modules' \
                -e 'doc/examples/cloud-config-mcollective.txt: WARNING WARNING' \
                -e 'doc/examples/cloud-config-power-state.txt: Bye Bye' \
-               -e 'doc/examples/cloud-config.txt: Bye Bye'
+               -e 'doc/examples/cloud-config.txt: Bye Bye' \
+               -e 'doc/rtd/topics/cli.rst: DOCS DOCS'
 
 
 # For CI we require a failing return code when spellintian finds spelling errors

bash_completion/cloud-init

@@ -45,6 +45,9 @@ _cloudinit_complete()
 
                 query)
                     COMPREPLY=($(compgen -W "--all --help --instance-data --list-keys --user-data --vendor-data --debug" -- $cur_word));;
+                schema)
+                    COMPREPLY=($(compgen -W "--help --config-file --docs --annotate --system" -- $cur_word))
+                    ;;
                 single)
                     COMPREPLY=($(compgen -W "--help --name --frequency --report" -- $cur_word))
                     ;;
@@ -72,9 +75,6 @@ _cloudinit_complete()
                     ;;
                 render)
                     COMPREPLY=($(compgen -W "--help --instance-data --debug" -- $cur_word));;
-                schema)
-                    COMPREPLY=($(compgen -W "--help --config-file --doc --annotate" -- $cur_word))
-                    ;;
                 show)
                     COMPREPLY=($(compgen -W "--help --format --infile --outfile" -- $cur_word))
                     ;;

cloudinit/cmd/devel/parser.py

@@ -6,8 +6,6 @@
 
 import argparse
 
-from cloudinit.config import schema
-
 from . import hotplug_hook, make_mime, net_convert, render
 
 
@@ -27,12 +25,6 @@ def get_parser(parser=None):
             hotplug_hook.get_parser,
             hotplug_hook.handle_args,
         ),
-        (
-            "schema",
-            "Validate cloud-config files for document schema",
-            schema.get_parser,
-            schema.handle_schema_args,
-        ),
         (
             net_convert.NAME,
             net_convert.__doc__,

cloudinit/cmd/main.py

@@ -836,8 +836,7 @@ def main_features(name, args):
 def main(sysv_args=None):
     if not sysv_args:
         sysv_args = sys.argv
-    parser = argparse.ArgumentParser(prog=sysv_args[0])
-    sysv_args = sysv_args[1:]
+    parser = argparse.ArgumentParser(prog=sysv_args.pop(0))
 
     # Top level args
     parser.add_argument(
@@ -956,7 +955,9 @@ def main(sysv_args=None):
         "analyze", help="Devel tool: Analyze cloud-init logs and data"
     )
 
-    parser_devel = subparsers.add_parser("devel", help="Run development tools")
+    parser_devel = subparsers.add_parser(
+        "devel", help="Run development tools."
+    )
 
     parser_collect_logs = subparsers.add_parser(
         "collect-logs", help="Collect and tar all cloud-init debug info"
@@ -970,19 +971,24 @@ def main(sysv_args=None):
         "status", help="Report cloud-init status or wait on completion."
     )
 
+    parser_schema = subparsers.add_parser(
+        "schema", help="Validate cloud-config files using jsonschema."
+    )
+
     if sysv_args:
         # Only load subparsers if subcommand is specified to avoid load cost
-        if sysv_args[0] == "analyze":
+        subcommand = sysv_args[0]
+        if subcommand == "analyze":
             from cloudinit.analyze.__main__ import get_parser as analyze_parser
 
             # Construct analyze subcommand parser
             analyze_parser(parser_analyze)
-        elif sysv_args[0] == "devel":
+        elif subcommand == "devel":
             from cloudinit.cmd.devel.parser import get_parser as devel_parser
 
             # Construct devel subcommand parser
             devel_parser(parser_devel)
-        elif sysv_args[0] == "collect-logs":
+        elif subcommand == "collect-logs":
             from cloudinit.cmd.devel.logs import (
                 get_parser as logs_parser,
                 handle_collect_logs_args,
@@ -992,23 +998,31 @@ def main(sysv_args=None):
             parser_collect_logs.set_defaults(
                 action=("collect-logs", handle_collect_logs_args)
             )
-        elif sysv_args[0] == "clean":
+        elif subcommand == "clean":
             from cloudinit.cmd.clean import (
                 get_parser as clean_parser,
                 handle_clean_args,
             )
 
             clean_parser(parser_clean)
             parser_clean.set_defaults(action=("clean", handle_clean_args))
-        elif sysv_args[0] == "query":
+        elif subcommand == "query":
             from cloudinit.cmd.query import (
                 get_parser as query_parser,
                 handle_args as handle_query_args,
             )
 
             query_parser(parser_query)
             parser_query.set_defaults(action=("render", handle_query_args))
-        elif sysv_args[0] == "status":
+        elif subcommand == "schema":
+            from cloudinit.config.schema import (
+                get_parser as schema_parser,
+                handle_schema_args,
+            )
+
+            schema_parser(parser_schema)
+            parser_schema.set_defaults(action=("schema", handle_schema_args))
+        elif subcommand == "status":
             from cloudinit.cmd.status import (
                 get_parser as status_parser,
                 handle_status_args,

doc/man/cloud-init.1

@@ -78,6 +78,10 @@ Activates modules using a given configuration key.
 .B "query"
 Query standardized instance metadata from the command line.
 
+.TP
+.B "schema"
+Validate cloud-config files using jsonschema.
+
 .TP
 .B "single"
 Run a single module.

doc/rtd/topics/cli.rst

@@ -36,6 +36,7 @@ option. This can be used against cloud-init itself or any of its subcommands.
         collect-logs        Collect and tar all cloud-init debug info
         clean               Remove logs and artifacts so cloud-init can re-run.
         status              Report cloud-init status or wait on completion.
+        schema              Validate cloud-config files using jsonschema.
 
 The rest of this document will give an overview of each of the subcommands.
 
@@ -114,11 +115,6 @@ Current subcommands:
    from ``/run/cloud-init/instance-data.json``. It accepts a user-data file
    containing  the jinja template header ``## template: jinja`` and renders
    that content with any instance-data.json variables present.
- * ``schema``: a **#cloud-config** format and schema
-   validator. It accepts a cloud-config YAML file and annotates potential
-   schema errors locally without the need for deployment. Schema
-   validation is work in progress and supports a subset of cloud-config
-   modules.
  * ``hotplug-hook``: respond to newly added system devices by retrieving
    updated system metadata and bringing up/down the corresponding device.
    This command is intended to be called via a systemd service and is
@@ -249,6 +245,29 @@ This data can then be formatted to generate custom strings or data:
   custom-i-0e91f69987f37ec74.us-east-2.aws.com
 
 
+.. _cli_schema:
+
+schema
+======
+
+Validate cloud-config files using jsonschema.
+
+* ``-h, --help``:            show this help message and exit
+* ``-c CONFIG_FILE, --config-file CONFIG_FILE``: Path of the cloud-config yaml
+  file to validate
+* ``--system``:              Validate the system cloud-config userdata
+* ``-d DOCS [DOCS ...], --docs DOCS [DOCS ...]``: Print schema module docs.
+  Choices: all or space-delimited cc_names.
+* ``--annotate``:            Annotate existing cloud-config file with errors
+
+The following example checks a config file and annotates the config file with
+errors on stdout.
+
+.. code-block:: shell-session
+
+  $ cloud-init schema -c ./config.yml --annotate
+
+
 .. _cli_single:
 
 single

doc/rtd/topics/faq.rst

@@ -146,7 +146,7 @@ provided to the system:
 
 .. code-block:: shell-session
 
-    $ cloud-init devel schema --system --annotate
+    $ cloud-init schema --system --annotate
 
 As launching instances in the cloud can cost money and take a bit longer,
 sometimes it is easier to launch instances locally using Multipass or LXD:

doc/rtd/topics/tutorial.rst

@@ -95,7 +95,7 @@ We can also assert the user data we provided is a valid cloud-config:
 
 .. code-block:: shell-session
 
-    $ cloud-init devel schema --system --annotate
+    $ cloud-init schema --system --annotate
     Valid cloud-config: system userdata
     $
 

tests/integration_tests/modules/test_cli.py

@@ -28,11 +28,11 @@
 
 @pytest.mark.user_data(VALID_USER_DATA)
 def test_valid_userdata(client: IntegrationInstance):
-    """Test `cloud-init devel schema` with valid userdata.
+    """Test `cloud-init schema` with valid userdata.
 
     PR #575
     """
-    result = client.execute("cloud-init devel schema --system")
+    result = client.execute("cloud-init schema --system")
     assert result.ok
     assert "Valid cloud-config: system userdata" == result.stdout.strip()
     result = client.execute("cloud-init status --long")
@@ -44,11 +44,11 @@ def test_valid_userdata(client: IntegrationInstance):
 
 @pytest.mark.user_data(INVALID_USER_DATA_HEADER)
 def test_invalid_userdata(client: IntegrationInstance):
-    """Test `cloud-init devel schema` with invalid userdata.
+    """Test `cloud-init schema` with invalid userdata.
 
     PR #575
     """
-    result = client.execute("cloud-init devel schema --system")
+    result = client.execute("cloud-init schema --system")
     assert not result.ok
     assert "Cloud config schema errors" in result.stderr
     assert 'needs to begin with "#cloud-config"' in result.stderr

tests/unittests/test_cli.py

@@ -134,6 +134,7 @@ def test_all_subcommands_represented_in_help(self):
             "init",
             "modules",
             "single",
+            "schema",
         ]
         for subcommand in expected_subcommands:
             self.assertIn(subcommand, error)
@@ -169,13 +170,15 @@ def test_conditional_subcommands_from_entry_point_sys_argv(self):
             "usage: cloud-init collect-logs",
             "usage: cloud-init devel",
             "usage: cloud-init status",
+            "usage: cloud-init schema",
         ]
         conditional_subcommands = [
             "analyze",
             "clean",
             "collect-logs",
             "devel",
             "status",
+            "schema",
         ]
         # The cloud-init entrypoint calls main without passing sys_argv
         for subcommand in conditional_subcommands:
@@ -220,18 +223,18 @@ def test_status_subcommand_parser(self):
         self._call_main(["cloud-init", "status", "-h"])
         self.assertIn("usage: cloud-init status", stdout.getvalue())
 
-    def test_devel_subcommand_parser(self):
-        """The subcommand cloud-init devel calls the correct subparser."""
-        self._call_main(["cloud-init", "devel"])
+    def test_subcommand_parser(self):
+        """The subcommand cloud-init schema calls the correct subparser."""
+        self._call_main(["cloud-init"])
         # These subcommands only valid for cloud-init schema script
         expected_subcommands = ["schema"]
         error = self.stderr.getvalue()
         for subcommand in expected_subcommands:
             self.assertIn(subcommand, error)
 
-    def test_wb_devel_schema_subcommand_parser(self):
+    def test_wb_schema_subcommand_parser(self):
         """The subcommand cloud-init schema calls the correct subparser."""
-        exit_code = self._call_main(["cloud-init", "devel", "schema"])
+        exit_code = self._call_main(["cloud-init", "schema"])
         self.assertEqual(1, exit_code)
         # Known whitebox output from schema subcommand
         self.assertEqual(
@@ -240,7 +243,7 @@ def test_wb_devel_schema_subcommand_parser(self):
             self.stderr.getvalue(),
         )
 
-    def test_wb_devel_schema_subcommand_doc_all_spot_check(self):
+    def test_wb_schema_subcommand_doc_all_spot_check(self):
         """Validate that doc content has correct values from known examples.
 
         Ensure that schema doc is returned
@@ -252,7 +255,7 @@ def test_wb_devel_schema_subcommand_doc_all_spot_check(self):
         # manager
         stdout = io.StringIO()
         with contextlib.redirect_stdout(stdout):
-            self._call_main(["cloud-init", "devel", "schema", "--docs", "all"])
+            self._call_main(["cloud-init", "schema", "--docs", "all"])
             expected_doc_sections = [
                 "**Supported distros:** all",
                 "**Supported distros:** almalinux, alpine, centos, "
@@ -267,7 +270,7 @@ def test_wb_devel_schema_subcommand_doc_all_spot_check(self):
         for expected in expected_doc_sections:
             self.assertIn(expected, stdout)
 
-    def test_wb_devel_schema_subcommand_single_spot_check(self):
+    def test_wb_schema_subcommand_single_spot_check(self):
         """Validate that doc content has correct values from known example.
 
         Validate 'all' arg
@@ -279,17 +282,15 @@ def test_wb_devel_schema_subcommand_single_spot_check(self):
         # manager
         stdout = io.StringIO()
         with contextlib.redirect_stdout(stdout):
-            self._call_main(
-                ["cloud-init", "devel", "schema", "--docs", "cc_runcmd"]
-            )
+            self._call_main(["cloud-init", "schema", "--docs", "cc_runcmd"])
             expected_doc_sections = [
                 "Runcmd\n------\n**Summary:** Run arbitrary commands"
             ]
         stdout = stdout.getvalue()
         for expected in expected_doc_sections:
             self.assertIn(expected, stdout)
 
-    def test_wb_devel_schema_subcommand_multiple_spot_check(self):
+    def test_wb_schema_subcommand_multiple_spot_check(self):
         """Validate that doc content has correct values from known example.
 
         Validate single arg
@@ -300,7 +301,6 @@ def test_wb_devel_schema_subcommand_multiple_spot_check(self):
             self._call_main(
                 [
                     "cloud-init",
-                    "devel",
                     "schema",
                     "--docs",
                     "cc_runcmd",
@@ -315,7 +315,7 @@ def test_wb_devel_schema_subcommand_multiple_spot_check(self):
         for expected in expected_doc_sections:
             self.assertIn(expected, stdout)
 
-    def test_wb_devel_schema_subcommand_bad_arg_fails(self):
+    def test_wb_schema_subcommand_bad_arg_fails(self):
         """Validate that doc content has correct values from known example.
 
         Validate multiple args
@@ -328,7 +328,7 @@ def test_wb_devel_schema_subcommand_bad_arg_fails(self):
         stderr = io.StringIO()
         with contextlib.redirect_stderr(stderr):
             self._call_main(
-                ["cloud-init", "devel", "schema", "--docs", "garbage_value"]
+                ["cloud-init", "schema", "--docs", "garbage_value"]
             )
             expected_doc_sections = ["Invalid --docs value"]
         stderr = stderr.getvalue()