From 28c5f256e6717174225d20172d761701bf0199be Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Mon, 25 Apr 2022 15:25:22 -0600 Subject: [PATCH 1/9] Promote schema to top level command --- cloudinit/cmd/devel/parser.py | 6 ------ cloudinit/cmd/main.py | 32 +++++++++++++++++++++++--------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/cloudinit/cmd/devel/parser.py b/cloudinit/cmd/devel/parser.py index 76b16c2eac7..61d9efea3dc 100644 --- a/cloudinit/cmd/devel/parser.py +++ b/cloudinit/cmd/devel/parser.py @@ -27,12 +27,6 @@ def get_parser(parser=None): hotplug_hook.get_parser, hotplug_hook.handle_args, ), - ( - "schema", - "Validate cloud-config files for document schema", - schema.get_parser, - schema.handle_schema_args, - ), ( net_convert.NAME, net_convert.__doc__, diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index c9be41b3496..afd0a8d81b6 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -836,8 +836,7 @@ def main_features(name, args): def main(sysv_args=None): if not sysv_args: sysv_args = sys.argv - parser = argparse.ArgumentParser(prog=sysv_args[0]) - sysv_args = sysv_args[1:] + parser = argparse.ArgumentParser(prog=sysv_args.pop(0)) # Top level args parser.add_argument( @@ -956,7 +955,9 @@ def main(sysv_args=None): "analyze", help="Devel tool: Analyze cloud-init logs and data" ) - parser_devel = subparsers.add_parser("devel", help="Run development tools") + parser_devel = subparsers.add_parser( + "devel", help="Run development tools." + ) parser_collect_logs = subparsers.add_parser( "collect-logs", help="Collect and tar all cloud-init debug info" @@ -970,19 +971,24 @@ def main(sysv_args=None): "status", help="Report cloud-init status or wait on completion." ) + parser_schema = subparsers.add_parser( + "schema", help="Validate cloud-config files using jsonschema." + ) + if sysv_args: # Only load subparsers if subcommand is specified to avoid load cost - if sysv_args[0] == "analyze": + subcommand = sysv_args[0] + if subcommand == "analyze": from cloudinit.analyze.__main__ import get_parser as analyze_parser # Construct analyze subcommand parser analyze_parser(parser_analyze) - elif sysv_args[0] == "devel": + elif subcommand == "devel": from cloudinit.cmd.devel.parser import get_parser as devel_parser # Construct devel subcommand parser devel_parser(parser_devel) - elif sysv_args[0] == "collect-logs": + elif subcommand == "collect-logs": from cloudinit.cmd.devel.logs import ( get_parser as logs_parser, handle_collect_logs_args, @@ -992,7 +998,7 @@ def main(sysv_args=None): parser_collect_logs.set_defaults( action=("collect-logs", handle_collect_logs_args) ) - elif sysv_args[0] == "clean": + elif subcommand == "clean": from cloudinit.cmd.clean import ( get_parser as clean_parser, handle_clean_args, @@ -1000,7 +1006,7 @@ def main(sysv_args=None): clean_parser(parser_clean) parser_clean.set_defaults(action=("clean", handle_clean_args)) - elif sysv_args[0] == "query": + elif subcommand == "query": from cloudinit.cmd.query import ( get_parser as query_parser, handle_args as handle_query_args, @@ -1008,7 +1014,15 @@ def main(sysv_args=None): query_parser(parser_query) parser_query.set_defaults(action=("render", handle_query_args)) - elif sysv_args[0] == "status": + elif subcommand == "schema": + from cloudinit.config.schema import ( + get_parser as schema_parser, + handle_schema_args, + ) + + schema_parser(parser_schema) + parser_schema.set_defaults(action=("schema", handle_schema_args)) + elif subcommand == "status": from cloudinit.cmd.status import ( get_parser as status_parser, handle_status_args, From 973d0b102023607c564e8b5cc549de03b413546b Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Mon, 25 Apr 2022 16:29:16 -0600 Subject: [PATCH 2/9] Add schema to cli interface doc on RTD --- doc/rtd/topics/cli.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/doc/rtd/topics/cli.rst b/doc/rtd/topics/cli.rst index e90706fcab4..03a2ef177f7 100644 --- a/doc/rtd/topics/cli.rst +++ b/doc/rtd/topics/cli.rst @@ -249,6 +249,29 @@ This data can then be formatted to generate custom strings or data: custom-i-0e91f69987f37ec74.us-east-2.aws.com +.. _cli_schema: + +schema +====== + +Validate cloud-config files using jsonschema. + +* ``-h, --help``: show this help message and exit +* ``-c CONFIG_FILE, --config-file CONFIG_FILE``: Path of the cloud-config yaml + file to validate +* ``--system``: Validate the system cloud-config userdata +* ``-d DOCS [DOCS ...], --docs DOCS [DOCS ...]``: Print schema module docs. + Choices: all or space-delimited cc_names. +* ``--annotate``: Annotate existing cloud-config file with errors + +The following example checks a config file and annotates the config file with +errors on stdout. + +.. code-block:: shell-session + + $ cloud-init schema -c ./config.yml --annotate + + .. _cli_single: single From ffbad71926e10ac7e973f10e6608484f5aa18832 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 26 Apr 2022 12:43:15 -0600 Subject: [PATCH 3/9] Add schema subcommand to man page --- doc/man/cloud-init.1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/man/cloud-init.1 b/doc/man/cloud-init.1 index 2cb6313598a..1da4335b6a4 100644 --- a/doc/man/cloud-init.1 +++ b/doc/man/cloud-init.1 @@ -78,6 +78,10 @@ Activates modules using a given configuration key. .B "query" Query standardized instance metadata from the command line. +.TP +.B "schema" +Validate cloud-config files using jsonschema. + .TP .B "single" Run a single module. From 5e40866992a354022ad27b19405e2781a3369f59 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 26 Apr 2022 13:17:02 -0600 Subject: [PATCH 4/9] Update tests --- tests/integration_tests/modules/test_cli.py | 8 +++--- tests/unittests/test_cli.py | 28 +++++++++++---------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/tests/integration_tests/modules/test_cli.py b/tests/integration_tests/modules/test_cli.py index baaa75678bf..e878176f465 100644 --- a/tests/integration_tests/modules/test_cli.py +++ b/tests/integration_tests/modules/test_cli.py @@ -28,11 +28,11 @@ @pytest.mark.user_data(VALID_USER_DATA) def test_valid_userdata(client: IntegrationInstance): - """Test `cloud-init devel schema` with valid userdata. + """Test `cloud-init schema` with valid userdata. PR #575 """ - result = client.execute("cloud-init devel schema --system") + result = client.execute("cloud-init schema --system") assert result.ok assert "Valid cloud-config: system userdata" == result.stdout.strip() result = client.execute("cloud-init status --long") @@ -44,11 +44,11 @@ def test_valid_userdata(client: IntegrationInstance): @pytest.mark.user_data(INVALID_USER_DATA_HEADER) def test_invalid_userdata(client: IntegrationInstance): - """Test `cloud-init devel schema` with invalid userdata. + """Test `cloud-init schema` with invalid userdata. PR #575 """ - result = client.execute("cloud-init devel schema --system") + result = client.execute("cloud-init schema --system") assert not result.ok assert "Cloud config schema errors" in result.stderr assert 'needs to begin with "#cloud-config"' in result.stderr diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index d2de9c8776a..fe8e4599e21 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -134,6 +134,7 @@ def test_all_subcommands_represented_in_help(self): "init", "modules", "single", + "schema", ] for subcommand in expected_subcommands: self.assertIn(subcommand, error) @@ -169,6 +170,7 @@ def test_conditional_subcommands_from_entry_point_sys_argv(self): "usage: cloud-init collect-logs", "usage: cloud-init devel", "usage: cloud-init status", + "usage: cloud-init schema", ] conditional_subcommands = [ "analyze", @@ -176,6 +178,7 @@ def test_conditional_subcommands_from_entry_point_sys_argv(self): "collect-logs", "devel", "status", + "schema", ] # The cloud-init entrypoint calls main without passing sys_argv for subcommand in conditional_subcommands: @@ -220,18 +223,18 @@ def test_status_subcommand_parser(self): self._call_main(["cloud-init", "status", "-h"]) self.assertIn("usage: cloud-init status", stdout.getvalue()) - def test_devel_subcommand_parser(self): - """The subcommand cloud-init devel calls the correct subparser.""" - self._call_main(["cloud-init", "devel"]) + def test_subcommand_parser(self): + """The subcommand cloud-init schema calls the correct subparser.""" + self._call_main(["cloud-init"]) # These subcommands only valid for cloud-init schema script expected_subcommands = ["schema"] error = self.stderr.getvalue() for subcommand in expected_subcommands: self.assertIn(subcommand, error) - def test_wb_devel_schema_subcommand_parser(self): + def test_wb_schema_subcommand_parser(self): """The subcommand cloud-init schema calls the correct subparser.""" - exit_code = self._call_main(["cloud-init", "devel", "schema"]) + exit_code = self._call_main(["cloud-init", "schema"]) self.assertEqual(1, exit_code) # Known whitebox output from schema subcommand self.assertEqual( @@ -240,7 +243,7 @@ def test_wb_devel_schema_subcommand_parser(self): self.stderr.getvalue(), ) - def test_wb_devel_schema_subcommand_doc_all_spot_check(self): + def test_wb_schema_subcommand_doc_all_spot_check(self): """Validate that doc content has correct values from known examples. Ensure that schema doc is returned @@ -252,7 +255,7 @@ def test_wb_devel_schema_subcommand_doc_all_spot_check(self): # manager stdout = io.StringIO() with contextlib.redirect_stdout(stdout): - self._call_main(["cloud-init", "devel", "schema", "--docs", "all"]) + self._call_main(["cloud-init", "schema", "--docs", "all"]) expected_doc_sections = [ "**Supported distros:** all", "**Supported distros:** almalinux, alpine, centos, " @@ -267,7 +270,7 @@ def test_wb_devel_schema_subcommand_doc_all_spot_check(self): for expected in expected_doc_sections: self.assertIn(expected, stdout) - def test_wb_devel_schema_subcommand_single_spot_check(self): + def test_wb_schema_subcommand_single_spot_check(self): """Validate that doc content has correct values from known example. Validate 'all' arg @@ -280,7 +283,7 @@ def test_wb_devel_schema_subcommand_single_spot_check(self): stdout = io.StringIO() with contextlib.redirect_stdout(stdout): self._call_main( - ["cloud-init", "devel", "schema", "--docs", "cc_runcmd"] + ["cloud-init", "schema", "--docs", "cc_runcmd"] ) expected_doc_sections = [ "Runcmd\n------\n**Summary:** Run arbitrary commands" @@ -289,7 +292,7 @@ def test_wb_devel_schema_subcommand_single_spot_check(self): for expected in expected_doc_sections: self.assertIn(expected, stdout) - def test_wb_devel_schema_subcommand_multiple_spot_check(self): + def test_wb_schema_subcommand_multiple_spot_check(self): """Validate that doc content has correct values from known example. Validate single arg @@ -300,7 +303,6 @@ def test_wb_devel_schema_subcommand_multiple_spot_check(self): self._call_main( [ "cloud-init", - "devel", "schema", "--docs", "cc_runcmd", @@ -315,7 +317,7 @@ def test_wb_devel_schema_subcommand_multiple_spot_check(self): for expected in expected_doc_sections: self.assertIn(expected, stdout) - def test_wb_devel_schema_subcommand_bad_arg_fails(self): + def test_wb_schema_subcommand_bad_arg_fails(self): """Validate that doc content has correct values from known example. Validate multiple args @@ -328,7 +330,7 @@ def test_wb_devel_schema_subcommand_bad_arg_fails(self): stderr = io.StringIO() with contextlib.redirect_stderr(stderr): self._call_main( - ["cloud-init", "devel", "schema", "--docs", "garbage_value"] + ["cloud-init", "schema", "--docs", "garbage_value"] ) expected_doc_sections = ["Invalid --docs value"] stderr = stderr.getvalue() From 7d635d46423c9cdefeab408c9865f14a6fabdfd9 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 26 Apr 2022 13:17:53 -0600 Subject: [PATCH 5/9] Update bash completion --- bash_completion/cloud-init | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bash_completion/cloud-init b/bash_completion/cloud-init index b9f137b13e6..1eceb472b61 100644 --- a/bash_completion/cloud-init +++ b/bash_completion/cloud-init @@ -45,6 +45,9 @@ _cloudinit_complete() query) COMPREPLY=($(compgen -W "--all --help --instance-data --list-keys --user-data --vendor-data --debug" -- $cur_word));; + schema) + COMPREPLY=($(compgen -W "--help --config-file --docs --annotate --system" -- $cur_word)) + ;; single) COMPREPLY=($(compgen -W "--help --name --frequency --report" -- $cur_word)) ;; @@ -72,9 +75,6 @@ _cloudinit_complete() ;; render) COMPREPLY=($(compgen -W "--help --instance-data --debug" -- $cur_word));; - schema) - COMPREPLY=($(compgen -W "--help --config-file --doc --annotate" -- $cur_word)) - ;; show) COMPREPLY=($(compgen -W "--help --format --infile --outfile" -- $cur_word)) ;; From 2e922a1a1f1e85cdcd1e936db481dd7a1f8b9542 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 26 Apr 2022 13:18:03 -0600 Subject: [PATCH 6/9] Update docs --- doc/rtd/topics/cli.rst | 7 ++----- doc/rtd/topics/faq.rst | 2 +- doc/rtd/topics/tutorial.rst | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/doc/rtd/topics/cli.rst b/doc/rtd/topics/cli.rst index 03a2ef177f7..542091fa938 100644 --- a/doc/rtd/topics/cli.rst +++ b/doc/rtd/topics/cli.rst @@ -36,6 +36,8 @@ option. This can be used against cloud-init itself or any of its subcommands. collect-logs Collect and tar all cloud-init debug info clean Remove logs and artifacts so cloud-init can re-run. status Report cloud-init status or wait on completion. + schema Validate cloud-config files using jsonschema. + The rest of this document will give an overview of each of the subcommands. @@ -114,11 +116,6 @@ Current subcommands: from ``/run/cloud-init/instance-data.json``. It accepts a user-data file containing the jinja template header ``## template: jinja`` and renders that content with any instance-data.json variables present. - * ``schema``: a **#cloud-config** format and schema - validator. It accepts a cloud-config YAML file and annotates potential - schema errors locally without the need for deployment. Schema - validation is work in progress and supports a subset of cloud-config - modules. * ``hotplug-hook``: respond to newly added system devices by retrieving updated system metadata and bringing up/down the corresponding device. This command is intended to be called via a systemd service and is diff --git a/doc/rtd/topics/faq.rst b/doc/rtd/topics/faq.rst index 59138c1dc7d..e5784b717ac 100644 --- a/doc/rtd/topics/faq.rst +++ b/doc/rtd/topics/faq.rst @@ -146,7 +146,7 @@ provided to the system: .. code-block:: shell-session - $ cloud-init devel schema --system --annotate + $ cloud-init schema --system --annotate As launching instances in the cloud can cost money and take a bit longer, sometimes it is easier to launch instances locally using Multipass or LXD: diff --git a/doc/rtd/topics/tutorial.rst b/doc/rtd/topics/tutorial.rst index ad04bbc547c..07b8fe5d11b 100644 --- a/doc/rtd/topics/tutorial.rst +++ b/doc/rtd/topics/tutorial.rst @@ -95,7 +95,7 @@ We can also assert the user data we provided is a valid cloud-config: .. code-block:: shell-session - $ cloud-init devel schema --system --annotate + $ cloud-init schema --system --annotate Valid cloud-config: system userdata $ From 7e8d93eb755107d7c7d6b4ce3f3c079a615c35a6 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 26 Apr 2022 13:34:28 -0600 Subject: [PATCH 7/9] fmt --- tests/unittests/test_cli.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index fe8e4599e21..7846d0d3341 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -282,9 +282,7 @@ def test_wb_schema_subcommand_single_spot_check(self): # manager stdout = io.StringIO() with contextlib.redirect_stdout(stdout): - self._call_main( - ["cloud-init", "schema", "--docs", "cc_runcmd"] - ) + self._call_main(["cloud-init", "schema", "--docs", "cc_runcmd"]) expected_doc_sections = [ "Runcmd\n------\n**Summary:** Run arbitrary commands" ] From fbc4ddb7f510bbb5d8c096c52a8fc080f10cd2cb Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 26 Apr 2022 14:12:54 -0600 Subject: [PATCH 8/9] remove unused import, silence false positive spelling warning --- Makefile | 3 ++- cloudinit/cmd/devel/parser.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5f0c5b1bdb8..5709a4cb08d 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,8 @@ _CHECK_SPELLING := find doc -type f -exec spellintian {} + | \ grep -v -e 'doc/rtd/topics/cli.rst: modules modules' \ -e 'doc/examples/cloud-config-mcollective.txt: WARNING WARNING' \ -e 'doc/examples/cloud-config-power-state.txt: Bye Bye' \ - -e 'doc/examples/cloud-config.txt: Bye Bye' + -e 'doc/examples/cloud-config.txt: Bye Bye' \ + -e 'doc/rtd/topics/cli.rst: DOCS DOCS' # For CI we require a failing return code when spellintian finds spelling errors diff --git a/cloudinit/cmd/devel/parser.py b/cloudinit/cmd/devel/parser.py index 61d9efea3dc..460b94b3431 100644 --- a/cloudinit/cmd/devel/parser.py +++ b/cloudinit/cmd/devel/parser.py @@ -6,8 +6,6 @@ import argparse -from cloudinit.config import schema - from . import hotplug_hook, make_mime, net_convert, render From 6358ccaf1d36a61c8f3e39ba0346c76e8345dbd9 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Wed, 27 Apr 2022 08:32:55 -0600 Subject: [PATCH 9/9] Whitespace test bump --- doc/rtd/topics/cli.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/rtd/topics/cli.rst b/doc/rtd/topics/cli.rst index 542091fa938..4a26cb45234 100644 --- a/doc/rtd/topics/cli.rst +++ b/doc/rtd/topics/cli.rst @@ -38,7 +38,6 @@ option. This can be used against cloud-init itself or any of its subcommands. status Report cloud-init status or wait on completion. schema Validate cloud-config files using jsonschema. - The rest of this document will give an overview of each of the subcommands.