diff --git a/stackinator/cache.py b/stackinator/cache.py index ddafb981..afeba5e2 100644 --- a/stackinator/cache.py +++ b/stackinator/cache.py @@ -12,7 +12,7 @@ def configuration_from_file(file, mount): raw = yaml.load(fid, Loader=yaml.Loader) # validate the yaml - schema.cache_validator.validate(raw) + schema.CacheValidator.validate(raw) # verify that the root path exists path = pathlib.Path(os.path.expandvars(raw["root"])) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 2e997983..473cd2e9 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -51,28 +51,6 @@ def __init__(self, args): # required config.yaml file self.config = self.path / "config.yaml" - # check the version of the recipe - if self.config["version"] != 2: - rversion = self.config["version"] - if rversion == 1: - self._logger.error( - "\nThe recipe is an old version 1 recipe for Spack v0.23 and earlier.\n" - "This version of Stackinator supports Spack 1.0, and has deprecated support for Spack v0.23.\n" - "Use version 5 of stackinator, which can be accessed via the releases/v5 branch:\n" - " git switch releases/v5\n\n" - "If this recipe is to be used with Spack 1.0, then please add the field 'version: 2' to\n" - "config.yaml in your recipe.\n\n" - "For more information: https://eth-cscs.github.io/stackinator/recipes/#configuration\n" - ) - raise RuntimeError("incompatible uenv recipe version") - else: - self._logger.error( - f"\nThe config.yaml file sets an unknown recipe version={rversion}.\n" - "This version of Stackinator supports version 2 recipes.\n\n" - "For more information: https://eth-cscs.github.io/stackinator/recipes/#configuration\n" - ) - raise RuntimeError("incompatible uenv recipe version") - # override the mount point if defined as a CLI argument if args.mount: self.config["store"] = args.mount @@ -89,7 +67,7 @@ def __init__(self, args): with compiler_path.open() as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.compilers_validator.validate(raw) + schema.CompilersValidator.validate(raw) self.generate_compiler_specs(raw) # required environments.yaml file @@ -110,7 +88,7 @@ def __init__(self, args): "specs": ["squashfs"], "views": {}, } - schema.environments_validator.validate(raw) + schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) # optional modules.yaml file @@ -269,7 +247,7 @@ def config(self, config_path): with config_path.open() as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.config_validator.validate(raw) + schema.ConfigValidator.validate(raw) self._config = raw # In Stackinator 6 we replaced logic required to determine the diff --git a/stackinator/schema.py b/stackinator/schema.py index 0f6379d6..4e981900 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -1,54 +1,114 @@ import json import pathlib +from textwrap import dedent import jsonschema import yaml +from . import root_logger + prefix = pathlib.Path(__file__).parent.resolve() -# create a validator that will insert optional fields with their default values -# if they have not been provided. + +def py2yaml(data, indent): + dump = yaml.dump(data) + lines = [ln for ln in dump.split("\n") if ln != ""] + res = ("\n" + " " * indent).join(lines) + return res -def extend_with_default(validator_class): - validate_properties = validator_class.VALIDATORS["properties"] +def validator(schema): + """ + Create a new validator class that will insert optional fields with their default values + if they have not been provided. + """ - def set_defaults(validator, properties, instance, schema): - # if instance is none, it's not possible to set any default for any sub-property - if instance is not None: - for property, subschema in properties.items(): - if "default" in subschema: - instance.setdefault(property, subschema["default"]) + def extend_with_default(validator_class): + validate_properties = validator_class.VALIDATORS["properties"] - for error in validate_properties( - validator, - properties, - instance, - schema, - ): - yield error + def set_defaults(validator, properties, instance, schema): + # if instance is none, it's not possible to set any default for any sub-property + if instance is not None: + for property, subschema in properties.items(): + if "default" in subschema: + instance.setdefault(property, subschema["default"]) - return jsonschema.validators.extend( - validator_class, - {"properties": set_defaults}, - ) + for error in validate_properties( + validator, + properties, + instance, + schema, + ): + yield error + return jsonschema.validators.extend( + validator_class, + {"properties": set_defaults}, + ) -def py2yaml(data, indent): - dump = yaml.dump(data) - lines = [ln for ln in dump.split("\n") if ln != ""] - res = ("\n" + " " * indent).join(lines) - return res + # try to read dialect metaschema from the $schema entry, otherwise fallback to a default one. + metaschema = jsonschema.validators.validator_for(schema) + + return extend_with_default(metaschema)(schema) + + +class ValidationError(jsonschema.ValidationError): + def __init__(self, name: str, errors: list[jsonschema.ValidationError]): + assert len(errors) != 0 + messages = [ + f"- Failed validating '{error.validator}' in {error.json_path} : {error.message}" for error in errors + ] + message = f"ValidationError in '{name}'\n" + message += "\n".join(messages) + super().__init__(message) + + +class SchemaValidator: + def __init__(self, schema_filepath: pathlib.Path, precheck=None): + self._validator = validator(json.load(open(schema_filepath))) + self._precheck = precheck + + def validate(self, instance: dict): + if self._precheck: + self._precheck(instance) + + errors = [error for error in self._validator.iter_errors(instance)] + + if len(errors) != 0: + raise ValidationError(self._validator.schema.get("title", "no-title"), errors) + + +def check_config_version(instance): + rversion = instance.get("version", 1) + if rversion != 2: + if rversion == 1: + root_logger.error( + dedent(""" + The recipe is an old version 1 recipe for Spack v0.23 and earlier. + This version of Stackinator supports Spack 1.0, and has deprecated support for Spack v0.23. + Use version 5 of stackinator, which can be accessed via the releases/v5 branch: + git switch releases/v5 + + If this recipe is to be used with Spack 1.0, then please add the field 'version: 2' to + config.yaml in your recipe. + + For more information: https://eth-cscs.github.io/stackinator/recipes/#configuration + """) + ) + raise RuntimeError("incompatible uenv recipe version") + else: + root_logger.error( + dedent(f""" + The config.yaml file sets an unknown recipe version={rversion}. + This version of Stackinator supports version 2 recipes. + For more information: https://eth-cscs.github.io/stackinator/recipes/#configuration + """) + ) + raise RuntimeError("incompatible uenv recipe version") -validator = extend_with_default(jsonschema.Draft7Validator) -# load recipe yaml schema -config_schema = json.load(open(prefix / "schema/config.json")) -config_validator = validator(config_schema) -compilers_schema = json.load(open(prefix / "schema/compilers.json")) -compilers_validator = validator(compilers_schema) -environments_schema = json.load(open(prefix / "schema/environments.json")) -environments_validator = validator(environments_schema) -cache_schema = json.load(open(prefix / "schema/cache.json")) -cache_validator = validator(cache_schema) +ConfigValidator = SchemaValidator(prefix / "schema/config.json", check_config_version) +CompilersValidator = SchemaValidator(prefix / "schema/compilers.json") +EnvironmentsValidator = SchemaValidator(prefix / "schema/environments.json") +CacheValidator = SchemaValidator(prefix / "schema/cache.json") diff --git a/unittests/recipes/base-nvgpu/config.yaml b/unittests/recipes/base-nvgpu/config.yaml index 0b6d54c1..a2b14813 100644 --- a/unittests/recipes/base-nvgpu/config.yaml +++ b/unittests/recipes/base-nvgpu/config.yaml @@ -5,3 +5,4 @@ spack: commit: 6408b51 mirror: enable: false +version: 2 diff --git a/unittests/recipes/cache/config.yaml b/unittests/recipes/cache/config.yaml index 175b89b4..ee25a292 100644 --- a/unittests/recipes/cache/config.yaml +++ b/unittests/recipes/cache/config.yaml @@ -6,3 +6,4 @@ spack: mirror: key: /scratch/e1000/bcumming/secret/spack-key.gpg enable: true +version: 2 diff --git a/unittests/recipes/host-recipe/config.yaml b/unittests/recipes/host-recipe/config.yaml index e108b998..30b15889 100644 --- a/unittests/recipes/host-recipe/config.yaml +++ b/unittests/recipes/host-recipe/config.yaml @@ -4,4 +4,4 @@ description: "An example gcc configuration for CPU-only development" spack: commit: releases/v0.23 repo: https://github.com/spack/spack.git - +version: 2 diff --git a/unittests/recipes/with-repo/config.yaml b/unittests/recipes/with-repo/config.yaml index 6f00c077..89c9dcac 100644 --- a/unittests/recipes/with-repo/config.yaml +++ b/unittests/recipes/with-repo/config.yaml @@ -3,3 +3,4 @@ store: '/user-environment' spack: repo: https://github.com/spack/spack.git commit: v21.0 +version: 2 diff --git a/unittests/test_schema.py b/unittests/test_schema.py index da1f8ed8..6850e5fe 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 import pathlib +from textwrap import dedent import jsonschema import pytest @@ -38,7 +39,7 @@ def test_config_yaml(yaml_path): # test that the defaults are set as expected with open(yaml_path / "config.defaults.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.config_schema).validate(raw) + schema.ConfigValidator.validate(raw) assert raw["store"] == "/user-environment" assert raw["spack"]["commit"] is None assert raw["modules"] == True # noqa: E712 @@ -47,33 +48,43 @@ def test_config_yaml(yaml_path): with open(yaml_path / "config.full.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.config_schema).validate(raw) + schema.ConfigValidator.validate(raw) assert raw["store"] == "/alternative-point" assert raw["spack"]["commit"] == "6408b51" assert raw["modules"] == False # noqa: E712 assert raw["mirror"] == {"enable": True, "key": "/home/bob/veryprivate.key"} assert raw["description"] == "a really useful environment" + # unsupported old version + with pytest.raises(RuntimeError, match="incompatible uenv recipe version"): + config = dedent(""" + name: cuda-env + spack: + repo: https://github.com/spack/spack.git + """) + raw = yaml.load(config, Loader=yaml.Loader) + schema.ConfigValidator.validate(raw) + def test_recipe_config_yaml(recipe_paths): # validate the config.yaml in the test recipes for p in recipe_paths: with open(p / "config.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.config_schema).validate(raw) + schema.ConfigValidator.validate(raw) def test_compilers_yaml(yaml_path): # test that the defaults are set as expected with open(yaml_path / "compilers.defaults.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.compilers_schema).validate(raw) + schema.CompilersValidator.validate(raw) assert raw["gcc"] == {"version": "10.2"} assert raw["llvm"] is None with open(yaml_path / "compilers.full.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.compilers_schema).validate(raw) + schema.CompilersValidator.validate(raw) assert raw["gcc"] == {"version": "11"} assert raw["llvm"] == {"version": "13"} assert raw["nvhpc"] == {"version": "25.1"} @@ -84,13 +95,13 @@ def test_recipe_compilers_yaml(recipe_paths): for p in recipe_paths: with open(p / "compilers.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.compilers_schema).validate(raw) + schema.CompilersValidator.validate(raw) def test_environments_yaml(yaml_path): with open(yaml_path / "environments.full.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.environments_schema).validate(raw) + schema.EnvironmentsValidator.validate(raw) # the defaults-env does not set fields # test that they have been set to the defaults correctly @@ -136,7 +147,7 @@ def test_environments_yaml(yaml_path): jsonschema.exceptions.ValidationError, match=r"Additional properties are not allowed \('providers' was unexpected", ): - schema.validator(schema.environments_schema).validate(raw) + schema.EnvironmentsValidator.validate(raw) def test_recipe_environments_yaml(recipe_paths): @@ -144,4 +155,4 @@ def test_recipe_environments_yaml(recipe_paths): for p in recipe_paths: with open(p / "environments.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.environments_schema).validate(raw) + schema.EnvironmentsValidator.validate(raw) diff --git a/unittests/yaml/config.defaults.yaml b/unittests/yaml/config.defaults.yaml index 4f90b1ea..0571b297 100644 --- a/unittests/yaml/config.defaults.yaml +++ b/unittests/yaml/config.defaults.yaml @@ -12,3 +12,4 @@ spack: #enable: True # default True #modules: True +version: 2 diff --git a/unittests/yaml/config.full.yaml b/unittests/yaml/config.full.yaml index fc9fa369..26267c6d 100644 --- a/unittests/yaml/config.full.yaml +++ b/unittests/yaml/config.full.yaml @@ -8,3 +8,4 @@ mirror: enable: True modules: False description: "a really useful environment" +version: 2