From 2070d4b8600c881dcd581f5d0962ef6184569f77 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Wed, 2 Jul 2025 18:07:46 +0200 Subject: [PATCH 1/6] config:spack:packages is a required entry --- stackinator/schema/config.json | 1 + 1 file changed, 1 insertion(+) diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index 49fae9b4..b91dce62 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -15,6 +15,7 @@ "spack" : { "type" : "object", "additionalProperties": false, + "required": ["packages"], "properties" : { "repo": { "type": "string" From 818c9e3978b5fc5da9878caf01287441d9c4fb6b Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Thu, 3 Jul 2025 14:56:15 +0200 Subject: [PATCH 2/6] switch to specific schema.*_validator in unittests + add version default check --- unittests/test_schema.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/unittests/test_schema.py b/unittests/test_schema.py index 9bc03264..07ed3742 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -40,7 +40,8 @@ def test_config_yaml(yaml_path): # test that the defaults are set as expected with open(yaml_path / "config.defaults.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.config_schema).validate(raw) + schema.config_validator.validate(raw) + assert raw["version"] == 1 assert raw["store"] == "/user-environment" assert raw["spack"]["commit"] is None assert raw["modules"] == True # noqa: E712 @@ -49,7 +50,8 @@ def test_config_yaml(yaml_path): with open(yaml_path / "config.full.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.config_schema).validate(raw) + schema.config_validator.validate(raw) + assert raw["version"] == 1 assert raw["store"] == "/alternative-point" assert raw["spack"]["commit"] == "6408b51" assert raw["modules"] == False # noqa: E712 @@ -62,21 +64,21 @@ def test_recipe_config_yaml(recipe_paths): for p in recipe_paths: with open(p / "config.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.config_schema).validate(raw) + schema.config_validator.validate(raw) def test_compilers_yaml(yaml_path): # test that the defaults are set as expected with open(yaml_path / "compilers.defaults.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.compilers_schema).validate(raw) + schema.compilers_validator.validate(raw) assert raw["bootstrap"] == {"spec": "gcc@11"} assert raw["gcc"] == {"specs": ["gcc@10.2"]} assert raw["llvm"] is None with open(yaml_path / "compilers.full.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.compilers_schema).validate(raw) + schema.compilers_validator.validate(raw) assert raw["bootstrap"]["spec"] == "gcc@11" assert raw["gcc"] == {"specs": ["gcc@11", "gcc@10.2", "gcc@9.3.0"]} assert raw["llvm"] == { @@ -90,13 +92,13 @@ def test_recipe_compilers_yaml(recipe_paths): for p in recipe_paths: with open(p / "compilers.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.compilers_schema).validate(raw) + schema.compilers_validator.validate(raw) def test_environments_yaml(yaml_path): with open(yaml_path / "environments.full.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.environments_schema).validate(raw) + schema.environments_validator.validate(raw) # the defaults-env does not set fields # test that they have been set to the defaults correctly @@ -145,7 +147,7 @@ def test_environments_yaml(yaml_path): jsonschema.exceptions.ValidationError, match=r"Additional properties are not allowed \('providers' was unexpected", ): - schema.validator(schema.environments_schema).validate(raw) + schema.environments_validator.validate(raw) def test_recipe_environments_yaml(recipe_paths): @@ -153,4 +155,4 @@ def test_recipe_environments_yaml(recipe_paths): for p in recipe_paths: with open(p / "environments.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) - schema.validator(schema.environments_schema).validate(raw) + schema.environments_validator.validate(raw) From 4442c05f4609dbd52f8de3cc9c961f4b2f29e49e Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Thu, 3 Jul 2025 14:57:04 +0200 Subject: [PATCH 3/6] switch config validator to draft-2019-09 + improve schema for unevaluatedProperties feature --- stackinator/schema.py | 2 +- stackinator/schema/config.json | 109 ++++++++++++++++++++++----------- 2 files changed, 74 insertions(+), 37 deletions(-) diff --git a/stackinator/schema.py b/stackinator/schema.py index 0f6379d6..90849d9a 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -45,7 +45,7 @@ def py2yaml(data, indent): # load recipe yaml schema config_schema = json.load(open(prefix / "schema/config.json")) -config_validator = validator(config_schema) +config_validator = extend_with_default(jsonschema.Draft201909Validator)(config_schema) compilers_schema = json.load(open(prefix / "schema/compilers.json")) compilers_validator = validator(compilers_schema) environments_schema = json.load(open(prefix / "schema/environments.json")) diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index b91dce62..16c3b39a 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -1,9 +1,9 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "http://json-schema.org/draft-2019-09/schema#", "title": "Schema for Spack Stack config.yaml recipe file", "type" : "object", - "additionalProperties": false, - "required": ["name", "spack"], + "unevaluatedProperties": false, + "required": ["name"], "properties" : { "name" : { "type": "string" @@ -12,39 +12,6 @@ "type" : "string", "default" : "/user-environment" }, - "spack" : { - "type" : "object", - "additionalProperties": false, - "required": ["packages"], - "properties" : { - "repo": { - "type": "string" - }, - "commit": { - "oneOf": [ - {"type" : "string"}, - {"type" : "null"} - ], - "default": null - }, - "packages" : { - "type" : "object", - "additionalProperties": false, - "properties" : { - "repo": { - "type": "string" - }, - "commit": { - "oneOf": [ - {"type" : "string"}, - {"type" : "null"} - ], - "default": null - } - } - } - } - }, "mirror" : { "type" : "object", "additionalProperties": false, @@ -78,5 +45,75 @@ "type": "number", "default": 1 } + }, + "allOf": [ + { + "if": { + "required": ["version"], + "properties": { + "version": {"const": 2 } + } + }, + "then": { + "required": ["spack"], + "properties": { + "spack": { + "$ref": "#/$defs/git-repo", + "unevaluatedProperties": false, + "required": ["packages"], + "properties": { + "packages": { + "$ref": "#/$defs/git-repo", + "unevaluatedProperties": false + } + } + } + } + } + }, + { + "if": { + "oneOf": [ + { + "not": { + "required": ["version"] + } + }, + { + "required": ["version"], + "properties": { + "version": {"const": 1 } + } + } + ] + }, + "then": { + "required": ["spack"], + "properties": { + "spack": { + "$ref": "#/$defs/git-repo", + "unevaluatedProperties": false + } + } + } + } + ], + "$defs": { + "git-repo": { + "type" : "object", + "required": ["repo"], + "properties" : { + "repo": { + "type": "string" + }, + "commit": { + "oneOf": [ + {"type" : "string"}, + {"type" : "null"} + ], + "default": null + } + } + } } } From 66c76832a341993e6045de3fda8f4a0b6ed04129 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Thu, 3 Jul 2025 15:21:52 +0200 Subject: [PATCH 4/6] basic test for config v2 defaults --- unittests/test_schema.py | 10 ++++++++++ unittests/yaml/config.defaults.v2.yaml | 13 +++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 unittests/yaml/config.defaults.v2.yaml diff --git a/unittests/test_schema.py b/unittests/test_schema.py index 07ed3742..4ba0e937 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -58,6 +58,16 @@ def test_config_yaml(yaml_path): assert raw["mirror"] == {"enable": True, "key": "/home/bob/veryprivate.key"} assert raw["description"] == "a really useful environment" + with open(yaml_path / "config.defaults.v2.yaml") as fid: + raw = yaml.load(fid, Loader=yaml.Loader) + schema.config_validator.validate(raw) + assert raw["store"] == "/user-environment" + assert raw["spack"]["commit"] is None + assert raw["spack"]["packages"]["commit"] is None + assert raw["modules"] == True # noqa: E712 + assert raw["mirror"] == {"enable": True, "key": None} + assert raw["description"] is None + def test_recipe_config_yaml(recipe_paths): # validate the config.yaml in the test recipes diff --git a/unittests/yaml/config.defaults.v2.yaml b/unittests/yaml/config.defaults.v2.yaml new file mode 100644 index 00000000..9720dcf0 --- /dev/null +++ b/unittests/yaml/config.defaults.v2.yaml @@ -0,0 +1,13 @@ +version: 2 +name: cuda-env +# store: /user-environment +spack: + repo: https://github.com/spack/spack.git + # commit: None + packages: + repo: https://github.com/spack/spack-packages.git + # commit: None +# mirror: + # key: None + # enable: True +# modules: True From 134b70d62966ef61d5c8d49fb2c7ab9d03ef61f9 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Thu, 3 Jul 2025 16:04:06 +0200 Subject: [PATCH 5/6] add version constraint (with spack property placeholder for better error message) --- stackinator/schema/config.json | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index 16c3b39a..f8a62c06 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -43,7 +43,9 @@ }, "version" : { "type": "number", - "default": 1 + "default": 1, + "minumum": 1, + "maximum": 2 } }, "allOf": [ @@ -96,6 +98,14 @@ } } } + }, + { + "if": { + "not": { "properties": { "version": { "minimum": 1, "maximum": 2} } } + }, + "then": { + "properties": { "spack": {} } + } } ], "$defs": { From b1d58444da7de075ddd7e0c4188fca69bee17cf1 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Fri, 4 Jul 2025 15:58:33 +0200 Subject: [PATCH 6/6] fix typo --- stackinator/schema/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index f8a62c06..99d7a2b8 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -44,7 +44,7 @@ "version" : { "type": "number", "default": 1, - "minumum": 1, + "minimum": 1, "maximum": 2 } },