From 997cafccd13be6e901db1731b34fd8979d7d8db6 Mon Sep 17 00:00:00 2001 From: Suejung Shin Date: Thu, 27 Mar 2025 01:30:39 -0700 Subject: [PATCH 01/10] fix: Fix yaml validation for key named "type" --- shared/validation/user_schema.py | 15 +- shared/validation/validator.py | 6 + shared/yaml/validation.py | 12 +- tests/unit/validation/test_validation.py | 2849 +++++++++++----------- 4 files changed, 1489 insertions(+), 1393 deletions(-) diff --git a/shared/validation/user_schema.py b/shared/validation/user_schema.py index 35e270e96..73f8d37bd 100644 --- a/shared/validation/user_schema.py +++ b/shared/validation/user_schema.py @@ -78,6 +78,7 @@ custom_status_common_config = { "name_prefix": {"type": "string", "regex": r"^[\w\-\.]+$"}, + # TODO - problem spot here "type": {"type": "string", "allowed": ("project", "patch", "changes")}, "target": percent_type_or_auto, "threshold": percent_type, @@ -133,9 +134,17 @@ } component_rule_basic_properties = { + # TODO - here "statuses": { - "type": "list", - "schema": {"type": "dict", "schema": component_status_attributes}, + "anyof": [ + { + "type": "list", + "schema": { + "type": "dict", + "schema": component_status_attributes, + } + } + ] }, "flag_regexes": {"type": "list", "schema": {"type": "string"}}, "paths": path_list_structure, @@ -548,6 +557,7 @@ "default_rules": { "type": "dict", "schema": component_rule_basic_properties, + # "return_error_when_dict": True, }, "individual_components": { "type": "list", @@ -558,6 +568,7 @@ "name": {"type": "string"}, "component_id": {"type": "string", "required": True}, }, + # "return_error_when_dict": True, }, }, }, diff --git a/shared/validation/validator.py b/shared/validation/validator.py index 69c5ccf43..c7c372b84 100644 --- a/shared/validation/validator.py +++ b/shared/validation/validator.py @@ -60,3 +60,9 @@ def _validate_comma_separated_strings(self, constraint, field, value): LayoutStructure().validate(value) except Invalid as exc: self._error(field, exc.error_message) + + # def _validate_return_error_when_dict(self, constraint, field, value): + # """{'type': 'boolean'}""" + + # print("field", field) + # self._error(field, "cannot be a dict") diff --git a/shared/yaml/validation.py b/shared/yaml/validation.py index 42631c30c..c92a70c73 100644 --- a/shared/yaml/validation.py +++ b/shared/yaml/validation.py @@ -159,7 +159,17 @@ def decode(cls, value, expected_prefix): def do_actual_validation(yaml_dict, show_secrets_for): validator = CodecovUserYamlValidator(show_secrets_for=show_secrets_for) full_schema = {**user_schema, **cli_schema} - is_valid = validator.validate(yaml_dict, full_schema) + + try: + is_valid = validator.validate(yaml_dict, full_schema) + except Exception as e: + print("hello") + print(e) + + print(validator._errors) + raise e + + if not is_valid: error_dict = validator.errors ( diff --git a/tests/unit/validation/test_validation.py b/tests/unit/validation/test_validation.py index 75c2fec0d..de0f43a51 100644 --- a/tests/unit/validation/test_validation.py +++ b/tests/unit/validation/test_validation.py @@ -7,481 +7,931 @@ from shared.validation.exceptions import InvalidYamlException from shared.yaml.validation import ( _calculate_error_location_and_message_from_error_dict, - do_actual_validation, - validate_yaml, -) + do_actual_validation, validate_yaml) from tests.base import BaseTestCase class TestUserYamlValidation(BaseTestCase): - def test_empty_case(self): - user_input = {} - expected_result = {} - assert validate_yaml(user_input) == expected_result - - @pytest.mark.parametrize("input_value", ["", 10, [], tuple(), set()]) - def test_wrong_object_type(self, input_value): - with pytest.raises(InvalidYamlException) as exc: - validate_yaml(input_value) - exception = exc.value - assert exception.error_location == [] - assert exception.error_message == "Yaml needs to be a dict" - assert exception.original_exc is None - - @pytest.mark.parametrize( - "user_input, expected_result", - [ - ( - { - "coverage": {"status": {"patch": True, "project": False}}, - "comment": False, - }, - { - "comment": False, - "coverage": {"status": {"patch": True, "project": False}}, - }, - ), - ( - { - "codecov": {"bot": "codecov-io", "require_ci_to_pass": False}, - "coverage": { - "status": { - "project": { - "default": {"target": "78%", "threshold": "5%"} - }, - "patch": {"default": {"target": "75%"}}, - } - }, - }, - { - "codecov": {"bot": "codecov-io", "require_ci_to_pass": False}, - "coverage": { - "status": { - "project": {"default": {"target": 78.0, "threshold": 5.0}}, - "patch": {"default": {"target": 75.0}}, - } - }, - }, - ), - ( - { - "coverage": { - "status": { - "project": False, - "patch": { - "default": {"informational": True}, - "ui": {"informational": True}, - }, - "changes": False, - } - }, - "comment": False, - "flags": {"ui": {"paths": ["/ui-v2/"]}}, - "github_checks": {"annotations": False}, - "ignore": [ - "agent/uiserver/bindata_assetfs.go", - "vendor/**/*", - "**/*.pb.go", - ], - }, - { - "coverage": { - "status": { - "project": False, - "patch": { - "default": {"informational": True}, - "ui": {"informational": True}, - }, - "changes": False, - } - }, - "comment": False, - "flags": {"ui": {"paths": ["^/ui-v2/.*"]}}, - "github_checks": {"annotations": False}, - "ignore": [ - "^agent/uiserver/bindata_assetfs.go.*", - "(?s:vendor/.*/[^\\/]*)\\Z", - "(?s:.*/[^\\/]*\\.pb\\.go)\\Z", - ], - }, - ), - ( - { - "comment": { - "require_head": True, - "require_base": True, - "layout": "diff", - "require_changes": True, - "branches": ["main"], - "behavior": "once", - "after_n_builds": 6, - "hide_project_coverage": True, - }, - "coverage": { - "status": { - "project": {"default": {"threshold": "1%"}}, - "patch": False, - } - }, - "github_checks": {"annotations": False}, - "fixes": [ - "/opt/conda/lib/python3.8/site-packages/::project/", - "C:/Users/circleci/project/build/win_tmp/build/::project/", - ], - "ignore": ["coffee", "party_man", "test"], - "codecov": {"notify": {"after_n_builds": 6}}, - }, - { - "comment": { - "require_head": True, - "require_base": True, - "layout": "diff", - "require_changes": [0b001], - "branches": ["^main$"], - "behavior": "once", - "after_n_builds": 6, - "hide_project_coverage": True, - }, - "coverage": { - "status": { - "project": {"default": {"threshold": 1}}, - "patch": False, - } - }, - "github_checks": {"annotations": False}, - "fixes": [ - "^/opt/conda/lib/python3.8/site-packages/::project/", - "^C:/Users/circleci/project/build/win_tmp/build/::project/", - ], - "ignore": ["^coffee.*", "^party_man.*", "^test.*"], - "codecov": {"notify": {"after_n_builds": 6}}, - }, - ), - ( - { - "ignore": ["js/plugins", "plugins"], - "coverage": { - "notify": { - "slack": { - "default": { - "url": "https://hooks.slack.com/services/testdazzd/testrf4k6py/test72nq0j0ke3prs2fdvfuj", - "only_pulls": False, - "branches": ["main", "qa", "dev"], - } - } - } - }, - }, - { - "ignore": ["^js/plugins.*", "^plugins.*"], - "coverage": { - "notify": { - "slack": { - "default": { - "url": "https://hooks.slack.com/services/testdazzd/testrf4k6py/test72nq0j0ke3prs2fdvfuj", - "only_pulls": False, - "branches": ["^main$", "^qa$", "^dev$"], - } - } - } - }, - }, - ), - ( - { - "codecov": {"notify": {"manual_trigger": True}}, - }, - { - "codecov": {"notify": {"manual_trigger": True}}, - }, - ), - ], - ) - def test_random_real_life_cases(self, user_input, expected_result): - # Some random cases based on real world examples - assert expected_result == validate_yaml(user_input) - - def test_case_with_experimental_turned_on_valid(self, mocker): - mocker.patch.dict(os.environ, {"CERBERUS_VALIDATOR_RATE": "1.0"}) - user_input = {"coverage": {"status": {"patch": True}}} - expected_result = {"coverage": {"status": {"patch": True}}} - assert expected_result == validate_yaml(user_input) - - def test_case_with_experimental_turned_invalid(self, mocker): - mocker.patch.dict(os.environ, {"CERBERUS_VALIDATOR_RATE": "1.0"}) - user_input = {"coverage": {"status": {"patch": "banana"}}} - with pytest.raises(InvalidYamlException) as ex: - validate_yaml(user_input) - assert ex.value.error_location == ["coverage", "status", "patch"] - assert ex.value.error_message == "must be of ['dict', 'boolean'] type" - - def test_many_flags_validation(self): - user_input = { - "codecov": {"max_report_age": False, "notify": {"after_n_builds": 10}}, - "comment": {"layout": "reach,footer"}, - "coverage": { - "status": { - "patch": { - "default": True, - "numbers_only": {"paths": ["wildwest/numbers.py"]}, - "strings_only": {"paths": ["wildwest/strings.py"]}, - "one": {"flags": ["one"]}, - "two": {"flags": ["two"]}, - "three": {"flags": ["three"]}, - "four": {"flags": ["four"]}, - "five": {"flags": ["five"]}, - "six": {"flags": ["six"]}, - "seven": {"flags": ["seven"]}, - "eight": {"flags": ["eight"]}, - "nine": {"flags": ["nine"]}, - "ten": {"flags": ["ten"]}, - "eleven": {"flags": ["eleven"]}, - "one_without_t": {"flags": ["one"], "paths": ["wildwest"]}, - "two_without_t": {"flags": ["two"], "paths": ["wildwest"]}, - "three_without_t": {"flags": ["three"], "paths": ["wildwest"]}, - "four_without_t": {"flags": ["four"], "paths": ["wildwest"]}, - "five_without_t": {"flags": ["five"], "paths": ["wildwest"]}, - "six_without_t": {"flags": ["six"], "paths": ["wildwest"]}, - "seven_without_t": {"flags": ["seven"], "paths": ["wildwest"]}, - "eight_without_t": {"flags": ["eight"], "paths": ["wildwest"]}, - "nine_without_t": {"flags": ["nine"], "paths": ["wildwest"]}, - "ten_without_t": {"flags": ["ten"], "paths": ["wildwest"]}, - "eleven_without_t": { - "flags": ["eleven"], - "paths": ["wildwest"], - }, - }, - "project": { - "default": True, - "numbers_only": {"paths": ["wildwest/numbers.py"]}, - "strings_only": {"paths": ["wildwest/strings.py"]}, - "one": {"flags": ["one"]}, - "two": {"flags": ["two"]}, - "three": {"flags": ["three"]}, - "four": {"flags": ["four"]}, - "five": {"flags": ["five"]}, - "six": {"flags": ["six"]}, - "seven": {"flags": ["seven"]}, - "eight": {"flags": ["eight"]}, - "nine": {"flags": ["nine"]}, - "ten": {"flags": ["ten"]}, - "eleven": {"flags": ["eleven"]}, - "one_without_t": {"flags": ["one"], "paths": ["wildwest"]}, - "two_without_t": {"flags": ["two"], "paths": ["wildwest"]}, - "three_without_t": {"flags": ["three"], "paths": ["wildwest"]}, - "four_without_t": {"flags": ["four"], "paths": ["wildwest"]}, - "five_without_t": {"flags": ["five"], "paths": ["wildwest"]}, - "six_without_t": {"flags": ["six"], "paths": ["wildwest"]}, - "seven_without_t": {"flags": ["seven"], "paths": ["wildwest"]}, - "eight_without_t": {"flags": ["eight"], "paths": ["wildwest"]}, - "nine_without_t": {"flags": ["nine"], "paths": ["wildwest"]}, - "ten_without_t": {"flags": ["ten"], "paths": ["wildwest"]}, - "eleven_without_t": { - "flags": ["eleven"], - "paths": ["wildwest"], - }, - }, - "changes": { - "numbers_only": {"paths": ["wildwest/numbers.py"]}, - "strings_only": {"paths": ["wildwest/strings.py"]}, - "default": True, - "one": {"flags": ["one"]}, - "two": {"flags": ["two"]}, - "three": {"flags": ["three"]}, - "four": {"flags": ["four"]}, - "five": {"flags": ["five"]}, - "six": {"flags": ["six"]}, - "seven": {"flags": ["seven"]}, - "eight": {"flags": ["eight"]}, - "nine": {"flags": ["nine"]}, - "ten": {"flags": ["ten"]}, - "eleven": {"flags": ["eleven"]}, - "one_without_t": {"flags": ["one"], "paths": ["wildwest"]}, - "two_without_t": {"flags": ["two"], "paths": ["wildwest"]}, - "three_without_t": {"flags": ["three"], "paths": ["wildwest"]}, - "four_without_t": {"flags": ["four"], "paths": ["wildwest"]}, - "five_without_t": {"flags": ["five"], "paths": ["wildwest"]}, - "six_without_t": {"flags": ["six"], "paths": ["wildwest"]}, - "seven_without_t": {"flags": ["seven"], "paths": ["wildwest"]}, - "eight_without_t": {"flags": ["eight"], "paths": ["wildwest"]}, - "nine_without_t": {"flags": ["nine"], "paths": ["wildwest"]}, - "ten_without_t": {"flags": ["ten"], "paths": ["wildwest"]}, - "eleven_without_t": { - "flags": ["eleven"], - "paths": ["wildwest"], - }, - }, - } - }, - "flag_management": { - "default_rules": { - "carryforward": False, - "statuses": [{"name_prefix": "aaa", "type": "patch"}], - }, - "individual_flags": [ - {"name": "cawcaw", "paths": ["banana"], "after_n_builds": 3} - ], - }, - } - expected_result = { - "codecov": {"max_report_age": False, "notify": {"after_n_builds": 10}}, - "comment": {"layout": "reach,footer"}, - "coverage": { - "status": { - "patch": { - "default": True, - "numbers_only": {"paths": ["^wildwest/numbers.py.*"]}, - "strings_only": {"paths": ["^wildwest/strings.py.*"]}, - "one": {"flags": ["one"]}, - "two": {"flags": ["two"]}, - "three": {"flags": ["three"]}, - "four": {"flags": ["four"]}, - "five": {"flags": ["five"]}, - "six": {"flags": ["six"]}, - "seven": {"flags": ["seven"]}, - "eight": {"flags": ["eight"]}, - "nine": {"flags": ["nine"]}, - "ten": {"flags": ["ten"]}, - "eleven": {"flags": ["eleven"]}, - "one_without_t": {"flags": ["one"], "paths": ["^wildwest.*"]}, - "two_without_t": {"flags": ["two"], "paths": ["^wildwest.*"]}, - "three_without_t": { - "flags": ["three"], - "paths": ["^wildwest.*"], - }, - "four_without_t": {"flags": ["four"], "paths": ["^wildwest.*"]}, - "five_without_t": {"flags": ["five"], "paths": ["^wildwest.*"]}, - "six_without_t": {"flags": ["six"], "paths": ["^wildwest.*"]}, - "seven_without_t": { - "flags": ["seven"], - "paths": ["^wildwest.*"], - }, - "eight_without_t": { - "flags": ["eight"], - "paths": ["^wildwest.*"], - }, - "nine_without_t": {"flags": ["nine"], "paths": ["^wildwest.*"]}, - "ten_without_t": {"flags": ["ten"], "paths": ["^wildwest.*"]}, - "eleven_without_t": { - "flags": ["eleven"], - "paths": ["^wildwest.*"], - }, - }, - "project": { - "default": True, - "numbers_only": {"paths": ["^wildwest/numbers.py.*"]}, - "strings_only": {"paths": ["^wildwest/strings.py.*"]}, - "one": {"flags": ["one"]}, - "two": {"flags": ["two"]}, - "three": {"flags": ["three"]}, - "four": {"flags": ["four"]}, - "five": {"flags": ["five"]}, - "six": {"flags": ["six"]}, - "seven": {"flags": ["seven"]}, - "eight": {"flags": ["eight"]}, - "nine": {"flags": ["nine"]}, - "ten": {"flags": ["ten"]}, - "eleven": {"flags": ["eleven"]}, - "one_without_t": {"flags": ["one"], "paths": ["^wildwest.*"]}, - "two_without_t": {"flags": ["two"], "paths": ["^wildwest.*"]}, - "three_without_t": { - "flags": ["three"], - "paths": ["^wildwest.*"], - }, - "four_without_t": {"flags": ["four"], "paths": ["^wildwest.*"]}, - "five_without_t": {"flags": ["five"], "paths": ["^wildwest.*"]}, - "six_without_t": {"flags": ["six"], "paths": ["^wildwest.*"]}, - "seven_without_t": { - "flags": ["seven"], - "paths": ["^wildwest.*"], - }, - "eight_without_t": { - "flags": ["eight"], - "paths": ["^wildwest.*"], - }, - "nine_without_t": {"flags": ["nine"], "paths": ["^wildwest.*"]}, - "ten_without_t": {"flags": ["ten"], "paths": ["^wildwest.*"]}, - "eleven_without_t": { - "flags": ["eleven"], - "paths": ["^wildwest.*"], - }, - }, - "changes": { - "numbers_only": {"paths": ["^wildwest/numbers.py.*"]}, - "strings_only": {"paths": ["^wildwest/strings.py.*"]}, - "default": True, - "one": {"flags": ["one"]}, - "two": {"flags": ["two"]}, - "three": {"flags": ["three"]}, - "four": {"flags": ["four"]}, - "five": {"flags": ["five"]}, - "six": {"flags": ["six"]}, - "seven": {"flags": ["seven"]}, - "eight": {"flags": ["eight"]}, - "nine": {"flags": ["nine"]}, - "ten": {"flags": ["ten"]}, - "eleven": {"flags": ["eleven"]}, - "one_without_t": {"flags": ["one"], "paths": ["^wildwest.*"]}, - "two_without_t": {"flags": ["two"], "paths": ["^wildwest.*"]}, - "three_without_t": { - "flags": ["three"], - "paths": ["^wildwest.*"], - }, - "four_without_t": {"flags": ["four"], "paths": ["^wildwest.*"]}, - "five_without_t": {"flags": ["five"], "paths": ["^wildwest.*"]}, - "six_without_t": {"flags": ["six"], "paths": ["^wildwest.*"]}, - "seven_without_t": { - "flags": ["seven"], - "paths": ["^wildwest.*"], - }, - "eight_without_t": { - "flags": ["eight"], - "paths": ["^wildwest.*"], - }, - "nine_without_t": {"flags": ["nine"], "paths": ["^wildwest.*"]}, - "ten_without_t": {"flags": ["ten"], "paths": ["^wildwest.*"]}, - "eleven_without_t": { - "flags": ["eleven"], - "paths": ["^wildwest.*"], - }, - }, - } - }, - "flag_management": { - "default_rules": { - "carryforward": False, - "statuses": [{"name_prefix": "aaa", "type": "patch"}], - }, - "individual_flags": [ - {"name": "cawcaw", "paths": ["^banana.*"], "after_n_builds": 3} - ], - }, - } - assert validate_yaml(user_input) == expected_result - - def test_validate_bot_none(self): - user_input = {"codecov": {"bot": None}} - expected_result = {"codecov": {"bot": None}} - result = validate_yaml(user_input) - assert result == expected_result - - def test_validate_flag_too_long(self): - user_input = {"flags": {"abcdefg" * 500: {"paths": ["banana"]}}} - with pytest.raises(InvalidYamlException) as exc: - validate_yaml(user_input) - assert exc.value.error_location == [ - "flags", - "abcdefg" * 500, - ] - - def test_validate_parser_only_field(self): - user_input = {"parsers": {"go": {"partials_as_hits": True}}} - expected_result = {"parsers": {"go": {"partials_as_hits": True}}} - result = validate_yaml(user_input) - assert result == expected_result - - def test_simple_case(self): - encoded_value = "secret:v1::zsV9A8pHadNle357DGJHbZCTyCYA+TXdUd9TN3IY2DIWcPOtgK3Pg1EgA6OZr9XJ1EsdpL765yWrN4pfR3elRdN2LUwiuv6RkNjpbiruHx45agsgxdu8fi24p5pkCLvjcW0HqdH2PTvmHauIp+ptgA==" + # def test_empty_case(self): + # user_input = {} + # expected_result = {} + # assert validate_yaml(user_input) == expected_result + + # @pytest.mark.parametrize("input_value", ["", 10, [], tuple(), set()]) + # def test_wrong_object_type(self, input_value): + # with pytest.raises(InvalidYamlException) as exc: + # validate_yaml(input_value) + # exception = exc.value + # assert exception.error_location == [] + # assert exception.error_message == "Yaml needs to be a dict" + # assert exception.original_exc is None + + # @pytest.mark.parametrize( + # "user_input, expected_result", + # [ + # ( + # { + # "coverage": {"status": {"patch": True, "project": False}}, + # "comment": False, + # }, + # { + # "comment": False, + # "coverage": {"status": {"patch": True, "project": False}}, + # }, + # ), + # ( + # { + # "codecov": {"bot": "codecov-io", "require_ci_to_pass": False}, + # "coverage": { + # "status": { + # "project": { + # "default": {"target": "78%", "threshold": "5%"} + # }, + # "patch": {"default": {"target": "75%"}}, + # } + # }, + # }, + # { + # "codecov": {"bot": "codecov-io", "require_ci_to_pass": False}, + # "coverage": { + # "status": { + # "project": {"default": {"target": 78.0, "threshold": 5.0}}, + # "patch": {"default": {"target": 75.0}}, + # } + # }, + # }, + # ), + # ( + # { + # "coverage": { + # "status": { + # "project": False, + # "patch": { + # "default": {"informational": True}, + # "ui": {"informational": True}, + # }, + # "changes": False, + # } + # }, + # "comment": False, + # "flags": {"ui": {"paths": ["/ui-v2/"]}}, + # "github_checks": {"annotations": False}, + # "ignore": [ + # "agent/uiserver/bindata_assetfs.go", + # "vendor/**/*", + # "**/*.pb.go", + # ], + # }, + # { + # "coverage": { + # "status": { + # "project": False, + # "patch": { + # "default": {"informational": True}, + # "ui": {"informational": True}, + # }, + # "changes": False, + # } + # }, + # "comment": False, + # "flags": {"ui": {"paths": ["^/ui-v2/.*"]}}, + # "github_checks": {"annotations": False}, + # "ignore": [ + # "^agent/uiserver/bindata_assetfs.go.*", + # "(?s:vendor/.*/[^\\/]*)\\Z", + # "(?s:.*/[^\\/]*\\.pb\\.go)\\Z", + # ], + # }, + # ), + # ( + # { + # "comment": { + # "require_head": True, + # "require_base": True, + # "layout": "diff", + # "require_changes": True, + # "branches": ["main"], + # "behavior": "once", + # "after_n_builds": 6, + # "hide_project_coverage": True, + # }, + # "coverage": { + # "status": { + # "project": {"default": {"threshold": "1%"}}, + # "patch": False, + # } + # }, + # "github_checks": {"annotations": False}, + # "fixes": [ + # "/opt/conda/lib/python3.8/site-packages/::project/", + # "C:/Users/circleci/project/build/win_tmp/build/::project/", + # ], + # "ignore": ["coffee", "party_man", "test"], + # "codecov": {"notify": {"after_n_builds": 6}}, + # }, + # { + # "comment": { + # "require_head": True, + # "require_base": True, + # "layout": "diff", + # "require_changes": [0b001], + # "branches": ["^main$"], + # "behavior": "once", + # "after_n_builds": 6, + # "hide_project_coverage": True, + # }, + # "coverage": { + # "status": { + # "project": {"default": {"threshold": 1}}, + # "patch": False, + # } + # }, + # "github_checks": {"annotations": False}, + # "fixes": [ + # "^/opt/conda/lib/python3.8/site-packages/::project/", + # "^C:/Users/circleci/project/build/win_tmp/build/::project/", + # ], + # "ignore": ["^coffee.*", "^party_man.*", "^test.*"], + # "codecov": {"notify": {"after_n_builds": 6}}, + # }, + # ), + # ( + # { + # "ignore": ["js/plugins", "plugins"], + # "coverage": { + # "notify": { + # "slack": { + # "default": { + # "url": "https://hooks.slack.com/services/testdazzd/testrf4k6py/test72nq0j0ke3prs2fdvfuj", + # "only_pulls": False, + # "branches": ["main", "qa", "dev"], + # } + # } + # } + # }, + # }, + # { + # "ignore": ["^js/plugins.*", "^plugins.*"], + # "coverage": { + # "notify": { + # "slack": { + # "default": { + # "url": "https://hooks.slack.com/services/testdazzd/testrf4k6py/test72nq0j0ke3prs2fdvfuj", + # "only_pulls": False, + # "branches": ["^main$", "^qa$", "^dev$"], + # } + # } + # } + # }, + # }, + # ), + # ( + # { + # "codecov": {"notify": {"manual_trigger": True}}, + # }, + # { + # "codecov": {"notify": {"manual_trigger": True}}, + # }, + # ), + # ], + # ) + # def test_random_real_life_cases(self, user_input, expected_result): + # # Some random cases based on real world examples + # assert expected_result == validate_yaml(user_input) + + # def test_case_with_experimental_turned_on_valid(self, mocker): + # mocker.patch.dict(os.environ, {"CERBERUS_VALIDATOR_RATE": "1.0"}) + # user_input = {"coverage": {"status": {"patch": True}}} + # expected_result = {"coverage": {"status": {"patch": True}}} + # assert expected_result == validate_yaml(user_input) + + # def test_case_with_experimental_turned_invalid(self, mocker): + # mocker.patch.dict(os.environ, {"CERBERUS_VALIDATOR_RATE": "1.0"}) + # user_input = {"coverage": {"status": {"patch": "banana"}}} + # with pytest.raises(InvalidYamlException) as ex: + # validate_yaml(user_input) + # assert ex.value.error_location == ["coverage", "status", "patch"] + # assert ex.value.error_message == "must be of ['dict', 'boolean'] type" + + # def test_many_flags_validation(self): + # user_input = { + # "codecov": {"max_report_age": False, "notify": {"after_n_builds": 10}}, + # "comment": {"layout": "reach,footer"}, + # "coverage": { + # "status": { + # "patch": { + # "default": True, + # "numbers_only": {"paths": ["wildwest/numbers.py"]}, + # "strings_only": {"paths": ["wildwest/strings.py"]}, + # "one": {"flags": ["one"]}, + # "two": {"flags": ["two"]}, + # "three": {"flags": ["three"]}, + # "four": {"flags": ["four"]}, + # "five": {"flags": ["five"]}, + # "six": {"flags": ["six"]}, + # "seven": {"flags": ["seven"]}, + # "eight": {"flags": ["eight"]}, + # "nine": {"flags": ["nine"]}, + # "ten": {"flags": ["ten"]}, + # "eleven": {"flags": ["eleven"]}, + # "one_without_t": {"flags": ["one"], "paths": ["wildwest"]}, + # "two_without_t": {"flags": ["two"], "paths": ["wildwest"]}, + # "three_without_t": {"flags": ["three"], "paths": ["wildwest"]}, + # "four_without_t": {"flags": ["four"], "paths": ["wildwest"]}, + # "five_without_t": {"flags": ["five"], "paths": ["wildwest"]}, + # "six_without_t": {"flags": ["six"], "paths": ["wildwest"]}, + # "seven_without_t": {"flags": ["seven"], "paths": ["wildwest"]}, + # "eight_without_t": {"flags": ["eight"], "paths": ["wildwest"]}, + # "nine_without_t": {"flags": ["nine"], "paths": ["wildwest"]}, + # "ten_without_t": {"flags": ["ten"], "paths": ["wildwest"]}, + # "eleven_without_t": { + # "flags": ["eleven"], + # "paths": ["wildwest"], + # }, + # }, + # "project": { + # "default": True, + # "numbers_only": {"paths": ["wildwest/numbers.py"]}, + # "strings_only": {"paths": ["wildwest/strings.py"]}, + # "one": {"flags": ["one"]}, + # "two": {"flags": ["two"]}, + # "three": {"flags": ["three"]}, + # "four": {"flags": ["four"]}, + # "five": {"flags": ["five"]}, + # "six": {"flags": ["six"]}, + # "seven": {"flags": ["seven"]}, + # "eight": {"flags": ["eight"]}, + # "nine": {"flags": ["nine"]}, + # "ten": {"flags": ["ten"]}, + # "eleven": {"flags": ["eleven"]}, + # "one_without_t": {"flags": ["one"], "paths": ["wildwest"]}, + # "two_without_t": {"flags": ["two"], "paths": ["wildwest"]}, + # "three_without_t": {"flags": ["three"], "paths": ["wildwest"]}, + # "four_without_t": {"flags": ["four"], "paths": ["wildwest"]}, + # "five_without_t": {"flags": ["five"], "paths": ["wildwest"]}, + # "six_without_t": {"flags": ["six"], "paths": ["wildwest"]}, + # "seven_without_t": {"flags": ["seven"], "paths": ["wildwest"]}, + # "eight_without_t": {"flags": ["eight"], "paths": ["wildwest"]}, + # "nine_without_t": {"flags": ["nine"], "paths": ["wildwest"]}, + # "ten_without_t": {"flags": ["ten"], "paths": ["wildwest"]}, + # "eleven_without_t": { + # "flags": ["eleven"], + # "paths": ["wildwest"], + # }, + # }, + # "changes": { + # "numbers_only": {"paths": ["wildwest/numbers.py"]}, + # "strings_only": {"paths": ["wildwest/strings.py"]}, + # "default": True, + # "one": {"flags": ["one"]}, + # "two": {"flags": ["two"]}, + # "three": {"flags": ["three"]}, + # "four": {"flags": ["four"]}, + # "five": {"flags": ["five"]}, + # "six": {"flags": ["six"]}, + # "seven": {"flags": ["seven"]}, + # "eight": {"flags": ["eight"]}, + # "nine": {"flags": ["nine"]}, + # "ten": {"flags": ["ten"]}, + # "eleven": {"flags": ["eleven"]}, + # "one_without_t": {"flags": ["one"], "paths": ["wildwest"]}, + # "two_without_t": {"flags": ["two"], "paths": ["wildwest"]}, + # "three_without_t": {"flags": ["three"], "paths": ["wildwest"]}, + # "four_without_t": {"flags": ["four"], "paths": ["wildwest"]}, + # "five_without_t": {"flags": ["five"], "paths": ["wildwest"]}, + # "six_without_t": {"flags": ["six"], "paths": ["wildwest"]}, + # "seven_without_t": {"flags": ["seven"], "paths": ["wildwest"]}, + # "eight_without_t": {"flags": ["eight"], "paths": ["wildwest"]}, + # "nine_without_t": {"flags": ["nine"], "paths": ["wildwest"]}, + # "ten_without_t": {"flags": ["ten"], "paths": ["wildwest"]}, + # "eleven_without_t": { + # "flags": ["eleven"], + # "paths": ["wildwest"], + # }, + # }, + # } + # }, + # "flag_management": { + # "default_rules": { + # "carryforward": False, + # "statuses": [{"name_prefix": "aaa", "type": "patch"}], + # }, + # "individual_flags": [ + # {"name": "cawcaw", "paths": ["banana"], "after_n_builds": 3} + # ], + # }, + # } + # expected_result = { + # "codecov": {"max_report_age": False, "notify": {"after_n_builds": 10}}, + # "comment": {"layout": "reach,footer"}, + # "coverage": { + # "status": { + # "patch": { + # "default": True, + # "numbers_only": {"paths": ["^wildwest/numbers.py.*"]}, + # "strings_only": {"paths": ["^wildwest/strings.py.*"]}, + # "one": {"flags": ["one"]}, + # "two": {"flags": ["two"]}, + # "three": {"flags": ["three"]}, + # "four": {"flags": ["four"]}, + # "five": {"flags": ["five"]}, + # "six": {"flags": ["six"]}, + # "seven": {"flags": ["seven"]}, + # "eight": {"flags": ["eight"]}, + # "nine": {"flags": ["nine"]}, + # "ten": {"flags": ["ten"]}, + # "eleven": {"flags": ["eleven"]}, + # "one_without_t": {"flags": ["one"], "paths": ["^wildwest.*"]}, + # "two_without_t": {"flags": ["two"], "paths": ["^wildwest.*"]}, + # "three_without_t": { + # "flags": ["three"], + # "paths": ["^wildwest.*"], + # }, + # "four_without_t": {"flags": ["four"], "paths": ["^wildwest.*"]}, + # "five_without_t": {"flags": ["five"], "paths": ["^wildwest.*"]}, + # "six_without_t": {"flags": ["six"], "paths": ["^wildwest.*"]}, + # "seven_without_t": { + # "flags": ["seven"], + # "paths": ["^wildwest.*"], + # }, + # "eight_without_t": { + # "flags": ["eight"], + # "paths": ["^wildwest.*"], + # }, + # "nine_without_t": {"flags": ["nine"], "paths": ["^wildwest.*"]}, + # "ten_without_t": {"flags": ["ten"], "paths": ["^wildwest.*"]}, + # "eleven_without_t": { + # "flags": ["eleven"], + # "paths": ["^wildwest.*"], + # }, + # }, + # "project": { + # "default": True, + # "numbers_only": {"paths": ["^wildwest/numbers.py.*"]}, + # "strings_only": {"paths": ["^wildwest/strings.py.*"]}, + # "one": {"flags": ["one"]}, + # "two": {"flags": ["two"]}, + # "three": {"flags": ["three"]}, + # "four": {"flags": ["four"]}, + # "five": {"flags": ["five"]}, + # "six": {"flags": ["six"]}, + # "seven": {"flags": ["seven"]}, + # "eight": {"flags": ["eight"]}, + # "nine": {"flags": ["nine"]}, + # "ten": {"flags": ["ten"]}, + # "eleven": {"flags": ["eleven"]}, + # "one_without_t": {"flags": ["one"], "paths": ["^wildwest.*"]}, + # "two_without_t": {"flags": ["two"], "paths": ["^wildwest.*"]}, + # "three_without_t": { + # "flags": ["three"], + # "paths": ["^wildwest.*"], + # }, + # "four_without_t": {"flags": ["four"], "paths": ["^wildwest.*"]}, + # "five_without_t": {"flags": ["five"], "paths": ["^wildwest.*"]}, + # "six_without_t": {"flags": ["six"], "paths": ["^wildwest.*"]}, + # "seven_without_t": { + # "flags": ["seven"], + # "paths": ["^wildwest.*"], + # }, + # "eight_without_t": { + # "flags": ["eight"], + # "paths": ["^wildwest.*"], + # }, + # "nine_without_t": {"flags": ["nine"], "paths": ["^wildwest.*"]}, + # "ten_without_t": {"flags": ["ten"], "paths": ["^wildwest.*"]}, + # "eleven_without_t": { + # "flags": ["eleven"], + # "paths": ["^wildwest.*"], + # }, + # }, + # "changes": { + # "numbers_only": {"paths": ["^wildwest/numbers.py.*"]}, + # "strings_only": {"paths": ["^wildwest/strings.py.*"]}, + # "default": True, + # "one": {"flags": ["one"]}, + # "two": {"flags": ["two"]}, + # "three": {"flags": ["three"]}, + # "four": {"flags": ["four"]}, + # "five": {"flags": ["five"]}, + # "six": {"flags": ["six"]}, + # "seven": {"flags": ["seven"]}, + # "eight": {"flags": ["eight"]}, + # "nine": {"flags": ["nine"]}, + # "ten": {"flags": ["ten"]}, + # "eleven": {"flags": ["eleven"]}, + # "one_without_t": {"flags": ["one"], "paths": ["^wildwest.*"]}, + # "two_without_t": {"flags": ["two"], "paths": ["^wildwest.*"]}, + # "three_without_t": { + # "flags": ["three"], + # "paths": ["^wildwest.*"], + # }, + # "four_without_t": {"flags": ["four"], "paths": ["^wildwest.*"]}, + # "five_without_t": {"flags": ["five"], "paths": ["^wildwest.*"]}, + # "six_without_t": {"flags": ["six"], "paths": ["^wildwest.*"]}, + # "seven_without_t": { + # "flags": ["seven"], + # "paths": ["^wildwest.*"], + # }, + # "eight_without_t": { + # "flags": ["eight"], + # "paths": ["^wildwest.*"], + # }, + # "nine_without_t": {"flags": ["nine"], "paths": ["^wildwest.*"]}, + # "ten_without_t": {"flags": ["ten"], "paths": ["^wildwest.*"]}, + # "eleven_without_t": { + # "flags": ["eleven"], + # "paths": ["^wildwest.*"], + # }, + # }, + # } + # }, + # "flag_management": { + # "default_rules": { + # "carryforward": False, + # "statuses": [{"name_prefix": "aaa", "type": "patch"}], + # }, + # "individual_flags": [ + # {"name": "cawcaw", "paths": ["^banana.*"], "after_n_builds": 3} + # ], + # }, + # } + # assert validate_yaml(user_input) == expected_result + + # def test_validate_bot_none(self): + # user_input = {"codecov": {"bot": None}} + # expected_result = {"codecov": {"bot": None}} + # result = validate_yaml(user_input) + # assert result == expected_result + + # def test_validate_flag_too_long(self): + # user_input = {"flags": {"abcdefg" * 500: {"paths": ["banana"]}}} + # with pytest.raises(InvalidYamlException) as exc: + # validate_yaml(user_input) + # assert exc.value.error_location == [ + # "flags", + # "abcdefg" * 500, + # ] + + # def test_validate_parser_only_field(self): + # user_input = {"parsers": {"go": {"partials_as_hits": True}}} + # expected_result = {"parsers": {"go": {"partials_as_hits": True}}} + # result = validate_yaml(user_input) + # assert result == expected_result + + # def test_simple_case(self): + # encoded_value = "secret:v1::zsV9A8pHadNle357DGJHbZCTyCYA+TXdUd9TN3IY2DIWcPOtgK3Pg1EgA6OZr9XJ1EsdpL765yWrN4pfR3elRdN2LUwiuv6RkNjpbiruHx45agsgxdu8fi24p5pkCLvjcW0HqdH2PTvmHauIp+ptgA==" + # user_input = { + # "coverage": { + # "precision": 2, + # "round": "down", + # "range": "70...100", + # "status": { + # "project": { + # "custom_project": { + # "carryforward_behavior": "exclude", + # "flag_coverage_not_uploaded_behavior": "exclude", + # } + # }, + # "patch": True, + # "changes": False, + # "default_rules": { + # "carryforward_behavior": "pass", + # "flag_coverage_not_uploaded_behavior": "pass", + # }, + # "no_upload_behavior": "pass", + # }, + # "notify": {"irc": {"user_given_title": {"password": encoded_value}}}, + # }, + # "codecov": {"notify": {"require_ci_to_pass": True}}, + # "comment": { + # "behavior": "default", + # "layout": "header, diff", + # "require_changes": False, + # "show_carryforward_flags": False, + # }, + # "parsers": { + # "gcov": { + # "branch_detection": { + # "conditional": True, + # "loop": True, + # "macro": False, + # "method": False, + # } + # }, + # "jacoco": {"partials_as_hits": True}, + # }, + # "cli": { + # "plugins": {"pycoverage": {"report_type": "json"}}, + # }, + # } + # expected_result = { + # "coverage": { + # "precision": 2, + # "round": "down", + # "range": [70, 100], + # "status": { + # "project": { + # "custom_project": { + # "carryforward_behavior": "exclude", + # "flag_coverage_not_uploaded_behavior": "exclude", + # } + # }, + # "patch": True, + # "changes": False, + # "default_rules": { + # "carryforward_behavior": "pass", + # "flag_coverage_not_uploaded_behavior": "pass", + # }, + # "no_upload_behavior": "pass", + # }, + # "notify": {"irc": {"user_given_title": {"password": encoded_value}}}, + # }, + # "codecov": {"notify": {}, "require_ci_to_pass": True}, + # "comment": { + # "behavior": "default", + # "layout": "header, diff", + # "require_changes": [0b000], + # "show_carryforward_flags": False, + # }, + # "parsers": { + # "gcov": { + # "branch_detection": { + # "conditional": True, + # "loop": True, + # "macro": False, + # "method": False, + # } + # }, + # "jacoco": {"partials_as_hits": True}, + # }, + # "cli": { + # "plugins": {"pycoverage": {"report_type": "json"}}, + # }, + # } + # assert validate_yaml(user_input) == expected_result + + # def test_negative_notify_after_n_builds(self): + # user_input = {"codecov": {"notify": {"after_n_builds": -1}}} + # with pytest.raises(InvalidYamlException) as exc: + # validate_yaml(user_input) + # assert exc.value.error_location == ["codecov", "notify", "after_n_builds"] + # assert exc.value.error_message == "min value is 0" + + # def test_positive_notify_after_n_builds(self): + # user_input = {"codecov": {"notify": {"after_n_builds": 1}}} + # res = validate_yaml(user_input) + # assert res == {"codecov": {"notify": {"after_n_builds": 1}}} + + # def test_negative_comments_after_n_builds(self): + # user_input = {"comment": {"after_n_builds": -1}} + # with pytest.raises(InvalidYamlException) as exc: + # validate_yaml(user_input) + # assert exc.value.error_location == ["comment", "after_n_builds"] + # assert exc.value.error_message == "min value is 0" + + # def test_invalid_yaml_case(self): + # user_input = { + # "coverage": { + # "round": "down", + # "precision": 2, + # "range": "70...100", + # "status": {"project": {"base": "auto", "aa": True}}, + # }, + # "ignore": ["Pods/.*"], + # } + # with pytest.raises(InvalidYamlException) as exc: + # validate_yaml(user_input) + # assert exc.value.error_location == ["coverage", "status", "project", "base"] + # assert exc.value.error_message == "must be of ['dict', 'boolean'] type" + + # def test_invalid_yaml_case_custom_validator(self): + # user_input = { + # "coverage": { + # "round": "down", + # "precision": 2, + # "range": "70...5000", + # "status": {"project": {"percent": "abc"}}, + # }, + # "ignore": ["Pods/.*"], + # } + # with pytest.raises(InvalidYamlException) as exc: + # validate_yaml(user_input) + # assert exc.value.error_location == ["coverage", "range"] + # assert exc.value.error_message == "must be of list type" + + # def test_invalid_yaml_case_no_upload_behavior(self): + # user_input = { + # "coverage": { + # "round": "down", + # "precision": 2, + # "range": "70...100", + # "status": { + # "project": {"percent": "abc"}, + # "no_upload_behavior": "no-pass", + # }, + # }, + # "ignore": ["Pods/.*"], + # } + # with pytest.raises(InvalidYamlException) as exc: + # validate_yaml(user_input) + # assert exc.value.error_location == ["coverage", "status", "no_upload_behavior"] + # assert exc.value.error_message == "unallowed value no-pass" + + # def test_yaml_with_null_threshold(self): + # user_input = { + # "codecov": {"notify": {}, "require_ci_to_pass": True}, + # "comment": { + # "behavior": "default", + # "branches": None, + # "layout": "reach, diff, flags, files", + # "require_base": False, + # "require_changes": False, + # "require_head": False, + # }, + # "coverage": { + # "precision": 2, + # "range": "50...80", + # "round": "down", + # "status": { + # "changes": False, + # "patch": True, + # "project": { + # "default": {"target": "auto", "threshold": None, "base": "auto"} + # }, + # }, + # }, + # } + # res = validate_yaml(user_input) + # expected_result = { + # "codecov": {"notify": {}, "require_ci_to_pass": True}, + # "comment": { + # "behavior": "default", + # "branches": None, + # "layout": "reach, diff, flags, files", + # "require_base": False, + # "require_changes": [0b000], + # "require_head": False, + # }, + # "coverage": { + # "precision": 2, + # "range": [50.0, 80.0], + # "round": "down", + # "status": { + # "changes": False, + # "patch": True, + # "project": { + # "default": {"target": "auto", "threshold": None, "base": "auto"} + # }, + # }, + # }, + # } + # assert res == expected_result + + # def test_yaml_with_status_case(self): + # user_input = { + # "coverage": { + # "round": "down", + # "precision": 2, + # "range": "70...100", + # "status": {"project": {"default": {"base": "auto"}}}, + # }, + # "ignore": ["Pods/.*"], + # } + # expected_result = { + # "coverage": { + # "round": "down", + # "precision": 2, + # "range": [70.0, 100.0], + # "status": {"project": {"default": {"base": "auto"}}}, + # }, + # "ignore": ["Pods/.*"], + # } + # result = validate_yaml(user_input) + # assert result == expected_result + + # def test_yaml_with_flag_management(self): + # user_input = { + # "flag_management": { + # "default_rules": { + # "carryforward": True, + # "statuses": [ + # { + # "type": "project", + # "name_prefix": "healthcare", + # "threshold": 80, + # } + # ], + # }, + # "individual_flags": [ + # { + # "name": "flag_banana", + # "statuses": [ + # { + # "type": "patch", + # "name_prefix": "alliance", + # "flag_coverage_not_uploaded_behavior": "include", + # } + # ], + # } + # ], + # } + # } + # expected_result = { + # "flag_management": { + # "individual_flags": [ + # { + # "name": "flag_banana", + # "statuses": [ + # { + # "type": "patch", + # "name_prefix": "alliance", + # "flag_coverage_not_uploaded_behavior": "include", + # } + # ], + # } + # ], + # "default_rules": { + # "carryforward": True, + # "statuses": [ + # { + # "type": "project", + # "name_prefix": "healthcare", + # "threshold": 80.0, + # } + # ], + # }, + # } + # } + # result = validate_yaml(user_input) + # assert result == expected_result + + # def test_yaml_with_flag_management_statuses_with_flags(self): + # user_input = { + # "flag_management": { + # "default_rules": { + # "carryforward": True, + # "statuses": [ + # { + # "type": "project", + # "name_prefix": "healthcare", + # "threshold": 80, + # "flags": ["hahaha"], + # } + # ], + # }, + # "individual_flags": [ + # { + # "name": "flag_banana", + # "statuses": [ + # { + # "type": "patch", + # "name_prefix": "alliance", + # "flag_coverage_not_uploaded_behavior": "include", + # } + # ], + # } + # ], + # } + # } + # with pytest.raises(InvalidYamlException) as exc: + # validate_yaml(user_input) + # assert exc.value.error_location == [ + # "flag_management", + # "default_rules", + # "statuses", + # 0, + # "flags", + # ] + # assert exc.value.error_message == "extra keys not allowed" + + # def test_github_checks(self): + # user_input = {"github_checks": True} + # expected_result = {"github_checks": True} + # assert validate_yaml(user_input) == expected_result + # user_input = {"github_checks": {"annotations": False}} + # expected_result = {"github_checks": {"annotations": False}} + # assert validate_yaml(user_input) == expected_result + + # def test_validate_jacoco_partials(self): + # user_input = {"parsers": {"jacoco": {"partials_as_hits": True}}} + # expected_result = {"parsers": {"jacoco": {"partials_as_hits": True}}} + # result = validate_yaml(user_input) + # assert result == expected_result + + # @pytest.mark.parametrize( + # "input, expected", + # [ + # pytest.param( + # {"comment": {"require_bundle_changes": False}}, + # {"comment": {"require_bundle_changes": False}}, + # id="no_bundle_changes_required", + # ), + # pytest.param( + # { + # "comment": { + # "require_bundle_changes": True, + # "bundle_change_threshold": 1200, + # } + # }, + # { + # "comment": { + # "require_bundle_changes": True, + # "bundle_change_threshold": ("absolute", 1200), + # } + # }, + # id="bundle_changes_with_threshold", + # ), + # pytest.param( + # { + # "comment": { + # "require_bundle_changes": "bundle_increase", + # "bundle_change_threshold": "1mb", + # } + # }, + # { + # "comment": { + # "require_bundle_changes": "bundle_increase", + # "bundle_change_threshold": ("absolute", 1000000), + # } + # }, + # id="bundle_increase_required_with_threshold", + # ), + # pytest.param( + # { + # "comment": { + # "require_bundle_changes": "bundle_increase", + # "bundle_change_threshold": "10%", + # } + # }, + # { + # "comment": { + # "require_bundle_changes": "bundle_increase", + # "bundle_change_threshold": ("percentage", 10.0), + # } + # }, + # id="bundle_increase_required_with_percentage_threshold", + # ), + # ], + # ) + # def test_bundle_analysis_comment_config(self, input, expected, mocker): + # mocker.patch.object(BUNDLE_THRESHOLD_FLAG, "check_value", return_value=True) + # result = validate_yaml(input) + # assert result == expected + + # @pytest.mark.parametrize( + # "input, expected", + # [ + # pytest.param( + # { + # "bundle_analysis": { + # "status": False, + # "warning_threshold": "10%", + # } + # }, + # { + # "bundle_analysis": { + # "status": False, + # "warning_threshold": ("percentage", 10.0), + # } + # }, + # id="status_off_percentage_threshold", + # ), + # pytest.param( + # { + # "bundle_analysis": { + # "status": True, + # "warning_threshold": "10kb", + # } + # }, + # { + # "bundle_analysis": { + # "status": True, + # "warning_threshold": ("absolute", 10000), + # } + # }, + # id="status_on_absolute_threshold", + # ), + # pytest.param( + # { + # "bundle_analysis": { + # "status": "informational", + # } + # }, + # { + # "bundle_analysis": { + # "status": "informational", + # } + # }, + # id="status_informational_no_threshold", + # ), + # ], + # ) + # def test_bundle_analysis_config(self, input, expected, mocker): + # mocker.patch.object(BUNDLE_THRESHOLD_FLAG, "check_value", return_value=True) + # result = validate_yaml(input) + # assert result == expected + + def test_bad_yaml(self): user_input = { "coverage": { "precision": 2, @@ -502,7 +952,7 @@ def test_simple_case(self): }, "no_upload_behavior": "pass", }, - "notify": {"irc": {"user_given_title": {"password": encoded_value}}}, + "notify": {"irc": {"user_given_title": {"password": "asdf"}}}, }, "codecov": {"notify": {"require_ci_to_pass": True}}, "comment": { @@ -525,929 +975,548 @@ def test_simple_case(self): "cli": { "plugins": {"pycoverage": {"report_type": "json"}}, }, - } - expected_result = { - "coverage": { - "precision": 2, - "round": "down", - "range": [70, 100], - "status": { - "project": { - "custom_project": { - "carryforward_behavior": "exclude", - "flag_coverage_not_uploaded_behavior": "exclude", - } - }, - "patch": True, - "changes": False, - "default_rules": { - "carryforward_behavior": "pass", - "flag_coverage_not_uploaded_behavior": "pass", - }, - "no_upload_behavior": "pass", - }, - "notify": {"irc": {"user_given_title": {"password": encoded_value}}}, - }, - "codecov": {"notify": {}, "require_ci_to_pass": True}, - "comment": { - "behavior": "default", - "layout": "header, diff", - "require_changes": [0b000], - "show_carryforward_flags": False, - }, - "parsers": { - "gcov": { - "branch_detection": { - "conditional": True, - "loop": True, - "macro": False, - "method": False, - } - }, - "jacoco": {"partials_as_hits": True}, - }, - "cli": { - "plugins": {"pycoverage": {"report_type": "json"}}, - }, - } - assert validate_yaml(user_input) == expected_result - - def test_negative_notify_after_n_builds(self): - user_input = {"codecov": {"notify": {"after_n_builds": -1}}} - with pytest.raises(InvalidYamlException) as exc: - validate_yaml(user_input) - assert exc.value.error_location == ["codecov", "notify", "after_n_builds"] - assert exc.value.error_message == "min value is 0" - - def test_positive_notify_after_n_builds(self): - user_input = {"codecov": {"notify": {"after_n_builds": 1}}} - res = validate_yaml(user_input) - assert res == {"codecov": {"notify": {"after_n_builds": 1}}} - - def test_negative_comments_after_n_builds(self): - user_input = {"comment": {"after_n_builds": -1}} - with pytest.raises(InvalidYamlException) as exc: - validate_yaml(user_input) - assert exc.value.error_location == ["comment", "after_n_builds"] - assert exc.value.error_message == "min value is 0" - - def test_invalid_yaml_case(self): - user_input = { - "coverage": { - "round": "down", - "precision": 2, - "range": "70...100", - "status": {"project": {"base": "auto", "aa": True}}, - }, - "ignore": ["Pods/.*"], - } - with pytest.raises(InvalidYamlException) as exc: - validate_yaml(user_input) - assert exc.value.error_location == ["coverage", "status", "project", "base"] - assert exc.value.error_message == "must be of ['dict', 'boolean'] type" - - def test_invalid_yaml_case_custom_validator(self): - user_input = { - "coverage": { - "round": "down", - "precision": 2, - "range": "70...5000", - "status": {"project": {"percent": "abc"}}, - }, - "ignore": ["Pods/.*"], - } - with pytest.raises(InvalidYamlException) as exc: - validate_yaml(user_input) - assert exc.value.error_location == ["coverage", "range"] - assert exc.value.error_message == "must be of list type" - - def test_invalid_yaml_case_no_upload_behavior(self): - user_input = { - "coverage": { - "round": "down", - "precision": 2, - "range": "70...100", - "status": { - "project": {"percent": "abc"}, - "no_upload_behavior": "no-pass", - }, - }, - "ignore": ["Pods/.*"], - } - with pytest.raises(InvalidYamlException) as exc: - validate_yaml(user_input) - assert exc.value.error_location == ["coverage", "status", "no_upload_behavior"] - assert exc.value.error_message == "unallowed value no-pass" - - def test_yaml_with_null_threshold(self): - user_input = { - "codecov": {"notify": {}, "require_ci_to_pass": True}, - "comment": { - "behavior": "default", - "branches": None, - "layout": "reach, diff, flags, files", - "require_base": False, - "require_changes": False, - "require_head": False, - }, - "coverage": { - "precision": 2, - "range": "50...80", - "round": "down", - "status": { - "changes": False, - "patch": True, - "project": { - "default": {"target": "auto", "threshold": None, "base": "auto"} - }, - }, - }, - } - res = validate_yaml(user_input) - expected_result = { - "codecov": {"notify": {}, "require_ci_to_pass": True}, - "comment": { - "behavior": "default", - "branches": None, - "layout": "reach, diff, flags, files", - "require_base": False, - "require_changes": [0b000], - "require_head": False, - }, - "coverage": { - "precision": 2, - "range": [50.0, 80.0], - "round": "down", - "status": { - "changes": False, - "patch": True, - "project": { - "default": {"target": "auto", "threshold": None, "base": "auto"} - }, - }, - }, - } - assert res == expected_result - - def test_yaml_with_status_case(self): - user_input = { - "coverage": { - "round": "down", - "precision": 2, - "range": "70...100", - "status": {"project": {"default": {"base": "auto"}}}, - }, - "ignore": ["Pods/.*"], - } - expected_result = { - "coverage": { - "round": "down", - "precision": 2, - "range": [70.0, 100.0], - "status": {"project": {"default": {"base": "auto"}}}, - }, - "ignore": ["Pods/.*"], - } - result = validate_yaml(user_input) - assert result == expected_result - - def test_yaml_with_flag_management(self): - user_input = { - "flag_management": { - "default_rules": { - "carryforward": True, - "statuses": [ - { - "type": "project", - "name_prefix": "healthcare", - "threshold": 80, - } - ], - }, - "individual_flags": [ - { - "name": "flag_banana", - "statuses": [ - { - "type": "patch", - "name_prefix": "alliance", - "flag_coverage_not_uploaded_behavior": "include", - } - ], - } - ], - } - } - expected_result = { - "flag_management": { - "individual_flags": [ - { - "name": "flag_banana", - "statuses": [ - { - "type": "patch", - "name_prefix": "alliance", - "flag_coverage_not_uploaded_behavior": "include", - } - ], - } - ], - "default_rules": { - "carryforward": True, - "statuses": [ - { - "type": "project", - "name_prefix": "healthcare", - "threshold": 80.0, - } - ], - }, - } - } - result = validate_yaml(user_input) - assert result == expected_result - - def test_yaml_with_flag_management_statuses_with_flags(self): - user_input = { - "flag_management": { - "default_rules": { - "carryforward": True, - "statuses": [ - { - "type": "project", - "name_prefix": "healthcare", - "threshold": 80, - "flags": ["hahaha"], - } - ], - }, - "individual_flags": [ - { - "name": "flag_banana", - "statuses": [ - { - "type": "patch", - "name_prefix": "alliance", - "flag_coverage_not_uploaded_behavior": "include", - } - ], - } - ], - } - } - with pytest.raises(InvalidYamlException) as exc: - validate_yaml(user_input) - assert exc.value.error_location == [ - "flag_management", - "default_rules", - "statuses", - 0, - "flags", - ] - assert exc.value.error_message == "extra keys not allowed" - - def test_github_checks(self): - user_input = {"github_checks": True} - expected_result = {"github_checks": True} - assert validate_yaml(user_input) == expected_result - user_input = {"github_checks": {"annotations": False}} - expected_result = {"github_checks": {"annotations": False}} - assert validate_yaml(user_input) == expected_result - - def test_validate_jacoco_partials(self): - user_input = {"parsers": {"jacoco": {"partials_as_hits": True}}} - expected_result = {"parsers": {"jacoco": {"partials_as_hits": True}}} - result = validate_yaml(user_input) - assert result == expected_result - - @pytest.mark.parametrize( - "input, expected", - [ - pytest.param( - {"comment": {"require_bundle_changes": False}}, - {"comment": {"require_bundle_changes": False}}, - id="no_bundle_changes_required", - ), - pytest.param( - { - "comment": { - "require_bundle_changes": True, - "bundle_change_threshold": 1200, - } - }, - { - "comment": { - "require_bundle_changes": True, - "bundle_change_threshold": ("absolute", 1200), - } - }, - id="bundle_changes_with_threshold", - ), - pytest.param( - { - "comment": { - "require_bundle_changes": "bundle_increase", - "bundle_change_threshold": "1mb", - } - }, - { - "comment": { - "require_bundle_changes": "bundle_increase", - "bundle_change_threshold": ("absolute", 1000000), - } - }, - id="bundle_increase_required_with_threshold", - ), - pytest.param( - { - "comment": { - "require_bundle_changes": "bundle_increase", - "bundle_change_threshold": "10%", - } - }, - { - "comment": { - "require_bundle_changes": "bundle_increase", - "bundle_change_threshold": ("percentage", 10.0), - } - }, - id="bundle_increase_required_with_percentage_threshold", - ), - ], - ) - def test_bundle_analysis_comment_config(self, input, expected, mocker): - mocker.patch.object(BUNDLE_THRESHOLD_FLAG, "check_value", return_value=True) - result = validate_yaml(input) - assert result == expected - - @pytest.mark.parametrize( - "input, expected", - [ - pytest.param( - { - "bundle_analysis": { - "status": False, - "warning_threshold": "10%", - } - }, - { - "bundle_analysis": { - "status": False, - "warning_threshold": ("percentage", 10.0), - } - }, - id="status_off_percentage_threshold", - ), - pytest.param( - { - "bundle_analysis": { - "status": True, - "warning_threshold": "10kb", - } - }, - { - "bundle_analysis": { - "status": True, - "warning_threshold": ("absolute", 10000), - } - }, - id="status_on_absolute_threshold", - ), - pytest.param( - { - "bundle_analysis": { - "status": "informational", - } - }, - { - "bundle_analysis": { - "status": "informational", - } - }, - id="status_informational_no_threshold", - ), - ], - ) - def test_bundle_analysis_config(self, input, expected, mocker): - mocker.patch.object(BUNDLE_THRESHOLD_FLAG, "check_value", return_value=True) - result = validate_yaml(input) - assert result == expected - - -class TestValidationConfig(object): - def test_validate_default_config_yaml(self, mocker): - mocker.patch.dict(os.environ, {}, clear=True) - mocker.patch.object( - ConfigHelper, "load_yaml_file", side_effect=FileNotFoundError() - ) - this_config = ConfigHelper() - mocker.patch("shared.config._get_config_instance", return_value=this_config) - expected_result = { - "codecov": {"require_ci_to_pass": True, "notify": {"wait_for_ci": True}}, - "coverage": { - "precision": 2, - "round": "down", - "range": [60.0, 80.0], - "status": { - "project": True, - "patch": True, - "changes": False, - "default_rules": {"flag_coverage_not_uploaded_behavior": "include"}, - }, - }, - "comment": { - "layout": "reach,diff,flags,tree,reach", - "behavior": "default", - "show_carryforward_flags": False, - }, - "github_checks": {"annotations": True}, - "slack_app": True, - } - res = validate_yaml( - get_config("site", default={}), - show_secrets_for=("github", "11934774", "154468867"), - ) - assert res == expected_result - - -def test_validation_with_branches(): - user_input = { - "comment": { - "require_head": True, - "require_base": True, - "layout": "diff", - "require_changes": True, - "branches": ["main"], - "behavior": "once", - "after_n_builds": 6, - }, - "coverage": { - "status": {"project": {"default": {"threshold": "1%"}}, "patch": False} - }, - "github_checks": {"annotations": False}, - "fixes": [ - "/opt/conda/lib/python3.8/site-packages/::project/", - "C:/Users/circleci/project/build/win_tmp/build/::project/", - ], - "ignore": ["coffee", "party_man", "test"], - "codecov": {"notify": {"after_n_builds": 6}}, - } - expected_result = { - "comment": { - "require_head": True, - "require_base": True, - "layout": "diff", - "require_changes": [0b001], - "branches": ["^main$"], - "behavior": "once", - "after_n_builds": 6, - }, - "coverage": { - "status": {"project": {"default": {"threshold": 1}}, "patch": False} - }, - "github_checks": {"annotations": False}, - "fixes": [ - "^/opt/conda/lib/python3.8/site-packages/::project/", - "^C:/Users/circleci/project/build/win_tmp/build/::project/", - ], - "ignore": ["^coffee.*", "^party_man.*", "^test.*"], - "codecov": {"notify": {"after_n_builds": 6}}, - } - res = do_actual_validation(user_input, show_secrets_for=None) - assert res == expected_result - - -def test_validation_with_flag_carryforward(): - user_input = { - "flags": { - "old-flag": { - "carryforward": True, - "carryforward_mode": "labels", - }, - "other-old-flag": { - "carryforward": True, - "carryforward_mode": "all", - }, - }, - "flag_management": { - "individual_flags": [ - {"name": "abcdef", "carryforward_mode": "all"}, - {"name": "abcdef", "carryforward_mode": "labels"}, - ] - }, - } - assert do_actual_validation(user_input, show_secrets_for=None) == user_input - - -def test_validation_with_flag_carryforward_invalid_mode(): - user_input = { - "flag_management": { - "individual_flags": [ - {"name": "abcdef", "carryforward_mode": "mario"}, - {"name": "abcdef", "carryforward_mode": "labels"}, - ] - }, - } - with pytest.raises(InvalidYamlException) as exp: - do_actual_validation(user_input, show_secrets_for=None) - assert exp.value.error_dict == { - "flag_management": [ - { - "individual_flags": [ - {0: [{"carryforward_mode": ["unallowed value mario"]}]} - ] - } - ] - } - - -def test_validation_with_null_on_paths(): - user_input = { - "comment": {"require_head": True, "behavior": "once", "after_n_builds": 6}, - "coverage": { - "status": {"project": {"default": {"threshold": "1%"}}, "patch": False}, - "notify": {"slack": {"default": {"paths": None}}}, - }, - "ignore": ["coffee", "test"], - } - expected_result = { - "comment": {"require_head": True, "behavior": "once", "after_n_builds": 6}, - "coverage": { - "status": {"project": {"default": {"threshold": 1.0}}, "patch": False}, - "notify": {"slack": {"default": {"paths": None}}}, - }, - "ignore": ["^coffee.*", "^test.*"], - } - res = do_actual_validation(user_input, show_secrets_for=None) - assert res == expected_result - - -def test_validation_with_null_on_status(): - user_input = { - "coverage": {"status": {"project": {"default": None}, "patch": False}}, - "ignore": ["coffee", "test"], - } - expected_result = { - "coverage": {"status": {"project": {"default": None}, "patch": False}}, - "ignore": ["^coffee.*", "^test.*"], - } - res = do_actual_validation(user_input, show_secrets_for=None) - assert res == expected_result - - -def test_improper_layout(): - user_input = { - "coverage": {"status": {"project": {"default": None}, "patch": False}}, - "comment": {"layout": "banana,apple"}, - } - with pytest.raises(InvalidYamlException) as exc: - do_actual_validation(user_input, show_secrets_for=None) - assert exc.value.error_dict == { - "comment": [{"layout": ["Unexpected values on layout: apple,banana"]}] - } - assert exc.value.error_location == ["comment", "layout"] - - -def test_proper_layout(): - user_input = { - "coverage": {"status": {"project": {"default": None}, "patch": False}}, - "comment": {"layout": "files:10,footer"}, - } - res = do_actual_validation(user_input, show_secrets_for=None) - assert res == { - "coverage": {"status": {"project": {"default": None}, "patch": False}}, - "comment": {"layout": "files:10,footer"}, - } - - -def test_codecov_branch(): - user_input = {"codecov": {"branch": "origin/pterosaur"}} - res = do_actual_validation(user_input, show_secrets_for=None) - assert res == {"codecov": {"branch": "pterosaur"}} - - -def test_calculate_error_location_and_message_from_error_dict(): - error_dict = {"comment": [{"layout": {"deep": [[[[[{"inside": [["value"]]}]]]]]}}]} - assert ( - ["comment", "layout", "deep", "inside"], - "value", - ) == _calculate_error_location_and_message_from_error_dict(error_dict) - # case where the value is just very nested. - # This is not a requirement of any kind. This is just so - # there are no cases where a customer can send some special yaml with loops - # and make us keep parsing this forever - # It might even be overkill - assert ( - ["value", "some", "thing"], - "[[['haha']]]", - ) == _calculate_error_location_and_message_from_error_dict( - {"value": {"some": {"thing": [[[[[[[[[[[[[[[[[[[["haha"]]]]]]]]]]]]]]]]]]]]}}} - ) - - -def test_email_field_with_and_without_secret(): - user_input = { - "coverage": { - "notify": { - "email": { - "default": { - "to": [ - "example@domain.com", - "secret:v1::hfxSizNpugwZzYZXXRxxlszUetU8tyVG1HXCEdK5qeC9XhtxkCsb/Z5nvp70mp4zlbfcinTy9C9lSGXZAmN8uGKuhWnwrFPfYupe7jQ5KQY=", - ], - "threshold": "1%", - "only_pulls": False, - "layout": "reach, diff, flags", - "flags": None, - "paths": None, - } - } - } - } - } - assert do_actual_validation( - user_input, show_secrets_for=("github", "11934774", "154468867") - ) == { - "coverage": { - "notify": { - "email": { - "default": { - "to": ["example@domain.com", "secondexample@seconddomain.com"], - "threshold": 1.0, - "only_pulls": False, - "layout": "reach, diff, flags", - "flags": None, - "paths": None, - } - } - } - } - } - assert do_actual_validation(user_input, show_secrets_for=None) == { - "coverage": { - "notify": { - "email": { - "default": { - "to": [ - "example@domain.com", - "secret:v1::hfxSizNpugwZzYZXXRxxlszUetU8tyVG1HXCEdK5qeC9XhtxkCsb/Z5nvp70mp4zlbfcinTy9C9lSGXZAmN8uGKuhWnwrFPfYupe7jQ5KQY=", - ], - "threshold": 1.0, - "only_pulls": False, - "layout": "reach, diff, flags", - "flags": None, - "paths": None, - } - } - } - } - } - - -def test_assume_flags(): - # It's deprecated, but still - user_input = {"flags": {"some_flag": {"assume": {"branches": ["main"]}}}} - assert do_actual_validation( - user_input, show_secrets_for=("github", "11934774", "154468867") - ) == {"flags": {"some_flag": {"assume": {"branches": ["^main$"]}}}} - - -def test_after_n_builds_flags(): - user_input = {"flags": {"some_flag": {"after_n_builds": 5}}} - assert do_actual_validation( - user_input, show_secrets_for=("github", "11934774", "154468867") - ) == {"flags": {"some_flag": {"after_n_builds": 5}}} - - -def test_profiling_schema(): - user_input = { - "profiling": { - "fixes": ["batata_something::batata.txt"], - "grouping_attributes": ["string", "str"], - "critical_files_paths": [ - "/path/to/file.extension", - "/path/to/dir", - r"/path/{src|bin}/regex.{txt|php|cpp}", - "/path/using/globs/**/file.extension", - ], - } - } - expected_result = { - "profiling": { - "fixes": ["^batata_something::batata.txt"], - "grouping_attributes": ["string", "str"], - "critical_files_paths": [ - "^/path/to/file.extension.*", - "^/path/to/dir.*", - "^/path/{src|bin}/regex.{txt|php|cpp}.*", - "(?s:/path/using/globs/.*/file\\.extension)\\Z", - ], - } - } - result = validate_yaml(user_input) - assert result == expected_result - - -def test_components_schema(): - user_input = { - "component_management": { - "default_rules": { - "flag_regexes": ["global_flag"], - }, - "individual_components": [ - { - "name": "fruits", - "component_id": "app_0", - "flag_regexes": ["fruit_.*", "^specific_flag$"], - "paths": ["src/.*"], - "statuses": [{"type": "patch", "name_prefix": "co", "target": 90}], - } - ], - } - } - expected = { - "component_management": { - "default_rules": { - "flag_regexes": ["global_flag"], - }, - "individual_components": [ - { - "name": "fruits", - "component_id": "app_0", - "flag_regexes": ["fruit_.*", "^specific_flag$"], - "paths": ["src/.*"], - "statuses": [ - {"type": "patch", "name_prefix": "co", "target": 90.0} - ], - } - ], - } - } - result = validate_yaml(user_input) - assert result == expected - - -def test_components_schema_error(): - user_input = { - "component_management": { + "component_management": { + # "default_rules": { + # # "statuses": [ + # # { + # # "type": "project", + # # "target": "auto", + # # "branches": ["!main"] + # # } + # # ] + # }, "individual_components": [ - { - "key": "extra", - "component_id": "app_0", - "flag_regexes": ["fruit_*", "^specific_flag$"], - "path_filter_regexes": ["src/.*"], - "statuses": [ - {"type": "patch", "name_prefix": "co", "target": 90.0} - ], - }, - { - "component_id": "app_0", - "flag_regexes": ["fruit_*", "^specific_flag$"], - "path_filter_regexes": ["src/.*"], - }, - ], - } - } - with pytest.raises(InvalidYamlException) as exp: - validate_yaml(user_input) - assert exp.error_location == [ - "component_management", - "individual_components", - 0, - "key", - ] - assert exp.error_message == "unknown field" - assert exp.error_dict == { - "component_management": [ - { - "individual_components": [ - { - 0: [{"key": ["unknown field"]}], - 1: [{"component_id": ["required field"]}], - } - ] - } - ] - } - - -def test_removed_code_behavior_config_valid(): - user_input = { - "coverage": { - "status": { - "project": { - "some_status": {"removed_code_behavior": "removals_only"}, - } - } - }, - "flag_management": { - "default_rules": { - "statuses": [ - {"name_prefix": "custom", "removed_code_behavior": "adjust_base"} - ] - }, - "individual_flags": [ - { - "name": "random", - "statuses": [ - { - "name_prefix": "random-custom", - "removed_code_behavior": False, - } - ], - } - ], - }, - "component_management": { - "default_rules": { - "statuses": [ { - "name_prefix": "custom", - "removed_code_behavior": "fully_covered_patch", + "component_id": "module_nps", + "name": "Team 2", + "statuses": { + "type": "patch", + "target": "auto" + }, + "paths": ["lib/nps/**"] } ] - }, - "individual_components": [ - { - "component_id": "random", - "statuses": [ - { - "name_prefix": "random-custom", - "removed_code_behavior": "off", - } - ], - } - ], - }, - } - result = validate_yaml(user_input) - # There's no change on the valid yaml - assert result == user_input - - -def test_offset_config_error(): - user_input = { - "flag_management": { - "default_rules": { - "statuses": [ - {"name_prefix": "custom", "removed_code_behavior": "banana"} - ] } - }, - } - - with pytest.raises(InvalidYamlException) as exp: - validate_yaml(user_input) - assert exp.error_dict == { - "coverage": [ - { - "status": [ - {"patch": [{"some_status": [{"offset": ["unknown field"]}]}]} - ] - } - ], - "flag_management": [ - { - "default_rules": [ - {"statuses": [{0: [{"offset": ["unallowed value banana"]}]}]} - ] - } - ], } -def test_cli_validation(): - user_input = { - "cli": { - "plugins": {"pycoverage": {"report_type": "json"}}, - "runners": { - "custom_runner": { - "module": "my_project.runner", - "class": "MyCustomRunner", - "params": {"randseed": 0}, - } - }, - } - } - result = validate_yaml(user_input) - # There's no change on the valid yaml - assert result == user_input - - -def test_slack_app_validation(): - user_input = {"slack_app": {"enabled": True}} - result = validate_yaml(user_input) - assert result == user_input - - -def test_slack_app_validation_boolean(): - user_input = {"slack_app": True} - result = validate_yaml(user_input) - assert result == user_input - - -def test_to_string_validation(): - user_input = {"to_string": {"abc": 123}} - expected_result = {} - result = validate_yaml(user_input) - assert result == expected_result + res = validate_yaml(user_input) + print(res) + +# class TestValidationConfig(object): +# def test_validate_default_config_yaml(self, mocker): +# mocker.patch.dict(os.environ, {}, clear=True) +# mocker.patch.object( +# ConfigHelper, "load_yaml_file", side_effect=FileNotFoundError() +# ) +# this_config = ConfigHelper() +# mocker.patch("shared.config._get_config_instance", return_value=this_config) +# expected_result = { +# "codecov": {"require_ci_to_pass": True, "notify": {"wait_for_ci": True}}, +# "coverage": { +# "precision": 2, +# "round": "down", +# "range": [60.0, 80.0], +# "status": { +# "project": True, +# "patch": True, +# "changes": False, +# "default_rules": {"flag_coverage_not_uploaded_behavior": "include"}, +# }, +# }, +# "comment": { +# "layout": "reach,diff,flags,tree,reach", +# "behavior": "default", +# "show_carryforward_flags": False, +# }, +# "github_checks": {"annotations": True}, +# "slack_app": True, +# } +# res = validate_yaml( +# get_config("site", default={}), +# show_secrets_for=("github", "11934774", "154468867"), +# ) +# assert res == expected_result + + +# def test_validation_with_branches(): +# user_input = { +# "comment": { +# "require_head": True, +# "require_base": True, +# "layout": "diff", +# "require_changes": True, +# "branches": ["main"], +# "behavior": "once", +# "after_n_builds": 6, +# }, +# "coverage": { +# "status": {"project": {"default": {"threshold": "1%"}}, "patch": False} +# }, +# "github_checks": {"annotations": False}, +# "fixes": [ +# "/opt/conda/lib/python3.8/site-packages/::project/", +# "C:/Users/circleci/project/build/win_tmp/build/::project/", +# ], +# "ignore": ["coffee", "party_man", "test"], +# "codecov": {"notify": {"after_n_builds": 6}}, +# } +# expected_result = { +# "comment": { +# "require_head": True, +# "require_base": True, +# "layout": "diff", +# "require_changes": [0b001], +# "branches": ["^main$"], +# "behavior": "once", +# "after_n_builds": 6, +# }, +# "coverage": { +# "status": {"project": {"default": {"threshold": 1}}, "patch": False} +# }, +# "github_checks": {"annotations": False}, +# "fixes": [ +# "^/opt/conda/lib/python3.8/site-packages/::project/", +# "^C:/Users/circleci/project/build/win_tmp/build/::project/", +# ], +# "ignore": ["^coffee.*", "^party_man.*", "^test.*"], +# "codecov": {"notify": {"after_n_builds": 6}}, +# } +# res = do_actual_validation(user_input, show_secrets_for=None) +# assert res == expected_result + + +# def test_validation_with_flag_carryforward(): +# user_input = { +# "flags": { +# "old-flag": { +# "carryforward": True, +# "carryforward_mode": "labels", +# }, +# "other-old-flag": { +# "carryforward": True, +# "carryforward_mode": "all", +# }, +# }, +# "flag_management": { +# "individual_flags": [ +# {"name": "abcdef", "carryforward_mode": "all"}, +# {"name": "abcdef", "carryforward_mode": "labels"}, +# ] +# }, +# } +# assert do_actual_validation(user_input, show_secrets_for=None) == user_input + + +# def test_validation_with_flag_carryforward_invalid_mode(): +# user_input = { +# "flag_management": { +# "individual_flags": [ +# {"name": "abcdef", "carryforward_mode": "mario"}, +# {"name": "abcdef", "carryforward_mode": "labels"}, +# ] +# }, +# } +# with pytest.raises(InvalidYamlException) as exp: +# do_actual_validation(user_input, show_secrets_for=None) +# assert exp.value.error_dict == { +# "flag_management": [ +# { +# "individual_flags": [ +# {0: [{"carryforward_mode": ["unallowed value mario"]}]} +# ] +# } +# ] +# } + + +# def test_validation_with_null_on_paths(): +# user_input = { +# "comment": {"require_head": True, "behavior": "once", "after_n_builds": 6}, +# "coverage": { +# "status": {"project": {"default": {"threshold": "1%"}}, "patch": False}, +# "notify": {"slack": {"default": {"paths": None}}}, +# }, +# "ignore": ["coffee", "test"], +# } +# expected_result = { +# "comment": {"require_head": True, "behavior": "once", "after_n_builds": 6}, +# "coverage": { +# "status": {"project": {"default": {"threshold": 1.0}}, "patch": False}, +# "notify": {"slack": {"default": {"paths": None}}}, +# }, +# "ignore": ["^coffee.*", "^test.*"], +# } +# res = do_actual_validation(user_input, show_secrets_for=None) +# assert res == expected_result + + +# def test_validation_with_null_on_status(): +# user_input = { +# "coverage": {"status": {"project": {"default": None}, "patch": False}}, +# "ignore": ["coffee", "test"], +# } +# expected_result = { +# "coverage": {"status": {"project": {"default": None}, "patch": False}}, +# "ignore": ["^coffee.*", "^test.*"], +# } +# res = do_actual_validation(user_input, show_secrets_for=None) +# assert res == expected_result + + +# def test_improper_layout(): +# user_input = { +# "coverage": {"status": {"project": {"default": None}, "patch": False}}, +# "comment": {"layout": "banana,apple"}, +# } +# with pytest.raises(InvalidYamlException) as exc: +# do_actual_validation(user_input, show_secrets_for=None) +# assert exc.value.error_dict == { +# "comment": [{"layout": ["Unexpected values on layout: apple,banana"]}] +# } +# assert exc.value.error_location == ["comment", "layout"] + + +# def test_proper_layout(): +# user_input = { +# "coverage": {"status": {"project": {"default": None}, "patch": False}}, +# "comment": {"layout": "files:10,footer"}, +# } +# res = do_actual_validation(user_input, show_secrets_for=None) +# assert res == { +# "coverage": {"status": {"project": {"default": None}, "patch": False}}, +# "comment": {"layout": "files:10,footer"}, +# } + + +# def test_codecov_branch(): +# user_input = {"codecov": {"branch": "origin/pterosaur"}} +# res = do_actual_validation(user_input, show_secrets_for=None) +# assert res == {"codecov": {"branch": "pterosaur"}} + + +# def test_calculate_error_location_and_message_from_error_dict(): +# error_dict = {"comment": [{"layout": {"deep": [[[[[{"inside": [["value"]]}]]]]]}}]} +# assert ( +# ["comment", "layout", "deep", "inside"], +# "value", +# ) == _calculate_error_location_and_message_from_error_dict(error_dict) +# # case where the value is just very nested. +# # This is not a requirement of any kind. This is just so +# # there are no cases where a customer can send some special yaml with loops +# # and make us keep parsing this forever +# # It might even be overkill +# assert ( +# ["value", "some", "thing"], +# "[[['haha']]]", +# ) == _calculate_error_location_and_message_from_error_dict( +# {"value": {"some": {"thing": [[[[[[[[[[[[[[[[[[[["haha"]]]]]]]]]]]]]]]]]]]]}}} +# ) + + +# def test_email_field_with_and_without_secret(): +# user_input = { +# "coverage": { +# "notify": { +# "email": { +# "default": { +# "to": [ +# "example@domain.com", +# "secret:v1::hfxSizNpugwZzYZXXRxxlszUetU8tyVG1HXCEdK5qeC9XhtxkCsb/Z5nvp70mp4zlbfcinTy9C9lSGXZAmN8uGKuhWnwrFPfYupe7jQ5KQY=", +# ], +# "threshold": "1%", +# "only_pulls": False, +# "layout": "reach, diff, flags", +# "flags": None, +# "paths": None, +# } +# } +# } +# } +# } +# assert do_actual_validation( +# user_input, show_secrets_for=("github", "11934774", "154468867") +# ) == { +# "coverage": { +# "notify": { +# "email": { +# "default": { +# "to": ["example@domain.com", "secondexample@seconddomain.com"], +# "threshold": 1.0, +# "only_pulls": False, +# "layout": "reach, diff, flags", +# "flags": None, +# "paths": None, +# } +# } +# } +# } +# } +# assert do_actual_validation(user_input, show_secrets_for=None) == { +# "coverage": { +# "notify": { +# "email": { +# "default": { +# "to": [ +# "example@domain.com", +# "secret:v1::hfxSizNpugwZzYZXXRxxlszUetU8tyVG1HXCEdK5qeC9XhtxkCsb/Z5nvp70mp4zlbfcinTy9C9lSGXZAmN8uGKuhWnwrFPfYupe7jQ5KQY=", +# ], +# "threshold": 1.0, +# "only_pulls": False, +# "layout": "reach, diff, flags", +# "flags": None, +# "paths": None, +# } +# } +# } +# } +# } + + +# def test_assume_flags(): +# # It's deprecated, but still +# user_input = {"flags": {"some_flag": {"assume": {"branches": ["main"]}}}} +# assert do_actual_validation( +# user_input, show_secrets_for=("github", "11934774", "154468867") +# ) == {"flags": {"some_flag": {"assume": {"branches": ["^main$"]}}}} + + +# def test_after_n_builds_flags(): +# user_input = {"flags": {"some_flag": {"after_n_builds": 5}}} +# assert do_actual_validation( +# user_input, show_secrets_for=("github", "11934774", "154468867") +# ) == {"flags": {"some_flag": {"after_n_builds": 5}}} + + +# def test_profiling_schema(): +# user_input = { +# "profiling": { +# "fixes": ["batata_something::batata.txt"], +# "grouping_attributes": ["string", "str"], +# "critical_files_paths": [ +# "/path/to/file.extension", +# "/path/to/dir", +# r"/path/{src|bin}/regex.{txt|php|cpp}", +# "/path/using/globs/**/file.extension", +# ], +# } +# } +# expected_result = { +# "profiling": { +# "fixes": ["^batata_something::batata.txt"], +# "grouping_attributes": ["string", "str"], +# "critical_files_paths": [ +# "^/path/to/file.extension.*", +# "^/path/to/dir.*", +# "^/path/{src|bin}/regex.{txt|php|cpp}.*", +# "(?s:/path/using/globs/.*/file\\.extension)\\Z", +# ], +# } +# } +# result = validate_yaml(user_input) +# assert result == expected_result + + +# def test_components_schema(): +# user_input = { +# "component_management": { +# "default_rules": { +# "flag_regexes": ["global_flag"], +# }, +# "individual_components": [ +# { +# "name": "fruits", +# "component_id": "app_0", +# "flag_regexes": ["fruit_.*", "^specific_flag$"], +# "paths": ["src/.*"], +# "statuses": [{"type": "patch", "name_prefix": "co", "target": 90}], +# } +# ], +# } +# } +# expected = { +# "component_management": { +# "default_rules": { +# "flag_regexes": ["global_flag"], +# }, +# "individual_components": [ +# { +# "name": "fruits", +# "component_id": "app_0", +# "flag_regexes": ["fruit_.*", "^specific_flag$"], +# "paths": ["src/.*"], +# "statuses": [ +# {"type": "patch", "name_prefix": "co", "target": 90.0} +# ], +# } +# ], +# } +# } +# result = validate_yaml(user_input) +# assert result == expected + + +# def test_components_schema_error(): +# user_input = { +# "component_management": { +# "individual_components": [ +# { +# "key": "extra", +# "component_id": "app_0", +# "flag_regexes": ["fruit_*", "^specific_flag$"], +# "path_filter_regexes": ["src/.*"], +# "statuses": [ +# {"type": "patch", "name_prefix": "co", "target": 90.0} +# ], +# }, +# { +# "component_id": "app_0", +# "flag_regexes": ["fruit_*", "^specific_flag$"], +# "path_filter_regexes": ["src/.*"], +# }, +# ], +# } +# } +# with pytest.raises(InvalidYamlException) as exp: +# validate_yaml(user_input) +# assert exp.error_location == [ +# "component_management", +# "individual_components", +# 0, +# "key", +# ] +# assert exp.error_message == "unknown field" +# assert exp.error_dict == { +# "component_management": [ +# { +# "individual_components": [ +# { +# 0: [{"key": ["unknown field"]}], +# 1: [{"component_id": ["required field"]}], +# } +# ] +# } +# ] +# } + + +# def test_removed_code_behavior_config_valid(): +# user_input = { +# "coverage": { +# "status": { +# "project": { +# "some_status": {"removed_code_behavior": "removals_only"}, +# } +# } +# }, +# "flag_management": { +# "default_rules": { +# "statuses": [ +# {"name_prefix": "custom", "removed_code_behavior": "adjust_base"} +# ] +# }, +# "individual_flags": [ +# { +# "name": "random", +# "statuses": [ +# { +# "name_prefix": "random-custom", +# "removed_code_behavior": False, +# } +# ], +# } +# ], +# }, +# "component_management": { +# "default_rules": { +# "statuses": [ +# { +# "name_prefix": "custom", +# "removed_code_behavior": "fully_covered_patch", +# } +# ] +# }, +# "individual_components": [ +# { +# "component_id": "random", +# "statuses": [ +# { +# "name_prefix": "random-custom", +# "removed_code_behavior": "off", +# } +# ], +# } +# ], +# }, +# } +# result = validate_yaml(user_input) +# # There's no change on the valid yaml +# assert result == user_input + + +# def test_offset_config_error(): +# user_input = { +# "flag_management": { +# "default_rules": { +# "statuses": [ +# {"name_prefix": "custom", "removed_code_behavior": "banana"} +# ] +# } +# }, +# } + +# with pytest.raises(InvalidYamlException) as exp: +# validate_yaml(user_input) +# assert exp.error_dict == { +# "coverage": [ +# { +# "status": [ +# {"patch": [{"some_status": [{"offset": ["unknown field"]}]}]} +# ] +# } +# ], +# "flag_management": [ +# { +# "default_rules": [ +# {"statuses": [{0: [{"offset": ["unallowed value banana"]}]}]} +# ] +# } +# ], +# } + + +# def test_cli_validation(): +# user_input = { +# "cli": { +# "plugins": {"pycoverage": {"report_type": "json"}}, +# "runners": { +# "custom_runner": { +# "module": "my_project.runner", +# "class": "MyCustomRunner", +# "params": {"randseed": 0}, +# } +# }, +# } +# } +# result = validate_yaml(user_input) +# # There's no change on the valid yaml +# assert result == user_input + + +# def test_slack_app_validation(): +# user_input = {"slack_app": {"enabled": True}} +# result = validate_yaml(user_input) +# assert result == user_input + + +# def test_slack_app_validation_boolean(): +# user_input = {"slack_app": True} +# result = validate_yaml(user_input) +# assert result == user_input + + +# def test_to_string_validation(): +# user_input = {"to_string": {"abc": 123}} +# expected_result = {} +# result = validate_yaml(user_input) +# assert result == expected_result From 8e3881d76fc4633d493d97915819495adce50baf Mon Sep 17 00:00:00 2001 From: Suejung Shin Date: Fri, 28 Mar 2025 14:32:08 -0700 Subject: [PATCH 02/10] cleanup --- shared/validation/user_schema.py | 14 ++++++++++---- shared/validation/validator.py | 6 ------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/shared/validation/user_schema.py b/shared/validation/user_schema.py index 73f8d37bd..8a2217eb2 100644 --- a/shared/validation/user_schema.py +++ b/shared/validation/user_schema.py @@ -78,7 +78,8 @@ custom_status_common_config = { "name_prefix": {"type": "string", "regex": r"^[\w\-\.]+$"}, - # TODO - problem spot here + # Note that "type" is a reserved word in Cerberus parser so use with caution as a + # key in the schema. See workaround at https://github.com/codecov/shared/pull/588 "type": {"type": "string", "allowed": ("project", "patch", "changes")}, "target": percent_type_or_auto, "threshold": percent_type, @@ -120,8 +121,13 @@ flags_rule_basic_properties = { "statuses": { - "type": "list", - "schema": {"type": "dict", "schema": flag_status_attributes}, + # Use "anyof" to avoid error in Cerberus when child has a key named "type". More background at https://github.com/codecov/shared/pull/588 + "anyof": [ + { + "type": "list", + "schema": {"type": "dict", "schema": flag_status_attributes}, + }, + ] }, "carryforward_mode": { "type": "string", @@ -134,8 +140,8 @@ } component_rule_basic_properties = { - # TODO - here "statuses": { + # Use "anyof" to avoid error in Cerberus when child has a key named "type". More background at https://github.com/codecov/shared/pull/588 "anyof": [ { "type": "list", diff --git a/shared/validation/validator.py b/shared/validation/validator.py index c7c372b84..69c5ccf43 100644 --- a/shared/validation/validator.py +++ b/shared/validation/validator.py @@ -60,9 +60,3 @@ def _validate_comma_separated_strings(self, constraint, field, value): LayoutStructure().validate(value) except Invalid as exc: self._error(field, exc.error_message) - - # def _validate_return_error_when_dict(self, constraint, field, value): - # """{'type': 'boolean'}""" - - # print("field", field) - # self._error(field, "cannot be a dict") From 67d2d1975ec5615988e54c3c550667c0dbac963b Mon Sep 17 00:00:00 2001 From: Suejung Shin Date: Tue, 8 Apr 2025 10:13:53 -0700 Subject: [PATCH 03/10] cleanup --- shared/validation/user_schema.py | 2 +- shared/yaml/validation.py | 12 +- tests/unit/validation/test_validation.py | 2849 +++++++++++----------- 3 files changed, 1392 insertions(+), 1471 deletions(-) diff --git a/shared/validation/user_schema.py b/shared/validation/user_schema.py index 8a2217eb2..c42b436d2 100644 --- a/shared/validation/user_schema.py +++ b/shared/validation/user_schema.py @@ -79,7 +79,7 @@ custom_status_common_config = { "name_prefix": {"type": "string", "regex": r"^[\w\-\.]+$"}, # Note that "type" is a reserved word in Cerberus parser so use with caution as a - # key in the schema. See workaround at https://github.com/codecov/shared/pull/588 + # key in the schema. See workaround at places that call this schema. "type": {"type": "string", "allowed": ("project", "patch", "changes")}, "target": percent_type_or_auto, "threshold": percent_type, diff --git a/shared/yaml/validation.py b/shared/yaml/validation.py index c92a70c73..42631c30c 100644 --- a/shared/yaml/validation.py +++ b/shared/yaml/validation.py @@ -159,17 +159,7 @@ def decode(cls, value, expected_prefix): def do_actual_validation(yaml_dict, show_secrets_for): validator = CodecovUserYamlValidator(show_secrets_for=show_secrets_for) full_schema = {**user_schema, **cli_schema} - - try: - is_valid = validator.validate(yaml_dict, full_schema) - except Exception as e: - print("hello") - print(e) - - print(validator._errors) - raise e - - + is_valid = validator.validate(yaml_dict, full_schema) if not is_valid: error_dict = validator.errors ( diff --git a/tests/unit/validation/test_validation.py b/tests/unit/validation/test_validation.py index de0f43a51..75c2fec0d 100644 --- a/tests/unit/validation/test_validation.py +++ b/tests/unit/validation/test_validation.py @@ -7,931 +7,481 @@ from shared.validation.exceptions import InvalidYamlException from shared.yaml.validation import ( _calculate_error_location_and_message_from_error_dict, - do_actual_validation, validate_yaml) + do_actual_validation, + validate_yaml, +) from tests.base import BaseTestCase class TestUserYamlValidation(BaseTestCase): - # def test_empty_case(self): - # user_input = {} - # expected_result = {} - # assert validate_yaml(user_input) == expected_result - - # @pytest.mark.parametrize("input_value", ["", 10, [], tuple(), set()]) - # def test_wrong_object_type(self, input_value): - # with pytest.raises(InvalidYamlException) as exc: - # validate_yaml(input_value) - # exception = exc.value - # assert exception.error_location == [] - # assert exception.error_message == "Yaml needs to be a dict" - # assert exception.original_exc is None - - # @pytest.mark.parametrize( - # "user_input, expected_result", - # [ - # ( - # { - # "coverage": {"status": {"patch": True, "project": False}}, - # "comment": False, - # }, - # { - # "comment": False, - # "coverage": {"status": {"patch": True, "project": False}}, - # }, - # ), - # ( - # { - # "codecov": {"bot": "codecov-io", "require_ci_to_pass": False}, - # "coverage": { - # "status": { - # "project": { - # "default": {"target": "78%", "threshold": "5%"} - # }, - # "patch": {"default": {"target": "75%"}}, - # } - # }, - # }, - # { - # "codecov": {"bot": "codecov-io", "require_ci_to_pass": False}, - # "coverage": { - # "status": { - # "project": {"default": {"target": 78.0, "threshold": 5.0}}, - # "patch": {"default": {"target": 75.0}}, - # } - # }, - # }, - # ), - # ( - # { - # "coverage": { - # "status": { - # "project": False, - # "patch": { - # "default": {"informational": True}, - # "ui": {"informational": True}, - # }, - # "changes": False, - # } - # }, - # "comment": False, - # "flags": {"ui": {"paths": ["/ui-v2/"]}}, - # "github_checks": {"annotations": False}, - # "ignore": [ - # "agent/uiserver/bindata_assetfs.go", - # "vendor/**/*", - # "**/*.pb.go", - # ], - # }, - # { - # "coverage": { - # "status": { - # "project": False, - # "patch": { - # "default": {"informational": True}, - # "ui": {"informational": True}, - # }, - # "changes": False, - # } - # }, - # "comment": False, - # "flags": {"ui": {"paths": ["^/ui-v2/.*"]}}, - # "github_checks": {"annotations": False}, - # "ignore": [ - # "^agent/uiserver/bindata_assetfs.go.*", - # "(?s:vendor/.*/[^\\/]*)\\Z", - # "(?s:.*/[^\\/]*\\.pb\\.go)\\Z", - # ], - # }, - # ), - # ( - # { - # "comment": { - # "require_head": True, - # "require_base": True, - # "layout": "diff", - # "require_changes": True, - # "branches": ["main"], - # "behavior": "once", - # "after_n_builds": 6, - # "hide_project_coverage": True, - # }, - # "coverage": { - # "status": { - # "project": {"default": {"threshold": "1%"}}, - # "patch": False, - # } - # }, - # "github_checks": {"annotations": False}, - # "fixes": [ - # "/opt/conda/lib/python3.8/site-packages/::project/", - # "C:/Users/circleci/project/build/win_tmp/build/::project/", - # ], - # "ignore": ["coffee", "party_man", "test"], - # "codecov": {"notify": {"after_n_builds": 6}}, - # }, - # { - # "comment": { - # "require_head": True, - # "require_base": True, - # "layout": "diff", - # "require_changes": [0b001], - # "branches": ["^main$"], - # "behavior": "once", - # "after_n_builds": 6, - # "hide_project_coverage": True, - # }, - # "coverage": { - # "status": { - # "project": {"default": {"threshold": 1}}, - # "patch": False, - # } - # }, - # "github_checks": {"annotations": False}, - # "fixes": [ - # "^/opt/conda/lib/python3.8/site-packages/::project/", - # "^C:/Users/circleci/project/build/win_tmp/build/::project/", - # ], - # "ignore": ["^coffee.*", "^party_man.*", "^test.*"], - # "codecov": {"notify": {"after_n_builds": 6}}, - # }, - # ), - # ( - # { - # "ignore": ["js/plugins", "plugins"], - # "coverage": { - # "notify": { - # "slack": { - # "default": { - # "url": "https://hooks.slack.com/services/testdazzd/testrf4k6py/test72nq0j0ke3prs2fdvfuj", - # "only_pulls": False, - # "branches": ["main", "qa", "dev"], - # } - # } - # } - # }, - # }, - # { - # "ignore": ["^js/plugins.*", "^plugins.*"], - # "coverage": { - # "notify": { - # "slack": { - # "default": { - # "url": "https://hooks.slack.com/services/testdazzd/testrf4k6py/test72nq0j0ke3prs2fdvfuj", - # "only_pulls": False, - # "branches": ["^main$", "^qa$", "^dev$"], - # } - # } - # } - # }, - # }, - # ), - # ( - # { - # "codecov": {"notify": {"manual_trigger": True}}, - # }, - # { - # "codecov": {"notify": {"manual_trigger": True}}, - # }, - # ), - # ], - # ) - # def test_random_real_life_cases(self, user_input, expected_result): - # # Some random cases based on real world examples - # assert expected_result == validate_yaml(user_input) - - # def test_case_with_experimental_turned_on_valid(self, mocker): - # mocker.patch.dict(os.environ, {"CERBERUS_VALIDATOR_RATE": "1.0"}) - # user_input = {"coverage": {"status": {"patch": True}}} - # expected_result = {"coverage": {"status": {"patch": True}}} - # assert expected_result == validate_yaml(user_input) - - # def test_case_with_experimental_turned_invalid(self, mocker): - # mocker.patch.dict(os.environ, {"CERBERUS_VALIDATOR_RATE": "1.0"}) - # user_input = {"coverage": {"status": {"patch": "banana"}}} - # with pytest.raises(InvalidYamlException) as ex: - # validate_yaml(user_input) - # assert ex.value.error_location == ["coverage", "status", "patch"] - # assert ex.value.error_message == "must be of ['dict', 'boolean'] type" - - # def test_many_flags_validation(self): - # user_input = { - # "codecov": {"max_report_age": False, "notify": {"after_n_builds": 10}}, - # "comment": {"layout": "reach,footer"}, - # "coverage": { - # "status": { - # "patch": { - # "default": True, - # "numbers_only": {"paths": ["wildwest/numbers.py"]}, - # "strings_only": {"paths": ["wildwest/strings.py"]}, - # "one": {"flags": ["one"]}, - # "two": {"flags": ["two"]}, - # "three": {"flags": ["three"]}, - # "four": {"flags": ["four"]}, - # "five": {"flags": ["five"]}, - # "six": {"flags": ["six"]}, - # "seven": {"flags": ["seven"]}, - # "eight": {"flags": ["eight"]}, - # "nine": {"flags": ["nine"]}, - # "ten": {"flags": ["ten"]}, - # "eleven": {"flags": ["eleven"]}, - # "one_without_t": {"flags": ["one"], "paths": ["wildwest"]}, - # "two_without_t": {"flags": ["two"], "paths": ["wildwest"]}, - # "three_without_t": {"flags": ["three"], "paths": ["wildwest"]}, - # "four_without_t": {"flags": ["four"], "paths": ["wildwest"]}, - # "five_without_t": {"flags": ["five"], "paths": ["wildwest"]}, - # "six_without_t": {"flags": ["six"], "paths": ["wildwest"]}, - # "seven_without_t": {"flags": ["seven"], "paths": ["wildwest"]}, - # "eight_without_t": {"flags": ["eight"], "paths": ["wildwest"]}, - # "nine_without_t": {"flags": ["nine"], "paths": ["wildwest"]}, - # "ten_without_t": {"flags": ["ten"], "paths": ["wildwest"]}, - # "eleven_without_t": { - # "flags": ["eleven"], - # "paths": ["wildwest"], - # }, - # }, - # "project": { - # "default": True, - # "numbers_only": {"paths": ["wildwest/numbers.py"]}, - # "strings_only": {"paths": ["wildwest/strings.py"]}, - # "one": {"flags": ["one"]}, - # "two": {"flags": ["two"]}, - # "three": {"flags": ["three"]}, - # "four": {"flags": ["four"]}, - # "five": {"flags": ["five"]}, - # "six": {"flags": ["six"]}, - # "seven": {"flags": ["seven"]}, - # "eight": {"flags": ["eight"]}, - # "nine": {"flags": ["nine"]}, - # "ten": {"flags": ["ten"]}, - # "eleven": {"flags": ["eleven"]}, - # "one_without_t": {"flags": ["one"], "paths": ["wildwest"]}, - # "two_without_t": {"flags": ["two"], "paths": ["wildwest"]}, - # "three_without_t": {"flags": ["three"], "paths": ["wildwest"]}, - # "four_without_t": {"flags": ["four"], "paths": ["wildwest"]}, - # "five_without_t": {"flags": ["five"], "paths": ["wildwest"]}, - # "six_without_t": {"flags": ["six"], "paths": ["wildwest"]}, - # "seven_without_t": {"flags": ["seven"], "paths": ["wildwest"]}, - # "eight_without_t": {"flags": ["eight"], "paths": ["wildwest"]}, - # "nine_without_t": {"flags": ["nine"], "paths": ["wildwest"]}, - # "ten_without_t": {"flags": ["ten"], "paths": ["wildwest"]}, - # "eleven_without_t": { - # "flags": ["eleven"], - # "paths": ["wildwest"], - # }, - # }, - # "changes": { - # "numbers_only": {"paths": ["wildwest/numbers.py"]}, - # "strings_only": {"paths": ["wildwest/strings.py"]}, - # "default": True, - # "one": {"flags": ["one"]}, - # "two": {"flags": ["two"]}, - # "three": {"flags": ["three"]}, - # "four": {"flags": ["four"]}, - # "five": {"flags": ["five"]}, - # "six": {"flags": ["six"]}, - # "seven": {"flags": ["seven"]}, - # "eight": {"flags": ["eight"]}, - # "nine": {"flags": ["nine"]}, - # "ten": {"flags": ["ten"]}, - # "eleven": {"flags": ["eleven"]}, - # "one_without_t": {"flags": ["one"], "paths": ["wildwest"]}, - # "two_without_t": {"flags": ["two"], "paths": ["wildwest"]}, - # "three_without_t": {"flags": ["three"], "paths": ["wildwest"]}, - # "four_without_t": {"flags": ["four"], "paths": ["wildwest"]}, - # "five_without_t": {"flags": ["five"], "paths": ["wildwest"]}, - # "six_without_t": {"flags": ["six"], "paths": ["wildwest"]}, - # "seven_without_t": {"flags": ["seven"], "paths": ["wildwest"]}, - # "eight_without_t": {"flags": ["eight"], "paths": ["wildwest"]}, - # "nine_without_t": {"flags": ["nine"], "paths": ["wildwest"]}, - # "ten_without_t": {"flags": ["ten"], "paths": ["wildwest"]}, - # "eleven_without_t": { - # "flags": ["eleven"], - # "paths": ["wildwest"], - # }, - # }, - # } - # }, - # "flag_management": { - # "default_rules": { - # "carryforward": False, - # "statuses": [{"name_prefix": "aaa", "type": "patch"}], - # }, - # "individual_flags": [ - # {"name": "cawcaw", "paths": ["banana"], "after_n_builds": 3} - # ], - # }, - # } - # expected_result = { - # "codecov": {"max_report_age": False, "notify": {"after_n_builds": 10}}, - # "comment": {"layout": "reach,footer"}, - # "coverage": { - # "status": { - # "patch": { - # "default": True, - # "numbers_only": {"paths": ["^wildwest/numbers.py.*"]}, - # "strings_only": {"paths": ["^wildwest/strings.py.*"]}, - # "one": {"flags": ["one"]}, - # "two": {"flags": ["two"]}, - # "three": {"flags": ["three"]}, - # "four": {"flags": ["four"]}, - # "five": {"flags": ["five"]}, - # "six": {"flags": ["six"]}, - # "seven": {"flags": ["seven"]}, - # "eight": {"flags": ["eight"]}, - # "nine": {"flags": ["nine"]}, - # "ten": {"flags": ["ten"]}, - # "eleven": {"flags": ["eleven"]}, - # "one_without_t": {"flags": ["one"], "paths": ["^wildwest.*"]}, - # "two_without_t": {"flags": ["two"], "paths": ["^wildwest.*"]}, - # "three_without_t": { - # "flags": ["three"], - # "paths": ["^wildwest.*"], - # }, - # "four_without_t": {"flags": ["four"], "paths": ["^wildwest.*"]}, - # "five_without_t": {"flags": ["five"], "paths": ["^wildwest.*"]}, - # "six_without_t": {"flags": ["six"], "paths": ["^wildwest.*"]}, - # "seven_without_t": { - # "flags": ["seven"], - # "paths": ["^wildwest.*"], - # }, - # "eight_without_t": { - # "flags": ["eight"], - # "paths": ["^wildwest.*"], - # }, - # "nine_without_t": {"flags": ["nine"], "paths": ["^wildwest.*"]}, - # "ten_without_t": {"flags": ["ten"], "paths": ["^wildwest.*"]}, - # "eleven_without_t": { - # "flags": ["eleven"], - # "paths": ["^wildwest.*"], - # }, - # }, - # "project": { - # "default": True, - # "numbers_only": {"paths": ["^wildwest/numbers.py.*"]}, - # "strings_only": {"paths": ["^wildwest/strings.py.*"]}, - # "one": {"flags": ["one"]}, - # "two": {"flags": ["two"]}, - # "three": {"flags": ["three"]}, - # "four": {"flags": ["four"]}, - # "five": {"flags": ["five"]}, - # "six": {"flags": ["six"]}, - # "seven": {"flags": ["seven"]}, - # "eight": {"flags": ["eight"]}, - # "nine": {"flags": ["nine"]}, - # "ten": {"flags": ["ten"]}, - # "eleven": {"flags": ["eleven"]}, - # "one_without_t": {"flags": ["one"], "paths": ["^wildwest.*"]}, - # "two_without_t": {"flags": ["two"], "paths": ["^wildwest.*"]}, - # "three_without_t": { - # "flags": ["three"], - # "paths": ["^wildwest.*"], - # }, - # "four_without_t": {"flags": ["four"], "paths": ["^wildwest.*"]}, - # "five_without_t": {"flags": ["five"], "paths": ["^wildwest.*"]}, - # "six_without_t": {"flags": ["six"], "paths": ["^wildwest.*"]}, - # "seven_without_t": { - # "flags": ["seven"], - # "paths": ["^wildwest.*"], - # }, - # "eight_without_t": { - # "flags": ["eight"], - # "paths": ["^wildwest.*"], - # }, - # "nine_without_t": {"flags": ["nine"], "paths": ["^wildwest.*"]}, - # "ten_without_t": {"flags": ["ten"], "paths": ["^wildwest.*"]}, - # "eleven_without_t": { - # "flags": ["eleven"], - # "paths": ["^wildwest.*"], - # }, - # }, - # "changes": { - # "numbers_only": {"paths": ["^wildwest/numbers.py.*"]}, - # "strings_only": {"paths": ["^wildwest/strings.py.*"]}, - # "default": True, - # "one": {"flags": ["one"]}, - # "two": {"flags": ["two"]}, - # "three": {"flags": ["three"]}, - # "four": {"flags": ["four"]}, - # "five": {"flags": ["five"]}, - # "six": {"flags": ["six"]}, - # "seven": {"flags": ["seven"]}, - # "eight": {"flags": ["eight"]}, - # "nine": {"flags": ["nine"]}, - # "ten": {"flags": ["ten"]}, - # "eleven": {"flags": ["eleven"]}, - # "one_without_t": {"flags": ["one"], "paths": ["^wildwest.*"]}, - # "two_without_t": {"flags": ["two"], "paths": ["^wildwest.*"]}, - # "three_without_t": { - # "flags": ["three"], - # "paths": ["^wildwest.*"], - # }, - # "four_without_t": {"flags": ["four"], "paths": ["^wildwest.*"]}, - # "five_without_t": {"flags": ["five"], "paths": ["^wildwest.*"]}, - # "six_without_t": {"flags": ["six"], "paths": ["^wildwest.*"]}, - # "seven_without_t": { - # "flags": ["seven"], - # "paths": ["^wildwest.*"], - # }, - # "eight_without_t": { - # "flags": ["eight"], - # "paths": ["^wildwest.*"], - # }, - # "nine_without_t": {"flags": ["nine"], "paths": ["^wildwest.*"]}, - # "ten_without_t": {"flags": ["ten"], "paths": ["^wildwest.*"]}, - # "eleven_without_t": { - # "flags": ["eleven"], - # "paths": ["^wildwest.*"], - # }, - # }, - # } - # }, - # "flag_management": { - # "default_rules": { - # "carryforward": False, - # "statuses": [{"name_prefix": "aaa", "type": "patch"}], - # }, - # "individual_flags": [ - # {"name": "cawcaw", "paths": ["^banana.*"], "after_n_builds": 3} - # ], - # }, - # } - # assert validate_yaml(user_input) == expected_result - - # def test_validate_bot_none(self): - # user_input = {"codecov": {"bot": None}} - # expected_result = {"codecov": {"bot": None}} - # result = validate_yaml(user_input) - # assert result == expected_result - - # def test_validate_flag_too_long(self): - # user_input = {"flags": {"abcdefg" * 500: {"paths": ["banana"]}}} - # with pytest.raises(InvalidYamlException) as exc: - # validate_yaml(user_input) - # assert exc.value.error_location == [ - # "flags", - # "abcdefg" * 500, - # ] - - # def test_validate_parser_only_field(self): - # user_input = {"parsers": {"go": {"partials_as_hits": True}}} - # expected_result = {"parsers": {"go": {"partials_as_hits": True}}} - # result = validate_yaml(user_input) - # assert result == expected_result - - # def test_simple_case(self): - # encoded_value = "secret:v1::zsV9A8pHadNle357DGJHbZCTyCYA+TXdUd9TN3IY2DIWcPOtgK3Pg1EgA6OZr9XJ1EsdpL765yWrN4pfR3elRdN2LUwiuv6RkNjpbiruHx45agsgxdu8fi24p5pkCLvjcW0HqdH2PTvmHauIp+ptgA==" - # user_input = { - # "coverage": { - # "precision": 2, - # "round": "down", - # "range": "70...100", - # "status": { - # "project": { - # "custom_project": { - # "carryforward_behavior": "exclude", - # "flag_coverage_not_uploaded_behavior": "exclude", - # } - # }, - # "patch": True, - # "changes": False, - # "default_rules": { - # "carryforward_behavior": "pass", - # "flag_coverage_not_uploaded_behavior": "pass", - # }, - # "no_upload_behavior": "pass", - # }, - # "notify": {"irc": {"user_given_title": {"password": encoded_value}}}, - # }, - # "codecov": {"notify": {"require_ci_to_pass": True}}, - # "comment": { - # "behavior": "default", - # "layout": "header, diff", - # "require_changes": False, - # "show_carryforward_flags": False, - # }, - # "parsers": { - # "gcov": { - # "branch_detection": { - # "conditional": True, - # "loop": True, - # "macro": False, - # "method": False, - # } - # }, - # "jacoco": {"partials_as_hits": True}, - # }, - # "cli": { - # "plugins": {"pycoverage": {"report_type": "json"}}, - # }, - # } - # expected_result = { - # "coverage": { - # "precision": 2, - # "round": "down", - # "range": [70, 100], - # "status": { - # "project": { - # "custom_project": { - # "carryforward_behavior": "exclude", - # "flag_coverage_not_uploaded_behavior": "exclude", - # } - # }, - # "patch": True, - # "changes": False, - # "default_rules": { - # "carryforward_behavior": "pass", - # "flag_coverage_not_uploaded_behavior": "pass", - # }, - # "no_upload_behavior": "pass", - # }, - # "notify": {"irc": {"user_given_title": {"password": encoded_value}}}, - # }, - # "codecov": {"notify": {}, "require_ci_to_pass": True}, - # "comment": { - # "behavior": "default", - # "layout": "header, diff", - # "require_changes": [0b000], - # "show_carryforward_flags": False, - # }, - # "parsers": { - # "gcov": { - # "branch_detection": { - # "conditional": True, - # "loop": True, - # "macro": False, - # "method": False, - # } - # }, - # "jacoco": {"partials_as_hits": True}, - # }, - # "cli": { - # "plugins": {"pycoverage": {"report_type": "json"}}, - # }, - # } - # assert validate_yaml(user_input) == expected_result - - # def test_negative_notify_after_n_builds(self): - # user_input = {"codecov": {"notify": {"after_n_builds": -1}}} - # with pytest.raises(InvalidYamlException) as exc: - # validate_yaml(user_input) - # assert exc.value.error_location == ["codecov", "notify", "after_n_builds"] - # assert exc.value.error_message == "min value is 0" - - # def test_positive_notify_after_n_builds(self): - # user_input = {"codecov": {"notify": {"after_n_builds": 1}}} - # res = validate_yaml(user_input) - # assert res == {"codecov": {"notify": {"after_n_builds": 1}}} - - # def test_negative_comments_after_n_builds(self): - # user_input = {"comment": {"after_n_builds": -1}} - # with pytest.raises(InvalidYamlException) as exc: - # validate_yaml(user_input) - # assert exc.value.error_location == ["comment", "after_n_builds"] - # assert exc.value.error_message == "min value is 0" - - # def test_invalid_yaml_case(self): - # user_input = { - # "coverage": { - # "round": "down", - # "precision": 2, - # "range": "70...100", - # "status": {"project": {"base": "auto", "aa": True}}, - # }, - # "ignore": ["Pods/.*"], - # } - # with pytest.raises(InvalidYamlException) as exc: - # validate_yaml(user_input) - # assert exc.value.error_location == ["coverage", "status", "project", "base"] - # assert exc.value.error_message == "must be of ['dict', 'boolean'] type" - - # def test_invalid_yaml_case_custom_validator(self): - # user_input = { - # "coverage": { - # "round": "down", - # "precision": 2, - # "range": "70...5000", - # "status": {"project": {"percent": "abc"}}, - # }, - # "ignore": ["Pods/.*"], - # } - # with pytest.raises(InvalidYamlException) as exc: - # validate_yaml(user_input) - # assert exc.value.error_location == ["coverage", "range"] - # assert exc.value.error_message == "must be of list type" - - # def test_invalid_yaml_case_no_upload_behavior(self): - # user_input = { - # "coverage": { - # "round": "down", - # "precision": 2, - # "range": "70...100", - # "status": { - # "project": {"percent": "abc"}, - # "no_upload_behavior": "no-pass", - # }, - # }, - # "ignore": ["Pods/.*"], - # } - # with pytest.raises(InvalidYamlException) as exc: - # validate_yaml(user_input) - # assert exc.value.error_location == ["coverage", "status", "no_upload_behavior"] - # assert exc.value.error_message == "unallowed value no-pass" - - # def test_yaml_with_null_threshold(self): - # user_input = { - # "codecov": {"notify": {}, "require_ci_to_pass": True}, - # "comment": { - # "behavior": "default", - # "branches": None, - # "layout": "reach, diff, flags, files", - # "require_base": False, - # "require_changes": False, - # "require_head": False, - # }, - # "coverage": { - # "precision": 2, - # "range": "50...80", - # "round": "down", - # "status": { - # "changes": False, - # "patch": True, - # "project": { - # "default": {"target": "auto", "threshold": None, "base": "auto"} - # }, - # }, - # }, - # } - # res = validate_yaml(user_input) - # expected_result = { - # "codecov": {"notify": {}, "require_ci_to_pass": True}, - # "comment": { - # "behavior": "default", - # "branches": None, - # "layout": "reach, diff, flags, files", - # "require_base": False, - # "require_changes": [0b000], - # "require_head": False, - # }, - # "coverage": { - # "precision": 2, - # "range": [50.0, 80.0], - # "round": "down", - # "status": { - # "changes": False, - # "patch": True, - # "project": { - # "default": {"target": "auto", "threshold": None, "base": "auto"} - # }, - # }, - # }, - # } - # assert res == expected_result - - # def test_yaml_with_status_case(self): - # user_input = { - # "coverage": { - # "round": "down", - # "precision": 2, - # "range": "70...100", - # "status": {"project": {"default": {"base": "auto"}}}, - # }, - # "ignore": ["Pods/.*"], - # } - # expected_result = { - # "coverage": { - # "round": "down", - # "precision": 2, - # "range": [70.0, 100.0], - # "status": {"project": {"default": {"base": "auto"}}}, - # }, - # "ignore": ["Pods/.*"], - # } - # result = validate_yaml(user_input) - # assert result == expected_result - - # def test_yaml_with_flag_management(self): - # user_input = { - # "flag_management": { - # "default_rules": { - # "carryforward": True, - # "statuses": [ - # { - # "type": "project", - # "name_prefix": "healthcare", - # "threshold": 80, - # } - # ], - # }, - # "individual_flags": [ - # { - # "name": "flag_banana", - # "statuses": [ - # { - # "type": "patch", - # "name_prefix": "alliance", - # "flag_coverage_not_uploaded_behavior": "include", - # } - # ], - # } - # ], - # } - # } - # expected_result = { - # "flag_management": { - # "individual_flags": [ - # { - # "name": "flag_banana", - # "statuses": [ - # { - # "type": "patch", - # "name_prefix": "alliance", - # "flag_coverage_not_uploaded_behavior": "include", - # } - # ], - # } - # ], - # "default_rules": { - # "carryforward": True, - # "statuses": [ - # { - # "type": "project", - # "name_prefix": "healthcare", - # "threshold": 80.0, - # } - # ], - # }, - # } - # } - # result = validate_yaml(user_input) - # assert result == expected_result - - # def test_yaml_with_flag_management_statuses_with_flags(self): - # user_input = { - # "flag_management": { - # "default_rules": { - # "carryforward": True, - # "statuses": [ - # { - # "type": "project", - # "name_prefix": "healthcare", - # "threshold": 80, - # "flags": ["hahaha"], - # } - # ], - # }, - # "individual_flags": [ - # { - # "name": "flag_banana", - # "statuses": [ - # { - # "type": "patch", - # "name_prefix": "alliance", - # "flag_coverage_not_uploaded_behavior": "include", - # } - # ], - # } - # ], - # } - # } - # with pytest.raises(InvalidYamlException) as exc: - # validate_yaml(user_input) - # assert exc.value.error_location == [ - # "flag_management", - # "default_rules", - # "statuses", - # 0, - # "flags", - # ] - # assert exc.value.error_message == "extra keys not allowed" - - # def test_github_checks(self): - # user_input = {"github_checks": True} - # expected_result = {"github_checks": True} - # assert validate_yaml(user_input) == expected_result - # user_input = {"github_checks": {"annotations": False}} - # expected_result = {"github_checks": {"annotations": False}} - # assert validate_yaml(user_input) == expected_result - - # def test_validate_jacoco_partials(self): - # user_input = {"parsers": {"jacoco": {"partials_as_hits": True}}} - # expected_result = {"parsers": {"jacoco": {"partials_as_hits": True}}} - # result = validate_yaml(user_input) - # assert result == expected_result - - # @pytest.mark.parametrize( - # "input, expected", - # [ - # pytest.param( - # {"comment": {"require_bundle_changes": False}}, - # {"comment": {"require_bundle_changes": False}}, - # id="no_bundle_changes_required", - # ), - # pytest.param( - # { - # "comment": { - # "require_bundle_changes": True, - # "bundle_change_threshold": 1200, - # } - # }, - # { - # "comment": { - # "require_bundle_changes": True, - # "bundle_change_threshold": ("absolute", 1200), - # } - # }, - # id="bundle_changes_with_threshold", - # ), - # pytest.param( - # { - # "comment": { - # "require_bundle_changes": "bundle_increase", - # "bundle_change_threshold": "1mb", - # } - # }, - # { - # "comment": { - # "require_bundle_changes": "bundle_increase", - # "bundle_change_threshold": ("absolute", 1000000), - # } - # }, - # id="bundle_increase_required_with_threshold", - # ), - # pytest.param( - # { - # "comment": { - # "require_bundle_changes": "bundle_increase", - # "bundle_change_threshold": "10%", - # } - # }, - # { - # "comment": { - # "require_bundle_changes": "bundle_increase", - # "bundle_change_threshold": ("percentage", 10.0), - # } - # }, - # id="bundle_increase_required_with_percentage_threshold", - # ), - # ], - # ) - # def test_bundle_analysis_comment_config(self, input, expected, mocker): - # mocker.patch.object(BUNDLE_THRESHOLD_FLAG, "check_value", return_value=True) - # result = validate_yaml(input) - # assert result == expected - - # @pytest.mark.parametrize( - # "input, expected", - # [ - # pytest.param( - # { - # "bundle_analysis": { - # "status": False, - # "warning_threshold": "10%", - # } - # }, - # { - # "bundle_analysis": { - # "status": False, - # "warning_threshold": ("percentage", 10.0), - # } - # }, - # id="status_off_percentage_threshold", - # ), - # pytest.param( - # { - # "bundle_analysis": { - # "status": True, - # "warning_threshold": "10kb", - # } - # }, - # { - # "bundle_analysis": { - # "status": True, - # "warning_threshold": ("absolute", 10000), - # } - # }, - # id="status_on_absolute_threshold", - # ), - # pytest.param( - # { - # "bundle_analysis": { - # "status": "informational", - # } - # }, - # { - # "bundle_analysis": { - # "status": "informational", - # } - # }, - # id="status_informational_no_threshold", - # ), - # ], - # ) - # def test_bundle_analysis_config(self, input, expected, mocker): - # mocker.patch.object(BUNDLE_THRESHOLD_FLAG, "check_value", return_value=True) - # result = validate_yaml(input) - # assert result == expected - - def test_bad_yaml(self): + def test_empty_case(self): + user_input = {} + expected_result = {} + assert validate_yaml(user_input) == expected_result + + @pytest.mark.parametrize("input_value", ["", 10, [], tuple(), set()]) + def test_wrong_object_type(self, input_value): + with pytest.raises(InvalidYamlException) as exc: + validate_yaml(input_value) + exception = exc.value + assert exception.error_location == [] + assert exception.error_message == "Yaml needs to be a dict" + assert exception.original_exc is None + + @pytest.mark.parametrize( + "user_input, expected_result", + [ + ( + { + "coverage": {"status": {"patch": True, "project": False}}, + "comment": False, + }, + { + "comment": False, + "coverage": {"status": {"patch": True, "project": False}}, + }, + ), + ( + { + "codecov": {"bot": "codecov-io", "require_ci_to_pass": False}, + "coverage": { + "status": { + "project": { + "default": {"target": "78%", "threshold": "5%"} + }, + "patch": {"default": {"target": "75%"}}, + } + }, + }, + { + "codecov": {"bot": "codecov-io", "require_ci_to_pass": False}, + "coverage": { + "status": { + "project": {"default": {"target": 78.0, "threshold": 5.0}}, + "patch": {"default": {"target": 75.0}}, + } + }, + }, + ), + ( + { + "coverage": { + "status": { + "project": False, + "patch": { + "default": {"informational": True}, + "ui": {"informational": True}, + }, + "changes": False, + } + }, + "comment": False, + "flags": {"ui": {"paths": ["/ui-v2/"]}}, + "github_checks": {"annotations": False}, + "ignore": [ + "agent/uiserver/bindata_assetfs.go", + "vendor/**/*", + "**/*.pb.go", + ], + }, + { + "coverage": { + "status": { + "project": False, + "patch": { + "default": {"informational": True}, + "ui": {"informational": True}, + }, + "changes": False, + } + }, + "comment": False, + "flags": {"ui": {"paths": ["^/ui-v2/.*"]}}, + "github_checks": {"annotations": False}, + "ignore": [ + "^agent/uiserver/bindata_assetfs.go.*", + "(?s:vendor/.*/[^\\/]*)\\Z", + "(?s:.*/[^\\/]*\\.pb\\.go)\\Z", + ], + }, + ), + ( + { + "comment": { + "require_head": True, + "require_base": True, + "layout": "diff", + "require_changes": True, + "branches": ["main"], + "behavior": "once", + "after_n_builds": 6, + "hide_project_coverage": True, + }, + "coverage": { + "status": { + "project": {"default": {"threshold": "1%"}}, + "patch": False, + } + }, + "github_checks": {"annotations": False}, + "fixes": [ + "/opt/conda/lib/python3.8/site-packages/::project/", + "C:/Users/circleci/project/build/win_tmp/build/::project/", + ], + "ignore": ["coffee", "party_man", "test"], + "codecov": {"notify": {"after_n_builds": 6}}, + }, + { + "comment": { + "require_head": True, + "require_base": True, + "layout": "diff", + "require_changes": [0b001], + "branches": ["^main$"], + "behavior": "once", + "after_n_builds": 6, + "hide_project_coverage": True, + }, + "coverage": { + "status": { + "project": {"default": {"threshold": 1}}, + "patch": False, + } + }, + "github_checks": {"annotations": False}, + "fixes": [ + "^/opt/conda/lib/python3.8/site-packages/::project/", + "^C:/Users/circleci/project/build/win_tmp/build/::project/", + ], + "ignore": ["^coffee.*", "^party_man.*", "^test.*"], + "codecov": {"notify": {"after_n_builds": 6}}, + }, + ), + ( + { + "ignore": ["js/plugins", "plugins"], + "coverage": { + "notify": { + "slack": { + "default": { + "url": "https://hooks.slack.com/services/testdazzd/testrf4k6py/test72nq0j0ke3prs2fdvfuj", + "only_pulls": False, + "branches": ["main", "qa", "dev"], + } + } + } + }, + }, + { + "ignore": ["^js/plugins.*", "^plugins.*"], + "coverage": { + "notify": { + "slack": { + "default": { + "url": "https://hooks.slack.com/services/testdazzd/testrf4k6py/test72nq0j0ke3prs2fdvfuj", + "only_pulls": False, + "branches": ["^main$", "^qa$", "^dev$"], + } + } + } + }, + }, + ), + ( + { + "codecov": {"notify": {"manual_trigger": True}}, + }, + { + "codecov": {"notify": {"manual_trigger": True}}, + }, + ), + ], + ) + def test_random_real_life_cases(self, user_input, expected_result): + # Some random cases based on real world examples + assert expected_result == validate_yaml(user_input) + + def test_case_with_experimental_turned_on_valid(self, mocker): + mocker.patch.dict(os.environ, {"CERBERUS_VALIDATOR_RATE": "1.0"}) + user_input = {"coverage": {"status": {"patch": True}}} + expected_result = {"coverage": {"status": {"patch": True}}} + assert expected_result == validate_yaml(user_input) + + def test_case_with_experimental_turned_invalid(self, mocker): + mocker.patch.dict(os.environ, {"CERBERUS_VALIDATOR_RATE": "1.0"}) + user_input = {"coverage": {"status": {"patch": "banana"}}} + with pytest.raises(InvalidYamlException) as ex: + validate_yaml(user_input) + assert ex.value.error_location == ["coverage", "status", "patch"] + assert ex.value.error_message == "must be of ['dict', 'boolean'] type" + + def test_many_flags_validation(self): + user_input = { + "codecov": {"max_report_age": False, "notify": {"after_n_builds": 10}}, + "comment": {"layout": "reach,footer"}, + "coverage": { + "status": { + "patch": { + "default": True, + "numbers_only": {"paths": ["wildwest/numbers.py"]}, + "strings_only": {"paths": ["wildwest/strings.py"]}, + "one": {"flags": ["one"]}, + "two": {"flags": ["two"]}, + "three": {"flags": ["three"]}, + "four": {"flags": ["four"]}, + "five": {"flags": ["five"]}, + "six": {"flags": ["six"]}, + "seven": {"flags": ["seven"]}, + "eight": {"flags": ["eight"]}, + "nine": {"flags": ["nine"]}, + "ten": {"flags": ["ten"]}, + "eleven": {"flags": ["eleven"]}, + "one_without_t": {"flags": ["one"], "paths": ["wildwest"]}, + "two_without_t": {"flags": ["two"], "paths": ["wildwest"]}, + "three_without_t": {"flags": ["three"], "paths": ["wildwest"]}, + "four_without_t": {"flags": ["four"], "paths": ["wildwest"]}, + "five_without_t": {"flags": ["five"], "paths": ["wildwest"]}, + "six_without_t": {"flags": ["six"], "paths": ["wildwest"]}, + "seven_without_t": {"flags": ["seven"], "paths": ["wildwest"]}, + "eight_without_t": {"flags": ["eight"], "paths": ["wildwest"]}, + "nine_without_t": {"flags": ["nine"], "paths": ["wildwest"]}, + "ten_without_t": {"flags": ["ten"], "paths": ["wildwest"]}, + "eleven_without_t": { + "flags": ["eleven"], + "paths": ["wildwest"], + }, + }, + "project": { + "default": True, + "numbers_only": {"paths": ["wildwest/numbers.py"]}, + "strings_only": {"paths": ["wildwest/strings.py"]}, + "one": {"flags": ["one"]}, + "two": {"flags": ["two"]}, + "three": {"flags": ["three"]}, + "four": {"flags": ["four"]}, + "five": {"flags": ["five"]}, + "six": {"flags": ["six"]}, + "seven": {"flags": ["seven"]}, + "eight": {"flags": ["eight"]}, + "nine": {"flags": ["nine"]}, + "ten": {"flags": ["ten"]}, + "eleven": {"flags": ["eleven"]}, + "one_without_t": {"flags": ["one"], "paths": ["wildwest"]}, + "two_without_t": {"flags": ["two"], "paths": ["wildwest"]}, + "three_without_t": {"flags": ["three"], "paths": ["wildwest"]}, + "four_without_t": {"flags": ["four"], "paths": ["wildwest"]}, + "five_without_t": {"flags": ["five"], "paths": ["wildwest"]}, + "six_without_t": {"flags": ["six"], "paths": ["wildwest"]}, + "seven_without_t": {"flags": ["seven"], "paths": ["wildwest"]}, + "eight_without_t": {"flags": ["eight"], "paths": ["wildwest"]}, + "nine_without_t": {"flags": ["nine"], "paths": ["wildwest"]}, + "ten_without_t": {"flags": ["ten"], "paths": ["wildwest"]}, + "eleven_without_t": { + "flags": ["eleven"], + "paths": ["wildwest"], + }, + }, + "changes": { + "numbers_only": {"paths": ["wildwest/numbers.py"]}, + "strings_only": {"paths": ["wildwest/strings.py"]}, + "default": True, + "one": {"flags": ["one"]}, + "two": {"flags": ["two"]}, + "three": {"flags": ["three"]}, + "four": {"flags": ["four"]}, + "five": {"flags": ["five"]}, + "six": {"flags": ["six"]}, + "seven": {"flags": ["seven"]}, + "eight": {"flags": ["eight"]}, + "nine": {"flags": ["nine"]}, + "ten": {"flags": ["ten"]}, + "eleven": {"flags": ["eleven"]}, + "one_without_t": {"flags": ["one"], "paths": ["wildwest"]}, + "two_without_t": {"flags": ["two"], "paths": ["wildwest"]}, + "three_without_t": {"flags": ["three"], "paths": ["wildwest"]}, + "four_without_t": {"flags": ["four"], "paths": ["wildwest"]}, + "five_without_t": {"flags": ["five"], "paths": ["wildwest"]}, + "six_without_t": {"flags": ["six"], "paths": ["wildwest"]}, + "seven_without_t": {"flags": ["seven"], "paths": ["wildwest"]}, + "eight_without_t": {"flags": ["eight"], "paths": ["wildwest"]}, + "nine_without_t": {"flags": ["nine"], "paths": ["wildwest"]}, + "ten_without_t": {"flags": ["ten"], "paths": ["wildwest"]}, + "eleven_without_t": { + "flags": ["eleven"], + "paths": ["wildwest"], + }, + }, + } + }, + "flag_management": { + "default_rules": { + "carryforward": False, + "statuses": [{"name_prefix": "aaa", "type": "patch"}], + }, + "individual_flags": [ + {"name": "cawcaw", "paths": ["banana"], "after_n_builds": 3} + ], + }, + } + expected_result = { + "codecov": {"max_report_age": False, "notify": {"after_n_builds": 10}}, + "comment": {"layout": "reach,footer"}, + "coverage": { + "status": { + "patch": { + "default": True, + "numbers_only": {"paths": ["^wildwest/numbers.py.*"]}, + "strings_only": {"paths": ["^wildwest/strings.py.*"]}, + "one": {"flags": ["one"]}, + "two": {"flags": ["two"]}, + "three": {"flags": ["three"]}, + "four": {"flags": ["four"]}, + "five": {"flags": ["five"]}, + "six": {"flags": ["six"]}, + "seven": {"flags": ["seven"]}, + "eight": {"flags": ["eight"]}, + "nine": {"flags": ["nine"]}, + "ten": {"flags": ["ten"]}, + "eleven": {"flags": ["eleven"]}, + "one_without_t": {"flags": ["one"], "paths": ["^wildwest.*"]}, + "two_without_t": {"flags": ["two"], "paths": ["^wildwest.*"]}, + "three_without_t": { + "flags": ["three"], + "paths": ["^wildwest.*"], + }, + "four_without_t": {"flags": ["four"], "paths": ["^wildwest.*"]}, + "five_without_t": {"flags": ["five"], "paths": ["^wildwest.*"]}, + "six_without_t": {"flags": ["six"], "paths": ["^wildwest.*"]}, + "seven_without_t": { + "flags": ["seven"], + "paths": ["^wildwest.*"], + }, + "eight_without_t": { + "flags": ["eight"], + "paths": ["^wildwest.*"], + }, + "nine_without_t": {"flags": ["nine"], "paths": ["^wildwest.*"]}, + "ten_without_t": {"flags": ["ten"], "paths": ["^wildwest.*"]}, + "eleven_without_t": { + "flags": ["eleven"], + "paths": ["^wildwest.*"], + }, + }, + "project": { + "default": True, + "numbers_only": {"paths": ["^wildwest/numbers.py.*"]}, + "strings_only": {"paths": ["^wildwest/strings.py.*"]}, + "one": {"flags": ["one"]}, + "two": {"flags": ["two"]}, + "three": {"flags": ["three"]}, + "four": {"flags": ["four"]}, + "five": {"flags": ["five"]}, + "six": {"flags": ["six"]}, + "seven": {"flags": ["seven"]}, + "eight": {"flags": ["eight"]}, + "nine": {"flags": ["nine"]}, + "ten": {"flags": ["ten"]}, + "eleven": {"flags": ["eleven"]}, + "one_without_t": {"flags": ["one"], "paths": ["^wildwest.*"]}, + "two_without_t": {"flags": ["two"], "paths": ["^wildwest.*"]}, + "three_without_t": { + "flags": ["three"], + "paths": ["^wildwest.*"], + }, + "four_without_t": {"flags": ["four"], "paths": ["^wildwest.*"]}, + "five_without_t": {"flags": ["five"], "paths": ["^wildwest.*"]}, + "six_without_t": {"flags": ["six"], "paths": ["^wildwest.*"]}, + "seven_without_t": { + "flags": ["seven"], + "paths": ["^wildwest.*"], + }, + "eight_without_t": { + "flags": ["eight"], + "paths": ["^wildwest.*"], + }, + "nine_without_t": {"flags": ["nine"], "paths": ["^wildwest.*"]}, + "ten_without_t": {"flags": ["ten"], "paths": ["^wildwest.*"]}, + "eleven_without_t": { + "flags": ["eleven"], + "paths": ["^wildwest.*"], + }, + }, + "changes": { + "numbers_only": {"paths": ["^wildwest/numbers.py.*"]}, + "strings_only": {"paths": ["^wildwest/strings.py.*"]}, + "default": True, + "one": {"flags": ["one"]}, + "two": {"flags": ["two"]}, + "three": {"flags": ["three"]}, + "four": {"flags": ["four"]}, + "five": {"flags": ["five"]}, + "six": {"flags": ["six"]}, + "seven": {"flags": ["seven"]}, + "eight": {"flags": ["eight"]}, + "nine": {"flags": ["nine"]}, + "ten": {"flags": ["ten"]}, + "eleven": {"flags": ["eleven"]}, + "one_without_t": {"flags": ["one"], "paths": ["^wildwest.*"]}, + "two_without_t": {"flags": ["two"], "paths": ["^wildwest.*"]}, + "three_without_t": { + "flags": ["three"], + "paths": ["^wildwest.*"], + }, + "four_without_t": {"flags": ["four"], "paths": ["^wildwest.*"]}, + "five_without_t": {"flags": ["five"], "paths": ["^wildwest.*"]}, + "six_without_t": {"flags": ["six"], "paths": ["^wildwest.*"]}, + "seven_without_t": { + "flags": ["seven"], + "paths": ["^wildwest.*"], + }, + "eight_without_t": { + "flags": ["eight"], + "paths": ["^wildwest.*"], + }, + "nine_without_t": {"flags": ["nine"], "paths": ["^wildwest.*"]}, + "ten_without_t": {"flags": ["ten"], "paths": ["^wildwest.*"]}, + "eleven_without_t": { + "flags": ["eleven"], + "paths": ["^wildwest.*"], + }, + }, + } + }, + "flag_management": { + "default_rules": { + "carryforward": False, + "statuses": [{"name_prefix": "aaa", "type": "patch"}], + }, + "individual_flags": [ + {"name": "cawcaw", "paths": ["^banana.*"], "after_n_builds": 3} + ], + }, + } + assert validate_yaml(user_input) == expected_result + + def test_validate_bot_none(self): + user_input = {"codecov": {"bot": None}} + expected_result = {"codecov": {"bot": None}} + result = validate_yaml(user_input) + assert result == expected_result + + def test_validate_flag_too_long(self): + user_input = {"flags": {"abcdefg" * 500: {"paths": ["banana"]}}} + with pytest.raises(InvalidYamlException) as exc: + validate_yaml(user_input) + assert exc.value.error_location == [ + "flags", + "abcdefg" * 500, + ] + + def test_validate_parser_only_field(self): + user_input = {"parsers": {"go": {"partials_as_hits": True}}} + expected_result = {"parsers": {"go": {"partials_as_hits": True}}} + result = validate_yaml(user_input) + assert result == expected_result + + def test_simple_case(self): + encoded_value = "secret:v1::zsV9A8pHadNle357DGJHbZCTyCYA+TXdUd9TN3IY2DIWcPOtgK3Pg1EgA6OZr9XJ1EsdpL765yWrN4pfR3elRdN2LUwiuv6RkNjpbiruHx45agsgxdu8fi24p5pkCLvjcW0HqdH2PTvmHauIp+ptgA==" user_input = { "coverage": { "precision": 2, @@ -952,7 +502,7 @@ def test_bad_yaml(self): }, "no_upload_behavior": "pass", }, - "notify": {"irc": {"user_given_title": {"password": "asdf"}}}, + "notify": {"irc": {"user_given_title": {"password": encoded_value}}}, }, "codecov": {"notify": {"require_ci_to_pass": True}}, "comment": { @@ -975,548 +525,929 @@ def test_bad_yaml(self): "cli": { "plugins": {"pycoverage": {"report_type": "json"}}, }, - "component_management": { - # "default_rules": { - # # "statuses": [ - # # { - # # "type": "project", - # # "target": "auto", - # # "branches": ["!main"] - # # } - # # ] - # }, + } + expected_result = { + "coverage": { + "precision": 2, + "round": "down", + "range": [70, 100], + "status": { + "project": { + "custom_project": { + "carryforward_behavior": "exclude", + "flag_coverage_not_uploaded_behavior": "exclude", + } + }, + "patch": True, + "changes": False, + "default_rules": { + "carryforward_behavior": "pass", + "flag_coverage_not_uploaded_behavior": "pass", + }, + "no_upload_behavior": "pass", + }, + "notify": {"irc": {"user_given_title": {"password": encoded_value}}}, + }, + "codecov": {"notify": {}, "require_ci_to_pass": True}, + "comment": { + "behavior": "default", + "layout": "header, diff", + "require_changes": [0b000], + "show_carryforward_flags": False, + }, + "parsers": { + "gcov": { + "branch_detection": { + "conditional": True, + "loop": True, + "macro": False, + "method": False, + } + }, + "jacoco": {"partials_as_hits": True}, + }, + "cli": { + "plugins": {"pycoverage": {"report_type": "json"}}, + }, + } + assert validate_yaml(user_input) == expected_result + + def test_negative_notify_after_n_builds(self): + user_input = {"codecov": {"notify": {"after_n_builds": -1}}} + with pytest.raises(InvalidYamlException) as exc: + validate_yaml(user_input) + assert exc.value.error_location == ["codecov", "notify", "after_n_builds"] + assert exc.value.error_message == "min value is 0" + + def test_positive_notify_after_n_builds(self): + user_input = {"codecov": {"notify": {"after_n_builds": 1}}} + res = validate_yaml(user_input) + assert res == {"codecov": {"notify": {"after_n_builds": 1}}} + + def test_negative_comments_after_n_builds(self): + user_input = {"comment": {"after_n_builds": -1}} + with pytest.raises(InvalidYamlException) as exc: + validate_yaml(user_input) + assert exc.value.error_location == ["comment", "after_n_builds"] + assert exc.value.error_message == "min value is 0" + + def test_invalid_yaml_case(self): + user_input = { + "coverage": { + "round": "down", + "precision": 2, + "range": "70...100", + "status": {"project": {"base": "auto", "aa": True}}, + }, + "ignore": ["Pods/.*"], + } + with pytest.raises(InvalidYamlException) as exc: + validate_yaml(user_input) + assert exc.value.error_location == ["coverage", "status", "project", "base"] + assert exc.value.error_message == "must be of ['dict', 'boolean'] type" + + def test_invalid_yaml_case_custom_validator(self): + user_input = { + "coverage": { + "round": "down", + "precision": 2, + "range": "70...5000", + "status": {"project": {"percent": "abc"}}, + }, + "ignore": ["Pods/.*"], + } + with pytest.raises(InvalidYamlException) as exc: + validate_yaml(user_input) + assert exc.value.error_location == ["coverage", "range"] + assert exc.value.error_message == "must be of list type" + + def test_invalid_yaml_case_no_upload_behavior(self): + user_input = { + "coverage": { + "round": "down", + "precision": 2, + "range": "70...100", + "status": { + "project": {"percent": "abc"}, + "no_upload_behavior": "no-pass", + }, + }, + "ignore": ["Pods/.*"], + } + with pytest.raises(InvalidYamlException) as exc: + validate_yaml(user_input) + assert exc.value.error_location == ["coverage", "status", "no_upload_behavior"] + assert exc.value.error_message == "unallowed value no-pass" + + def test_yaml_with_null_threshold(self): + user_input = { + "codecov": {"notify": {}, "require_ci_to_pass": True}, + "comment": { + "behavior": "default", + "branches": None, + "layout": "reach, diff, flags, files", + "require_base": False, + "require_changes": False, + "require_head": False, + }, + "coverage": { + "precision": 2, + "range": "50...80", + "round": "down", + "status": { + "changes": False, + "patch": True, + "project": { + "default": {"target": "auto", "threshold": None, "base": "auto"} + }, + }, + }, + } + res = validate_yaml(user_input) + expected_result = { + "codecov": {"notify": {}, "require_ci_to_pass": True}, + "comment": { + "behavior": "default", + "branches": None, + "layout": "reach, diff, flags, files", + "require_base": False, + "require_changes": [0b000], + "require_head": False, + }, + "coverage": { + "precision": 2, + "range": [50.0, 80.0], + "round": "down", + "status": { + "changes": False, + "patch": True, + "project": { + "default": {"target": "auto", "threshold": None, "base": "auto"} + }, + }, + }, + } + assert res == expected_result + + def test_yaml_with_status_case(self): + user_input = { + "coverage": { + "round": "down", + "precision": 2, + "range": "70...100", + "status": {"project": {"default": {"base": "auto"}}}, + }, + "ignore": ["Pods/.*"], + } + expected_result = { + "coverage": { + "round": "down", + "precision": 2, + "range": [70.0, 100.0], + "status": {"project": {"default": {"base": "auto"}}}, + }, + "ignore": ["Pods/.*"], + } + result = validate_yaml(user_input) + assert result == expected_result + + def test_yaml_with_flag_management(self): + user_input = { + "flag_management": { + "default_rules": { + "carryforward": True, + "statuses": [ + { + "type": "project", + "name_prefix": "healthcare", + "threshold": 80, + } + ], + }, + "individual_flags": [ + { + "name": "flag_banana", + "statuses": [ + { + "type": "patch", + "name_prefix": "alliance", + "flag_coverage_not_uploaded_behavior": "include", + } + ], + } + ], + } + } + expected_result = { + "flag_management": { + "individual_flags": [ + { + "name": "flag_banana", + "statuses": [ + { + "type": "patch", + "name_prefix": "alliance", + "flag_coverage_not_uploaded_behavior": "include", + } + ], + } + ], + "default_rules": { + "carryforward": True, + "statuses": [ + { + "type": "project", + "name_prefix": "healthcare", + "threshold": 80.0, + } + ], + }, + } + } + result = validate_yaml(user_input) + assert result == expected_result + + def test_yaml_with_flag_management_statuses_with_flags(self): + user_input = { + "flag_management": { + "default_rules": { + "carryforward": True, + "statuses": [ + { + "type": "project", + "name_prefix": "healthcare", + "threshold": 80, + "flags": ["hahaha"], + } + ], + }, + "individual_flags": [ + { + "name": "flag_banana", + "statuses": [ + { + "type": "patch", + "name_prefix": "alliance", + "flag_coverage_not_uploaded_behavior": "include", + } + ], + } + ], + } + } + with pytest.raises(InvalidYamlException) as exc: + validate_yaml(user_input) + assert exc.value.error_location == [ + "flag_management", + "default_rules", + "statuses", + 0, + "flags", + ] + assert exc.value.error_message == "extra keys not allowed" + + def test_github_checks(self): + user_input = {"github_checks": True} + expected_result = {"github_checks": True} + assert validate_yaml(user_input) == expected_result + user_input = {"github_checks": {"annotations": False}} + expected_result = {"github_checks": {"annotations": False}} + assert validate_yaml(user_input) == expected_result + + def test_validate_jacoco_partials(self): + user_input = {"parsers": {"jacoco": {"partials_as_hits": True}}} + expected_result = {"parsers": {"jacoco": {"partials_as_hits": True}}} + result = validate_yaml(user_input) + assert result == expected_result + + @pytest.mark.parametrize( + "input, expected", + [ + pytest.param( + {"comment": {"require_bundle_changes": False}}, + {"comment": {"require_bundle_changes": False}}, + id="no_bundle_changes_required", + ), + pytest.param( + { + "comment": { + "require_bundle_changes": True, + "bundle_change_threshold": 1200, + } + }, + { + "comment": { + "require_bundle_changes": True, + "bundle_change_threshold": ("absolute", 1200), + } + }, + id="bundle_changes_with_threshold", + ), + pytest.param( + { + "comment": { + "require_bundle_changes": "bundle_increase", + "bundle_change_threshold": "1mb", + } + }, + { + "comment": { + "require_bundle_changes": "bundle_increase", + "bundle_change_threshold": ("absolute", 1000000), + } + }, + id="bundle_increase_required_with_threshold", + ), + pytest.param( + { + "comment": { + "require_bundle_changes": "bundle_increase", + "bundle_change_threshold": "10%", + } + }, + { + "comment": { + "require_bundle_changes": "bundle_increase", + "bundle_change_threshold": ("percentage", 10.0), + } + }, + id="bundle_increase_required_with_percentage_threshold", + ), + ], + ) + def test_bundle_analysis_comment_config(self, input, expected, mocker): + mocker.patch.object(BUNDLE_THRESHOLD_FLAG, "check_value", return_value=True) + result = validate_yaml(input) + assert result == expected + + @pytest.mark.parametrize( + "input, expected", + [ + pytest.param( + { + "bundle_analysis": { + "status": False, + "warning_threshold": "10%", + } + }, + { + "bundle_analysis": { + "status": False, + "warning_threshold": ("percentage", 10.0), + } + }, + id="status_off_percentage_threshold", + ), + pytest.param( + { + "bundle_analysis": { + "status": True, + "warning_threshold": "10kb", + } + }, + { + "bundle_analysis": { + "status": True, + "warning_threshold": ("absolute", 10000), + } + }, + id="status_on_absolute_threshold", + ), + pytest.param( + { + "bundle_analysis": { + "status": "informational", + } + }, + { + "bundle_analysis": { + "status": "informational", + } + }, + id="status_informational_no_threshold", + ), + ], + ) + def test_bundle_analysis_config(self, input, expected, mocker): + mocker.patch.object(BUNDLE_THRESHOLD_FLAG, "check_value", return_value=True) + result = validate_yaml(input) + assert result == expected + + +class TestValidationConfig(object): + def test_validate_default_config_yaml(self, mocker): + mocker.patch.dict(os.environ, {}, clear=True) + mocker.patch.object( + ConfigHelper, "load_yaml_file", side_effect=FileNotFoundError() + ) + this_config = ConfigHelper() + mocker.patch("shared.config._get_config_instance", return_value=this_config) + expected_result = { + "codecov": {"require_ci_to_pass": True, "notify": {"wait_for_ci": True}}, + "coverage": { + "precision": 2, + "round": "down", + "range": [60.0, 80.0], + "status": { + "project": True, + "patch": True, + "changes": False, + "default_rules": {"flag_coverage_not_uploaded_behavior": "include"}, + }, + }, + "comment": { + "layout": "reach,diff,flags,tree,reach", + "behavior": "default", + "show_carryforward_flags": False, + }, + "github_checks": {"annotations": True}, + "slack_app": True, + } + res = validate_yaml( + get_config("site", default={}), + show_secrets_for=("github", "11934774", "154468867"), + ) + assert res == expected_result + + +def test_validation_with_branches(): + user_input = { + "comment": { + "require_head": True, + "require_base": True, + "layout": "diff", + "require_changes": True, + "branches": ["main"], + "behavior": "once", + "after_n_builds": 6, + }, + "coverage": { + "status": {"project": {"default": {"threshold": "1%"}}, "patch": False} + }, + "github_checks": {"annotations": False}, + "fixes": [ + "/opt/conda/lib/python3.8/site-packages/::project/", + "C:/Users/circleci/project/build/win_tmp/build/::project/", + ], + "ignore": ["coffee", "party_man", "test"], + "codecov": {"notify": {"after_n_builds": 6}}, + } + expected_result = { + "comment": { + "require_head": True, + "require_base": True, + "layout": "diff", + "require_changes": [0b001], + "branches": ["^main$"], + "behavior": "once", + "after_n_builds": 6, + }, + "coverage": { + "status": {"project": {"default": {"threshold": 1}}, "patch": False} + }, + "github_checks": {"annotations": False}, + "fixes": [ + "^/opt/conda/lib/python3.8/site-packages/::project/", + "^C:/Users/circleci/project/build/win_tmp/build/::project/", + ], + "ignore": ["^coffee.*", "^party_man.*", "^test.*"], + "codecov": {"notify": {"after_n_builds": 6}}, + } + res = do_actual_validation(user_input, show_secrets_for=None) + assert res == expected_result + + +def test_validation_with_flag_carryforward(): + user_input = { + "flags": { + "old-flag": { + "carryforward": True, + "carryforward_mode": "labels", + }, + "other-old-flag": { + "carryforward": True, + "carryforward_mode": "all", + }, + }, + "flag_management": { + "individual_flags": [ + {"name": "abcdef", "carryforward_mode": "all"}, + {"name": "abcdef", "carryforward_mode": "labels"}, + ] + }, + } + assert do_actual_validation(user_input, show_secrets_for=None) == user_input + + +def test_validation_with_flag_carryforward_invalid_mode(): + user_input = { + "flag_management": { + "individual_flags": [ + {"name": "abcdef", "carryforward_mode": "mario"}, + {"name": "abcdef", "carryforward_mode": "labels"}, + ] + }, + } + with pytest.raises(InvalidYamlException) as exp: + do_actual_validation(user_input, show_secrets_for=None) + assert exp.value.error_dict == { + "flag_management": [ + { + "individual_flags": [ + {0: [{"carryforward_mode": ["unallowed value mario"]}]} + ] + } + ] + } + + +def test_validation_with_null_on_paths(): + user_input = { + "comment": {"require_head": True, "behavior": "once", "after_n_builds": 6}, + "coverage": { + "status": {"project": {"default": {"threshold": "1%"}}, "patch": False}, + "notify": {"slack": {"default": {"paths": None}}}, + }, + "ignore": ["coffee", "test"], + } + expected_result = { + "comment": {"require_head": True, "behavior": "once", "after_n_builds": 6}, + "coverage": { + "status": {"project": {"default": {"threshold": 1.0}}, "patch": False}, + "notify": {"slack": {"default": {"paths": None}}}, + }, + "ignore": ["^coffee.*", "^test.*"], + } + res = do_actual_validation(user_input, show_secrets_for=None) + assert res == expected_result + + +def test_validation_with_null_on_status(): + user_input = { + "coverage": {"status": {"project": {"default": None}, "patch": False}}, + "ignore": ["coffee", "test"], + } + expected_result = { + "coverage": {"status": {"project": {"default": None}, "patch": False}}, + "ignore": ["^coffee.*", "^test.*"], + } + res = do_actual_validation(user_input, show_secrets_for=None) + assert res == expected_result + + +def test_improper_layout(): + user_input = { + "coverage": {"status": {"project": {"default": None}, "patch": False}}, + "comment": {"layout": "banana,apple"}, + } + with pytest.raises(InvalidYamlException) as exc: + do_actual_validation(user_input, show_secrets_for=None) + assert exc.value.error_dict == { + "comment": [{"layout": ["Unexpected values on layout: apple,banana"]}] + } + assert exc.value.error_location == ["comment", "layout"] + + +def test_proper_layout(): + user_input = { + "coverage": {"status": {"project": {"default": None}, "patch": False}}, + "comment": {"layout": "files:10,footer"}, + } + res = do_actual_validation(user_input, show_secrets_for=None) + assert res == { + "coverage": {"status": {"project": {"default": None}, "patch": False}}, + "comment": {"layout": "files:10,footer"}, + } + + +def test_codecov_branch(): + user_input = {"codecov": {"branch": "origin/pterosaur"}} + res = do_actual_validation(user_input, show_secrets_for=None) + assert res == {"codecov": {"branch": "pterosaur"}} + + +def test_calculate_error_location_and_message_from_error_dict(): + error_dict = {"comment": [{"layout": {"deep": [[[[[{"inside": [["value"]]}]]]]]}}]} + assert ( + ["comment", "layout", "deep", "inside"], + "value", + ) == _calculate_error_location_and_message_from_error_dict(error_dict) + # case where the value is just very nested. + # This is not a requirement of any kind. This is just so + # there are no cases where a customer can send some special yaml with loops + # and make us keep parsing this forever + # It might even be overkill + assert ( + ["value", "some", "thing"], + "[[['haha']]]", + ) == _calculate_error_location_and_message_from_error_dict( + {"value": {"some": {"thing": [[[[[[[[[[[[[[[[[[[["haha"]]]]]]]]]]]]]]]]]]]]}}} + ) + + +def test_email_field_with_and_without_secret(): + user_input = { + "coverage": { + "notify": { + "email": { + "default": { + "to": [ + "example@domain.com", + "secret:v1::hfxSizNpugwZzYZXXRxxlszUetU8tyVG1HXCEdK5qeC9XhtxkCsb/Z5nvp70mp4zlbfcinTy9C9lSGXZAmN8uGKuhWnwrFPfYupe7jQ5KQY=", + ], + "threshold": "1%", + "only_pulls": False, + "layout": "reach, diff, flags", + "flags": None, + "paths": None, + } + } + } + } + } + assert do_actual_validation( + user_input, show_secrets_for=("github", "11934774", "154468867") + ) == { + "coverage": { + "notify": { + "email": { + "default": { + "to": ["example@domain.com", "secondexample@seconddomain.com"], + "threshold": 1.0, + "only_pulls": False, + "layout": "reach, diff, flags", + "flags": None, + "paths": None, + } + } + } + } + } + assert do_actual_validation(user_input, show_secrets_for=None) == { + "coverage": { + "notify": { + "email": { + "default": { + "to": [ + "example@domain.com", + "secret:v1::hfxSizNpugwZzYZXXRxxlszUetU8tyVG1HXCEdK5qeC9XhtxkCsb/Z5nvp70mp4zlbfcinTy9C9lSGXZAmN8uGKuhWnwrFPfYupe7jQ5KQY=", + ], + "threshold": 1.0, + "only_pulls": False, + "layout": "reach, diff, flags", + "flags": None, + "paths": None, + } + } + } + } + } + + +def test_assume_flags(): + # It's deprecated, but still + user_input = {"flags": {"some_flag": {"assume": {"branches": ["main"]}}}} + assert do_actual_validation( + user_input, show_secrets_for=("github", "11934774", "154468867") + ) == {"flags": {"some_flag": {"assume": {"branches": ["^main$"]}}}} + + +def test_after_n_builds_flags(): + user_input = {"flags": {"some_flag": {"after_n_builds": 5}}} + assert do_actual_validation( + user_input, show_secrets_for=("github", "11934774", "154468867") + ) == {"flags": {"some_flag": {"after_n_builds": 5}}} + + +def test_profiling_schema(): + user_input = { + "profiling": { + "fixes": ["batata_something::batata.txt"], + "grouping_attributes": ["string", "str"], + "critical_files_paths": [ + "/path/to/file.extension", + "/path/to/dir", + r"/path/{src|bin}/regex.{txt|php|cpp}", + "/path/using/globs/**/file.extension", + ], + } + } + expected_result = { + "profiling": { + "fixes": ["^batata_something::batata.txt"], + "grouping_attributes": ["string", "str"], + "critical_files_paths": [ + "^/path/to/file.extension.*", + "^/path/to/dir.*", + "^/path/{src|bin}/regex.{txt|php|cpp}.*", + "(?s:/path/using/globs/.*/file\\.extension)\\Z", + ], + } + } + result = validate_yaml(user_input) + assert result == expected_result + + +def test_components_schema(): + user_input = { + "component_management": { + "default_rules": { + "flag_regexes": ["global_flag"], + }, + "individual_components": [ + { + "name": "fruits", + "component_id": "app_0", + "flag_regexes": ["fruit_.*", "^specific_flag$"], + "paths": ["src/.*"], + "statuses": [{"type": "patch", "name_prefix": "co", "target": 90}], + } + ], + } + } + expected = { + "component_management": { + "default_rules": { + "flag_regexes": ["global_flag"], + }, + "individual_components": [ + { + "name": "fruits", + "component_id": "app_0", + "flag_regexes": ["fruit_.*", "^specific_flag$"], + "paths": ["src/.*"], + "statuses": [ + {"type": "patch", "name_prefix": "co", "target": 90.0} + ], + } + ], + } + } + result = validate_yaml(user_input) + assert result == expected + + +def test_components_schema_error(): + user_input = { + "component_management": { "individual_components": [ + { + "key": "extra", + "component_id": "app_0", + "flag_regexes": ["fruit_*", "^specific_flag$"], + "path_filter_regexes": ["src/.*"], + "statuses": [ + {"type": "patch", "name_prefix": "co", "target": 90.0} + ], + }, + { + "component_id": "app_0", + "flag_regexes": ["fruit_*", "^specific_flag$"], + "path_filter_regexes": ["src/.*"], + }, + ], + } + } + with pytest.raises(InvalidYamlException) as exp: + validate_yaml(user_input) + assert exp.error_location == [ + "component_management", + "individual_components", + 0, + "key", + ] + assert exp.error_message == "unknown field" + assert exp.error_dict == { + "component_management": [ + { + "individual_components": [ + { + 0: [{"key": ["unknown field"]}], + 1: [{"component_id": ["required field"]}], + } + ] + } + ] + } + + +def test_removed_code_behavior_config_valid(): + user_input = { + "coverage": { + "status": { + "project": { + "some_status": {"removed_code_behavior": "removals_only"}, + } + } + }, + "flag_management": { + "default_rules": { + "statuses": [ + {"name_prefix": "custom", "removed_code_behavior": "adjust_base"} + ] + }, + "individual_flags": [ + { + "name": "random", + "statuses": [ + { + "name_prefix": "random-custom", + "removed_code_behavior": False, + } + ], + } + ], + }, + "component_management": { + "default_rules": { + "statuses": [ { - "component_id": "module_nps", - "name": "Team 2", - "statuses": { - "type": "patch", - "target": "auto" - }, - "paths": ["lib/nps/**"] + "name_prefix": "custom", + "removed_code_behavior": "fully_covered_patch", } ] + }, + "individual_components": [ + { + "component_id": "random", + "statuses": [ + { + "name_prefix": "random-custom", + "removed_code_behavior": "off", + } + ], + } + ], + }, + } + result = validate_yaml(user_input) + # There's no change on the valid yaml + assert result == user_input + + +def test_offset_config_error(): + user_input = { + "flag_management": { + "default_rules": { + "statuses": [ + {"name_prefix": "custom", "removed_code_behavior": "banana"} + ] } + }, + } + + with pytest.raises(InvalidYamlException) as exp: + validate_yaml(user_input) + assert exp.error_dict == { + "coverage": [ + { + "status": [ + {"patch": [{"some_status": [{"offset": ["unknown field"]}]}]} + ] + } + ], + "flag_management": [ + { + "default_rules": [ + {"statuses": [{0: [{"offset": ["unallowed value banana"]}]}]} + ] + } + ], } - res = validate_yaml(user_input) - print(res) - -# class TestValidationConfig(object): -# def test_validate_default_config_yaml(self, mocker): -# mocker.patch.dict(os.environ, {}, clear=True) -# mocker.patch.object( -# ConfigHelper, "load_yaml_file", side_effect=FileNotFoundError() -# ) -# this_config = ConfigHelper() -# mocker.patch("shared.config._get_config_instance", return_value=this_config) -# expected_result = { -# "codecov": {"require_ci_to_pass": True, "notify": {"wait_for_ci": True}}, -# "coverage": { -# "precision": 2, -# "round": "down", -# "range": [60.0, 80.0], -# "status": { -# "project": True, -# "patch": True, -# "changes": False, -# "default_rules": {"flag_coverage_not_uploaded_behavior": "include"}, -# }, -# }, -# "comment": { -# "layout": "reach,diff,flags,tree,reach", -# "behavior": "default", -# "show_carryforward_flags": False, -# }, -# "github_checks": {"annotations": True}, -# "slack_app": True, -# } -# res = validate_yaml( -# get_config("site", default={}), -# show_secrets_for=("github", "11934774", "154468867"), -# ) -# assert res == expected_result - - -# def test_validation_with_branches(): -# user_input = { -# "comment": { -# "require_head": True, -# "require_base": True, -# "layout": "diff", -# "require_changes": True, -# "branches": ["main"], -# "behavior": "once", -# "after_n_builds": 6, -# }, -# "coverage": { -# "status": {"project": {"default": {"threshold": "1%"}}, "patch": False} -# }, -# "github_checks": {"annotations": False}, -# "fixes": [ -# "/opt/conda/lib/python3.8/site-packages/::project/", -# "C:/Users/circleci/project/build/win_tmp/build/::project/", -# ], -# "ignore": ["coffee", "party_man", "test"], -# "codecov": {"notify": {"after_n_builds": 6}}, -# } -# expected_result = { -# "comment": { -# "require_head": True, -# "require_base": True, -# "layout": "diff", -# "require_changes": [0b001], -# "branches": ["^main$"], -# "behavior": "once", -# "after_n_builds": 6, -# }, -# "coverage": { -# "status": {"project": {"default": {"threshold": 1}}, "patch": False} -# }, -# "github_checks": {"annotations": False}, -# "fixes": [ -# "^/opt/conda/lib/python3.8/site-packages/::project/", -# "^C:/Users/circleci/project/build/win_tmp/build/::project/", -# ], -# "ignore": ["^coffee.*", "^party_man.*", "^test.*"], -# "codecov": {"notify": {"after_n_builds": 6}}, -# } -# res = do_actual_validation(user_input, show_secrets_for=None) -# assert res == expected_result - - -# def test_validation_with_flag_carryforward(): -# user_input = { -# "flags": { -# "old-flag": { -# "carryforward": True, -# "carryforward_mode": "labels", -# }, -# "other-old-flag": { -# "carryforward": True, -# "carryforward_mode": "all", -# }, -# }, -# "flag_management": { -# "individual_flags": [ -# {"name": "abcdef", "carryforward_mode": "all"}, -# {"name": "abcdef", "carryforward_mode": "labels"}, -# ] -# }, -# } -# assert do_actual_validation(user_input, show_secrets_for=None) == user_input - - -# def test_validation_with_flag_carryforward_invalid_mode(): -# user_input = { -# "flag_management": { -# "individual_flags": [ -# {"name": "abcdef", "carryforward_mode": "mario"}, -# {"name": "abcdef", "carryforward_mode": "labels"}, -# ] -# }, -# } -# with pytest.raises(InvalidYamlException) as exp: -# do_actual_validation(user_input, show_secrets_for=None) -# assert exp.value.error_dict == { -# "flag_management": [ -# { -# "individual_flags": [ -# {0: [{"carryforward_mode": ["unallowed value mario"]}]} -# ] -# } -# ] -# } - - -# def test_validation_with_null_on_paths(): -# user_input = { -# "comment": {"require_head": True, "behavior": "once", "after_n_builds": 6}, -# "coverage": { -# "status": {"project": {"default": {"threshold": "1%"}}, "patch": False}, -# "notify": {"slack": {"default": {"paths": None}}}, -# }, -# "ignore": ["coffee", "test"], -# } -# expected_result = { -# "comment": {"require_head": True, "behavior": "once", "after_n_builds": 6}, -# "coverage": { -# "status": {"project": {"default": {"threshold": 1.0}}, "patch": False}, -# "notify": {"slack": {"default": {"paths": None}}}, -# }, -# "ignore": ["^coffee.*", "^test.*"], -# } -# res = do_actual_validation(user_input, show_secrets_for=None) -# assert res == expected_result - - -# def test_validation_with_null_on_status(): -# user_input = { -# "coverage": {"status": {"project": {"default": None}, "patch": False}}, -# "ignore": ["coffee", "test"], -# } -# expected_result = { -# "coverage": {"status": {"project": {"default": None}, "patch": False}}, -# "ignore": ["^coffee.*", "^test.*"], -# } -# res = do_actual_validation(user_input, show_secrets_for=None) -# assert res == expected_result - - -# def test_improper_layout(): -# user_input = { -# "coverage": {"status": {"project": {"default": None}, "patch": False}}, -# "comment": {"layout": "banana,apple"}, -# } -# with pytest.raises(InvalidYamlException) as exc: -# do_actual_validation(user_input, show_secrets_for=None) -# assert exc.value.error_dict == { -# "comment": [{"layout": ["Unexpected values on layout: apple,banana"]}] -# } -# assert exc.value.error_location == ["comment", "layout"] - - -# def test_proper_layout(): -# user_input = { -# "coverage": {"status": {"project": {"default": None}, "patch": False}}, -# "comment": {"layout": "files:10,footer"}, -# } -# res = do_actual_validation(user_input, show_secrets_for=None) -# assert res == { -# "coverage": {"status": {"project": {"default": None}, "patch": False}}, -# "comment": {"layout": "files:10,footer"}, -# } - - -# def test_codecov_branch(): -# user_input = {"codecov": {"branch": "origin/pterosaur"}} -# res = do_actual_validation(user_input, show_secrets_for=None) -# assert res == {"codecov": {"branch": "pterosaur"}} - - -# def test_calculate_error_location_and_message_from_error_dict(): -# error_dict = {"comment": [{"layout": {"deep": [[[[[{"inside": [["value"]]}]]]]]}}]} -# assert ( -# ["comment", "layout", "deep", "inside"], -# "value", -# ) == _calculate_error_location_and_message_from_error_dict(error_dict) -# # case where the value is just very nested. -# # This is not a requirement of any kind. This is just so -# # there are no cases where a customer can send some special yaml with loops -# # and make us keep parsing this forever -# # It might even be overkill -# assert ( -# ["value", "some", "thing"], -# "[[['haha']]]", -# ) == _calculate_error_location_and_message_from_error_dict( -# {"value": {"some": {"thing": [[[[[[[[[[[[[[[[[[[["haha"]]]]]]]]]]]]]]]]]]]]}}} -# ) - - -# def test_email_field_with_and_without_secret(): -# user_input = { -# "coverage": { -# "notify": { -# "email": { -# "default": { -# "to": [ -# "example@domain.com", -# "secret:v1::hfxSizNpugwZzYZXXRxxlszUetU8tyVG1HXCEdK5qeC9XhtxkCsb/Z5nvp70mp4zlbfcinTy9C9lSGXZAmN8uGKuhWnwrFPfYupe7jQ5KQY=", -# ], -# "threshold": "1%", -# "only_pulls": False, -# "layout": "reach, diff, flags", -# "flags": None, -# "paths": None, -# } -# } -# } -# } -# } -# assert do_actual_validation( -# user_input, show_secrets_for=("github", "11934774", "154468867") -# ) == { -# "coverage": { -# "notify": { -# "email": { -# "default": { -# "to": ["example@domain.com", "secondexample@seconddomain.com"], -# "threshold": 1.0, -# "only_pulls": False, -# "layout": "reach, diff, flags", -# "flags": None, -# "paths": None, -# } -# } -# } -# } -# } -# assert do_actual_validation(user_input, show_secrets_for=None) == { -# "coverage": { -# "notify": { -# "email": { -# "default": { -# "to": [ -# "example@domain.com", -# "secret:v1::hfxSizNpugwZzYZXXRxxlszUetU8tyVG1HXCEdK5qeC9XhtxkCsb/Z5nvp70mp4zlbfcinTy9C9lSGXZAmN8uGKuhWnwrFPfYupe7jQ5KQY=", -# ], -# "threshold": 1.0, -# "only_pulls": False, -# "layout": "reach, diff, flags", -# "flags": None, -# "paths": None, -# } -# } -# } -# } -# } - - -# def test_assume_flags(): -# # It's deprecated, but still -# user_input = {"flags": {"some_flag": {"assume": {"branches": ["main"]}}}} -# assert do_actual_validation( -# user_input, show_secrets_for=("github", "11934774", "154468867") -# ) == {"flags": {"some_flag": {"assume": {"branches": ["^main$"]}}}} - - -# def test_after_n_builds_flags(): -# user_input = {"flags": {"some_flag": {"after_n_builds": 5}}} -# assert do_actual_validation( -# user_input, show_secrets_for=("github", "11934774", "154468867") -# ) == {"flags": {"some_flag": {"after_n_builds": 5}}} - - -# def test_profiling_schema(): -# user_input = { -# "profiling": { -# "fixes": ["batata_something::batata.txt"], -# "grouping_attributes": ["string", "str"], -# "critical_files_paths": [ -# "/path/to/file.extension", -# "/path/to/dir", -# r"/path/{src|bin}/regex.{txt|php|cpp}", -# "/path/using/globs/**/file.extension", -# ], -# } -# } -# expected_result = { -# "profiling": { -# "fixes": ["^batata_something::batata.txt"], -# "grouping_attributes": ["string", "str"], -# "critical_files_paths": [ -# "^/path/to/file.extension.*", -# "^/path/to/dir.*", -# "^/path/{src|bin}/regex.{txt|php|cpp}.*", -# "(?s:/path/using/globs/.*/file\\.extension)\\Z", -# ], -# } -# } -# result = validate_yaml(user_input) -# assert result == expected_result - - -# def test_components_schema(): -# user_input = { -# "component_management": { -# "default_rules": { -# "flag_regexes": ["global_flag"], -# }, -# "individual_components": [ -# { -# "name": "fruits", -# "component_id": "app_0", -# "flag_regexes": ["fruit_.*", "^specific_flag$"], -# "paths": ["src/.*"], -# "statuses": [{"type": "patch", "name_prefix": "co", "target": 90}], -# } -# ], -# } -# } -# expected = { -# "component_management": { -# "default_rules": { -# "flag_regexes": ["global_flag"], -# }, -# "individual_components": [ -# { -# "name": "fruits", -# "component_id": "app_0", -# "flag_regexes": ["fruit_.*", "^specific_flag$"], -# "paths": ["src/.*"], -# "statuses": [ -# {"type": "patch", "name_prefix": "co", "target": 90.0} -# ], -# } -# ], -# } -# } -# result = validate_yaml(user_input) -# assert result == expected - - -# def test_components_schema_error(): -# user_input = { -# "component_management": { -# "individual_components": [ -# { -# "key": "extra", -# "component_id": "app_0", -# "flag_regexes": ["fruit_*", "^specific_flag$"], -# "path_filter_regexes": ["src/.*"], -# "statuses": [ -# {"type": "patch", "name_prefix": "co", "target": 90.0} -# ], -# }, -# { -# "component_id": "app_0", -# "flag_regexes": ["fruit_*", "^specific_flag$"], -# "path_filter_regexes": ["src/.*"], -# }, -# ], -# } -# } -# with pytest.raises(InvalidYamlException) as exp: -# validate_yaml(user_input) -# assert exp.error_location == [ -# "component_management", -# "individual_components", -# 0, -# "key", -# ] -# assert exp.error_message == "unknown field" -# assert exp.error_dict == { -# "component_management": [ -# { -# "individual_components": [ -# { -# 0: [{"key": ["unknown field"]}], -# 1: [{"component_id": ["required field"]}], -# } -# ] -# } -# ] -# } - - -# def test_removed_code_behavior_config_valid(): -# user_input = { -# "coverage": { -# "status": { -# "project": { -# "some_status": {"removed_code_behavior": "removals_only"}, -# } -# } -# }, -# "flag_management": { -# "default_rules": { -# "statuses": [ -# {"name_prefix": "custom", "removed_code_behavior": "adjust_base"} -# ] -# }, -# "individual_flags": [ -# { -# "name": "random", -# "statuses": [ -# { -# "name_prefix": "random-custom", -# "removed_code_behavior": False, -# } -# ], -# } -# ], -# }, -# "component_management": { -# "default_rules": { -# "statuses": [ -# { -# "name_prefix": "custom", -# "removed_code_behavior": "fully_covered_patch", -# } -# ] -# }, -# "individual_components": [ -# { -# "component_id": "random", -# "statuses": [ -# { -# "name_prefix": "random-custom", -# "removed_code_behavior": "off", -# } -# ], -# } -# ], -# }, -# } -# result = validate_yaml(user_input) -# # There's no change on the valid yaml -# assert result == user_input - - -# def test_offset_config_error(): -# user_input = { -# "flag_management": { -# "default_rules": { -# "statuses": [ -# {"name_prefix": "custom", "removed_code_behavior": "banana"} -# ] -# } -# }, -# } - -# with pytest.raises(InvalidYamlException) as exp: -# validate_yaml(user_input) -# assert exp.error_dict == { -# "coverage": [ -# { -# "status": [ -# {"patch": [{"some_status": [{"offset": ["unknown field"]}]}]} -# ] -# } -# ], -# "flag_management": [ -# { -# "default_rules": [ -# {"statuses": [{0: [{"offset": ["unallowed value banana"]}]}]} -# ] -# } -# ], -# } - - -# def test_cli_validation(): -# user_input = { -# "cli": { -# "plugins": {"pycoverage": {"report_type": "json"}}, -# "runners": { -# "custom_runner": { -# "module": "my_project.runner", -# "class": "MyCustomRunner", -# "params": {"randseed": 0}, -# } -# }, -# } -# } -# result = validate_yaml(user_input) -# # There's no change on the valid yaml -# assert result == user_input - - -# def test_slack_app_validation(): -# user_input = {"slack_app": {"enabled": True}} -# result = validate_yaml(user_input) -# assert result == user_input - - -# def test_slack_app_validation_boolean(): -# user_input = {"slack_app": True} -# result = validate_yaml(user_input) -# assert result == user_input - - -# def test_to_string_validation(): -# user_input = {"to_string": {"abc": 123}} -# expected_result = {} -# result = validate_yaml(user_input) -# assert result == expected_result +def test_cli_validation(): + user_input = { + "cli": { + "plugins": {"pycoverage": {"report_type": "json"}}, + "runners": { + "custom_runner": { + "module": "my_project.runner", + "class": "MyCustomRunner", + "params": {"randseed": 0}, + } + }, + } + } + result = validate_yaml(user_input) + # There's no change on the valid yaml + assert result == user_input + + +def test_slack_app_validation(): + user_input = {"slack_app": {"enabled": True}} + result = validate_yaml(user_input) + assert result == user_input + + +def test_slack_app_validation_boolean(): + user_input = {"slack_app": True} + result = validate_yaml(user_input) + assert result == user_input + + +def test_to_string_validation(): + user_input = {"to_string": {"abc": 123}} + expected_result = {} + result = validate_yaml(user_input) + assert result == expected_result From 4203249a2bab549af8d484572bc79cd2cedcd640 Mon Sep 17 00:00:00 2001 From: Suejung Shin Date: Tue, 8 Apr 2025 10:27:52 -0700 Subject: [PATCH 04/10] add test --- shared/validation/user_schema.py | 2 -- tests/unit/validation/test_validation.py | 40 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/shared/validation/user_schema.py b/shared/validation/user_schema.py index c42b436d2..cc6b346d5 100644 --- a/shared/validation/user_schema.py +++ b/shared/validation/user_schema.py @@ -563,7 +563,6 @@ "default_rules": { "type": "dict", "schema": component_rule_basic_properties, - # "return_error_when_dict": True, }, "individual_components": { "type": "list", @@ -574,7 +573,6 @@ "name": {"type": "string"}, "component_id": {"type": "string", "required": True}, }, - # "return_error_when_dict": True, }, }, }, diff --git a/tests/unit/validation/test_validation.py b/tests/unit/validation/test_validation.py index 75c2fec0d..46f682ff6 100644 --- a/tests/unit/validation/test_validation.py +++ b/tests/unit/validation/test_validation.py @@ -1330,6 +1330,46 @@ def test_components_schema_error(): ] } +def test_components_schema_error_for_key_named_type(): + user_input = { + "component_management": { + "default_rules": { + "flag_regexes": ["global_flag"], + }, + "individual_components": [ + { + "name": "fruits", + "component_id": "app_0", + "flag_regexes": ["fruit_.*", "^specific_flag$"], + "paths": ["src/.*"], + # expected "statuses" to be a list but is an object instead & that + # object contains a key named "type" (reserved word) + "statuses": {"type": "patch", "name_prefix": "co", "target": 90}, + } + ], + } + } + with pytest.raises(InvalidYamlException) as exp: + validate_yaml(user_input) + assert exp.error_location == [ + "component_management", + "individual_components", + 0, + "statuses", + ] + assert exp.error_message == "must be of list type" + assert exp.error_dict == { + "component_management": [ + { + "individual_components": [ + { + 0: [{"statuses": ["must be of list type"]}], + } + ] + } + ] + } + def test_removed_code_behavior_config_valid(): user_input = { From b07f800254c9570b9b460fcebc1acd3d13d37a32 Mon Sep 17 00:00:00 2001 From: Suejung Shin Date: Tue, 8 Apr 2025 13:35:33 -0700 Subject: [PATCH 05/10] add tests --- shared/validation/user_schema.py | 11 +- shared/validation/validator.py | 22 ++- tests/unit/validation/test_validation.py | 182 +++++++++++++++-------- 3 files changed, 139 insertions(+), 76 deletions(-) diff --git a/shared/validation/user_schema.py b/shared/validation/user_schema.py index cc6b346d5..a6658ccc6 100644 --- a/shared/validation/user_schema.py +++ b/shared/validation/user_schema.py @@ -125,7 +125,10 @@ "anyof": [ { "type": "list", - "schema": {"type": "dict", "schema": flag_status_attributes}, + "schema": { + "type": "dict", + "schema": flag_status_attributes, + }, }, ] }, @@ -146,11 +149,11 @@ { "type": "list", "schema": { - "type": "dict", + "type": "dict", "schema": component_status_attributes, - } + }, } - ] + ], }, "flag_regexes": {"type": "list", "schema": {"type": "string"}}, "paths": path_list_structure, diff --git a/shared/validation/validator.py b/shared/validation/validator.py index 69c5ccf43..077e68b1b 100644 --- a/shared/validation/validator.py +++ b/shared/validation/validator.py @@ -1,18 +1,14 @@ from cerberus import Validator -from shared.validation.helpers import ( - BranchSchemaField, - BundleSizeThresholdSchemaField, - ByteSizeSchemaField, - CoverageCommentRequirementSchemaField, - CoverageRangeSchemaField, - CustomFixPathSchemaField, - Invalid, - LayoutStructure, - PathPatternSchemaField, - PercentSchemaField, - UserGivenBranchRegex, -) +from shared.validation.helpers import (BranchSchemaField, + BundleSizeThresholdSchemaField, + ByteSizeSchemaField, + CoverageCommentRequirementSchemaField, + CoverageRangeSchemaField, + CustomFixPathSchemaField, Invalid, + LayoutStructure, PathPatternSchemaField, + PercentSchemaField, + UserGivenBranchRegex) class CodecovYamlValidator(Validator): diff --git a/tests/unit/validation/test_validation.py b/tests/unit/validation/test_validation.py index 46f682ff6..093a392ab 100644 --- a/tests/unit/validation/test_validation.py +++ b/tests/unit/validation/test_validation.py @@ -7,9 +7,7 @@ from shared.validation.exceptions import InvalidYamlException from shared.yaml.validation import ( _calculate_error_location_and_message_from_error_dict, - do_actual_validation, - validate_yaml, -) + do_actual_validation, validate_yaml) from tests.base import BaseTestCase @@ -459,6 +457,58 @@ def test_many_flags_validation(self): } assert validate_yaml(user_input) == expected_result + def test_flags_schema_error_for_key_named_type(self): + user_input = { + "flag_management": { + "default_rules": { + "carryforward": False, + "statuses": [{"name_prefix": "aaa", "type": "patch"}], + }, + "individual_flags": [ + {"name": "cawcaw", + "paths": ["banana"], + "after_n_builds": 3, + # expected "statuses" to be a list but is an object instead & that + # object contains a key named "type" (reserved word) + "statuses": {"type": "patch"} + } + ], + }, + } + + with pytest.raises(Exception) as exp: + validate_yaml(user_input) + + err = exp.value + assert err is not None, "validate_yaml didn't raise anything" + assert err.error_location == [ + "flag_management", + "individual_flags", + 0, + "statuses", + ] + assert err.error_message == "no definitions validate" + assert err.error_dict == { + "flag_management": [ + { + "individual_flags": [ + { + 0: [ + { + "statuses": [ + "no definitions validate", + {"anyof definition 0": [ + "must be of list type" + ]} + ] + } + ], + } + ] + } + ] + } + def test_validate_bot_none(self): user_input = {"codecov": {"bot": None}} expected_result = {"codecov": {"bot": None}} @@ -767,44 +817,44 @@ def test_yaml_with_flag_management(self): result = validate_yaml(user_input) assert result == expected_result - def test_yaml_with_flag_management_statuses_with_flags(self): - user_input = { - "flag_management": { - "default_rules": { - "carryforward": True, - "statuses": [ - { - "type": "project", - "name_prefix": "healthcare", - "threshold": 80, - "flags": ["hahaha"], - } - ], - }, - "individual_flags": [ - { - "name": "flag_banana", - "statuses": [ - { - "type": "patch", - "name_prefix": "alliance", - "flag_coverage_not_uploaded_behavior": "include", - } - ], - } - ], - } - } - with pytest.raises(InvalidYamlException) as exc: - validate_yaml(user_input) - assert exc.value.error_location == [ - "flag_management", - "default_rules", - "statuses", - 0, - "flags", - ] - assert exc.value.error_message == "extra keys not allowed" + # def test_yaml_with_flag_management_statuses_with_flags(self): + # user_input = { + # "flag_management": { + # "default_rules": { + # "carryforward": True, + # "statuses": [ + # { + # "type": "project", + # "name_prefix": "healthcare", + # "threshold": 80, + # "flags": ["hahaha"], + # } + # ], + # }, + # "individual_flags": [ + # { + # "name": "flag_banana", + # "statuses": [ + # { + # "type": "patch", + # "name_prefix": "alliance", + # "flag_coverage_not_uploaded_behavior": "include", + # } + # ], + # } + # ], + # } + # } + # with pytest.raises(InvalidYamlException) as exc: + # validate_yaml(user_input) + # assert exc.value.error_location == [ + # "flag_management", + # "default_rules", + # "statuses", + # 0, + # "flags", + # ] + # assert exc.value.error_message == "extra keys not allowed" def test_github_checks(self): user_input = {"github_checks": True} @@ -1349,26 +1399,40 @@ def test_components_schema_error_for_key_named_type(): ], } } - with pytest.raises(InvalidYamlException) as exp: + + with pytest.raises(Exception) as exp: validate_yaml(user_input) - assert exp.error_location == [ - "component_management", - "individual_components", - 0, - "statuses", + + err = exp.value + assert err is not None, "validate_yaml didn't raise anything" + assert err.error_location == [ + "component_management", + "individual_components", + 0, + "statuses", + ] + assert err.error_message == "no definitions validate" + assert err.error_dict == { + "component_management": [ + { + "individual_components": [ + { + 0: [ + { + "statuses": [ + "no definitions validate", + {"anyof definition 0": [ + "must be of list type" + ]} + ] + } + ], + } + ] + } ] - assert exp.error_message == "must be of list type" - assert exp.error_dict == { - "component_management": [ - { - "individual_components": [ - { - 0: [{"statuses": ["must be of list type"]}], - } - ] - } - ] - } + } + def test_removed_code_behavior_config_valid(): From 2ac5c3a759ab0c085246b8c55b47381ed02b9d96 Mon Sep 17 00:00:00 2001 From: Suejung Shin Date: Tue, 8 Apr 2025 13:46:06 -0700 Subject: [PATCH 06/10] cleanup --- shared/validation/validator.py | 22 ++++--- tests/unit/validation/test_validation.py | 76 ++++++++++++------------ 2 files changed, 51 insertions(+), 47 deletions(-) diff --git a/shared/validation/validator.py b/shared/validation/validator.py index 077e68b1b..69c5ccf43 100644 --- a/shared/validation/validator.py +++ b/shared/validation/validator.py @@ -1,14 +1,18 @@ from cerberus import Validator -from shared.validation.helpers import (BranchSchemaField, - BundleSizeThresholdSchemaField, - ByteSizeSchemaField, - CoverageCommentRequirementSchemaField, - CoverageRangeSchemaField, - CustomFixPathSchemaField, Invalid, - LayoutStructure, PathPatternSchemaField, - PercentSchemaField, - UserGivenBranchRegex) +from shared.validation.helpers import ( + BranchSchemaField, + BundleSizeThresholdSchemaField, + ByteSizeSchemaField, + CoverageCommentRequirementSchemaField, + CoverageRangeSchemaField, + CustomFixPathSchemaField, + Invalid, + LayoutStructure, + PathPatternSchemaField, + PercentSchemaField, + UserGivenBranchRegex, +) class CodecovYamlValidator(Validator): diff --git a/tests/unit/validation/test_validation.py b/tests/unit/validation/test_validation.py index 093a392ab..0fe714f79 100644 --- a/tests/unit/validation/test_validation.py +++ b/tests/unit/validation/test_validation.py @@ -817,44 +817,44 @@ def test_yaml_with_flag_management(self): result = validate_yaml(user_input) assert result == expected_result - # def test_yaml_with_flag_management_statuses_with_flags(self): - # user_input = { - # "flag_management": { - # "default_rules": { - # "carryforward": True, - # "statuses": [ - # { - # "type": "project", - # "name_prefix": "healthcare", - # "threshold": 80, - # "flags": ["hahaha"], - # } - # ], - # }, - # "individual_flags": [ - # { - # "name": "flag_banana", - # "statuses": [ - # { - # "type": "patch", - # "name_prefix": "alliance", - # "flag_coverage_not_uploaded_behavior": "include", - # } - # ], - # } - # ], - # } - # } - # with pytest.raises(InvalidYamlException) as exc: - # validate_yaml(user_input) - # assert exc.value.error_location == [ - # "flag_management", - # "default_rules", - # "statuses", - # 0, - # "flags", - # ] - # assert exc.value.error_message == "extra keys not allowed" + def test_yaml_with_flag_management_statuses_with_flags(self): + user_input = { + "flag_management": { + "default_rules": { + "carryforward": True, + "statuses": [ + { + "type": "project", + "name_prefix": "healthcare", + "threshold": 80, + "flags": ["hahaha"], + } + ], + }, + "individual_flags": [ + { + "name": "flag_banana", + "statuses": [ + { + "type": "patch", + "name_prefix": "alliance", + "flag_coverage_not_uploaded_behavior": "include", + } + ], + } + ], + } + } + with pytest.raises(InvalidYamlException) as exc: + validate_yaml(user_input) + assert exc.value.error_location == [ + "flag_management", + "default_rules", + "statuses", + 0, + "flags", + ] + assert exc.value.error_message == "extra keys not allowed" def test_github_checks(self): user_input = {"github_checks": True} From fac86cf840c67c8a9b98eec88f3819b47b98cbda Mon Sep 17 00:00:00 2001 From: Suejung Shin Date: Tue, 8 Apr 2025 14:18:52 -0700 Subject: [PATCH 07/10] fix allow_unknown --- shared/validation/user_schema.py | 4 ++- tests/unit/validation/test_validation.py | 36 +++++++++++++++++++----- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/shared/validation/user_schema.py b/shared/validation/user_schema.py index a6658ccc6..cc30cb572 100644 --- a/shared/validation/user_schema.py +++ b/shared/validation/user_schema.py @@ -128,7 +128,8 @@ "schema": { "type": "dict", "schema": flag_status_attributes, - }, + "allow_unknown": False + } }, ] }, @@ -151,6 +152,7 @@ "schema": { "type": "dict", "schema": component_status_attributes, + "allow_unknown": False }, } ], diff --git a/tests/unit/validation/test_validation.py b/tests/unit/validation/test_validation.py index 0fe714f79..3410aa6db 100644 --- a/tests/unit/validation/test_validation.py +++ b/tests/unit/validation/test_validation.py @@ -847,14 +847,36 @@ def test_yaml_with_flag_management_statuses_with_flags(self): } with pytest.raises(InvalidYamlException) as exc: validate_yaml(user_input) - assert exc.value.error_location == [ - "flag_management", - "default_rules", - "statuses", - 0, - "flags", + assert exc.value.error_location == [ + "flag_management", + "default_rules", + "statuses" + ] + assert exc.value.error_message == "no definitions validate" + assert exc.value.error_dict == { + "flag_management": [ + { + "default_rules": [ + { + "statuses": [ + "no definitions validate", + { + "anyof definition 0": [ + { + 0: [ + { + "flags": ["unknown field"] + } + ] + } + ] + } + ] + } + ] + } ] - assert exc.value.error_message == "extra keys not allowed" + } def test_github_checks(self): user_input = {"github_checks": True} From 9278747f51ceab6049c18d8bc28fd4e2d0478906 Mon Sep 17 00:00:00 2001 From: Suejung Shin Date: Tue, 8 Apr 2025 14:33:34 -0700 Subject: [PATCH 08/10] retrigger tests --- tests/unit/validation/test_validation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/validation/test_validation.py b/tests/unit/validation/test_validation.py index 3410aa6db..9f128ab5a 100644 --- a/tests/unit/validation/test_validation.py +++ b/tests/unit/validation/test_validation.py @@ -1424,7 +1424,6 @@ def test_components_schema_error_for_key_named_type(): with pytest.raises(Exception) as exp: validate_yaml(user_input) - err = exp.value assert err is not None, "validate_yaml didn't raise anything" assert err.error_location == [ From 4506cae1c2e3cb72e4b11bd128c860bf1a4d75d9 Mon Sep 17 00:00:00 2001 From: Suejung Shin Date: Tue, 8 Apr 2025 14:34:53 -0700 Subject: [PATCH 09/10] run lint --- shared/validation/user_schema.py | 2 +- tests/unit/validation/test_validation.py | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/shared/validation/user_schema.py b/shared/validation/user_schema.py index cc30cb572..5649697ae 100644 --- a/shared/validation/user_schema.py +++ b/shared/validation/user_schema.py @@ -78,7 +78,7 @@ custom_status_common_config = { "name_prefix": {"type": "string", "regex": r"^[\w\-\.]+$"}, - # Note that "type" is a reserved word in Cerberus parser so use with caution as a + # Note that "type" is a reserved word in Cerberus parser so use with caution as a # key in the schema. See workaround at places that call this schema. "type": {"type": "string", "allowed": ("project", "patch", "changes")}, "target": percent_type_or_auto, diff --git a/tests/unit/validation/test_validation.py b/tests/unit/validation/test_validation.py index 9f128ab5a..269e374f7 100644 --- a/tests/unit/validation/test_validation.py +++ b/tests/unit/validation/test_validation.py @@ -7,7 +7,9 @@ from shared.validation.exceptions import InvalidYamlException from shared.yaml.validation import ( _calculate_error_location_and_message_from_error_dict, - do_actual_validation, validate_yaml) + do_actual_validation, + validate_yaml, +) from tests.base import BaseTestCase @@ -465,10 +467,10 @@ def test_flags_schema_error_for_key_named_type(self): "statuses": [{"name_prefix": "aaa", "type": "patch"}], }, "individual_flags": [ - {"name": "cawcaw", - "paths": ["banana"], - "after_n_builds": 3, - # expected "statuses" to be a list but is an object instead & that + {"name": "cawcaw", + "paths": ["banana"], + "after_n_builds": 3, + # expected "statuses" to be a list but is an object instead & that # object contains a key named "type" (reserved word) "statuses": {"type": "patch"} } @@ -478,7 +480,7 @@ def test_flags_schema_error_for_key_named_type(self): with pytest.raises(Exception) as exp: validate_yaml(user_input) - + err = exp.value assert err is not None, "validate_yaml didn't raise anything" assert err.error_location == [ @@ -1414,7 +1416,7 @@ def test_components_schema_error_for_key_named_type(): "component_id": "app_0", "flag_regexes": ["fruit_.*", "^specific_flag$"], "paths": ["src/.*"], - # expected "statuses" to be a list but is an object instead & that + # expected "statuses" to be a list but is an object instead & that # object contains a key named "type" (reserved word) "statuses": {"type": "patch", "name_prefix": "co", "target": 90}, } @@ -1453,7 +1455,7 @@ def test_components_schema_error_for_key_named_type(): } ] } - + def test_removed_code_behavior_config_valid(): From 1c27a0bb7255988e016a353e2e22855f8ed7750e Mon Sep 17 00:00:00 2001 From: Suejung Shin Date: Tue, 8 Apr 2025 14:38:31 -0700 Subject: [PATCH 10/10] fix lint --- shared/validation/user_schema.py | 6 ++-- tests/unit/validation/test_validation.py | 41 +++++++++++------------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/shared/validation/user_schema.py b/shared/validation/user_schema.py index 5649697ae..fa60276c8 100644 --- a/shared/validation/user_schema.py +++ b/shared/validation/user_schema.py @@ -128,8 +128,8 @@ "schema": { "type": "dict", "schema": flag_status_attributes, - "allow_unknown": False - } + "allow_unknown": False, + }, }, ] }, @@ -152,7 +152,7 @@ "schema": { "type": "dict", "schema": component_status_attributes, - "allow_unknown": False + "allow_unknown": False, }, } ], diff --git a/tests/unit/validation/test_validation.py b/tests/unit/validation/test_validation.py index 269e374f7..add107f88 100644 --- a/tests/unit/validation/test_validation.py +++ b/tests/unit/validation/test_validation.py @@ -467,13 +467,14 @@ def test_flags_schema_error_for_key_named_type(self): "statuses": [{"name_prefix": "aaa", "type": "patch"}], }, "individual_flags": [ - {"name": "cawcaw", - "paths": ["banana"], - "after_n_builds": 3, - # expected "statuses" to be a list but is an object instead & that - # object contains a key named "type" (reserved word) - "statuses": {"type": "patch"} - } + { + "name": "cawcaw", + "paths": ["banana"], + "after_n_builds": 3, + # expected "statuses" to be a list but is an object instead & that + # object contains a key named "type" (reserved word) + "statuses": {"type": "patch"}, + } ], }, } @@ -499,9 +500,11 @@ def test_flags_schema_error_for_key_named_type(self): { "statuses": [ "no definitions validate", - {"anyof definition 0": [ - "must be of list type" - ]} + { + "anyof definition 0": [ + "must be of list type" + ] + }, ] } ], @@ -852,7 +855,7 @@ def test_yaml_with_flag_management_statuses_with_flags(self): assert exc.value.error_location == [ "flag_management", "default_rules", - "statuses" + "statuses", ] assert exc.value.error_message == "no definitions validate" assert exc.value.error_dict == { @@ -864,15 +867,9 @@ def test_yaml_with_flag_management_statuses_with_flags(self): "no definitions validate", { "anyof definition 0": [ - { - 0: [ - { - "flags": ["unknown field"] - } - ] - } + {0: [{"flags": ["unknown field"]}]} ] - } + }, ] } ] @@ -1404,6 +1401,7 @@ def test_components_schema_error(): ] } + def test_components_schema_error_for_key_named_type(): user_input = { "component_management": { @@ -1444,9 +1442,7 @@ def test_components_schema_error_for_key_named_type(): { "statuses": [ "no definitions validate", - {"anyof definition 0": [ - "must be of list type" - ]} + {"anyof definition 0": ["must be of list type"]}, ] } ], @@ -1457,7 +1453,6 @@ def test_components_schema_error_for_key_named_type(): } - def test_removed_code_behavior_config_valid(): user_input = { "coverage": {