From 62a3c1b2b47fe8e616b2a40f8da0fe1cd29fdf8e Mon Sep 17 00:00:00 2001 From: Florin Blanaru Date: Tue, 27 Sep 2022 16:23:54 +0300 Subject: [PATCH] [ci] Add support to the skipped tests script to assert tests in the `required_tests_to_run` file are not skipped. --- ci/scripts/github_skipped_tests_comment.py | 77 +++++++++++++-- ci/scripts/required_tests_to_run.json | 11 +++ tests/python/ci/test_ci.py | 108 ++++++++++++++++++++- 3 files changed, 183 insertions(+), 13 deletions(-) create mode 100644 ci/scripts/required_tests_to_run.json diff --git a/ci/scripts/github_skipped_tests_comment.py b/ci/scripts/github_skipped_tests_comment.py index 7a62f16a5b81..46e579105324 100755 --- a/ci/scripts/github_skipped_tests_comment.py +++ b/ci/scripts/github_skipped_tests_comment.py @@ -15,6 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import inspect +import json import os import logging import subprocess @@ -102,10 +104,30 @@ def to_node_name(dir_name: str): return dir_name.replace("_", ": ", 1) +def build_diff_comment_with_main( + common_commit_sha, + skipped_list, + commit_sha, +): + if len(skipped_list) == 0: + return f"No diff in skipped tests with main found in this branch for commit {commit_sha}.\n" + + text = ( + f"The list below shows tests that ran in main {common_commit_sha} but were " + f"skipped in the CI build of {commit_sha}:\n" + f"```\n" + ) + for skip in skipped_list: + text += skip + "\n" + text += f"```\n" + return text + + def build_comment( common_commit_sha, common_main_build, skipped_list, + additional_skipped_list, pr_number, build_number, commit_sha, @@ -114,18 +136,21 @@ def build_comment( if common_main_build["state"] != "success": return f"Unable to run tests bot because main failed to pass CI at {common_commit_sha}." - if len(skipped_list) == 0: - return f"No additional skipped tests found in this branch for commit {commit_sha}." + text = build_diff_comment_with_main(common_commit_sha, skipped_list, commit_sha) + + if len(additional_skipped_list) != 0: + text += "\n" + text += ( + f"Additional tests that were skipped in the CI build and present in the [`required_tests_to_run`]" + f"(https://github.com/apache/tvm/blob/main/ci/scripts/required_tests_to_run.json) file:" + f"\n```\n" + ) + for skip in additional_skipped_list: + text += skip + "\n" + text += f"```\n" - text = ( - f"The list below shows some tests that ran in main {common_commit_sha} but were " - f"skipped in the CI build of {commit_sha}:\n" - f"```\n" - ) - for skip in skipped_list: - text += skip + "\n" text += ( - f"```\nA detailed report of ran tests is [here](https://{jenkins_prefix}/job/tvm/job/PR-{str(pr_number)}" + f"A detailed report of ran tests is [here](https://{jenkins_prefix}/job/tvm/job/PR-{str(pr_number)}" f"/{str(build_number)}/testReport/)." ) return text @@ -148,6 +173,7 @@ def get_skipped_tests_comment( main_test_report_dir: str = "main-reports", common_commit_sha: Optional[str] = None, common_main_build: Optional[Dict[str, Any]] = None, + additional_tests_to_check_file: str = "required_tests_to_run.json", ) -> str: pr_head = pr["commits"]["nodes"][0]["commit"] target_url = find_target_url(pr_head) @@ -195,10 +221,41 @@ def get_skipped_tests_comment( if len(skipped_list) == 0: logging.info("No skipped tests found.") + if not is_dry_run: + current_file = Path(__file__).resolve() + additional_tests_to_check_file = Path(current_file).parent / "required_tests_to_run.json" + + logging.info( + f"Checking additional tests in file {additional_tests_to_check_file} are not skipped." + ) + try: + with open(additional_tests_to_check_file, "r") as f: + additional_tests_to_check = json.load(f) + except IOError: + logging.info( + f"Failed to read additional tests from file: {additional_tests_to_check_file}." + ) + additional_tests_to_check = {} + + # Assert that tests present in "required_tests_to_run.json" are not skipped. + additional_skipped_tests = [] + for subdir, test_set in additional_tests_to_check.items(): + if subdir not in build_tests.keys(): + logging.warning(f"Could not find directory {subdir} in the build test set.") + continue + + for test in test_set: + if test in build_tests[subdir]: + additional_skipped_tests.append(f"{to_node_name(subdir)} -> {test}") + + if len(additional_skipped_tests) == 0: + logging.info("No skipped tests found in the additional list.") + body = build_comment( common_commit_sha, common_main_build, skipped_list, + additional_skipped_tests, pr_and_build["pr_number"], pr_and_build["build_number"], commit_sha, diff --git a/ci/scripts/required_tests_to_run.json b/ci/scripts/required_tests_to_run.json new file mode 100644 index 000000000000..8bd265c04fe0 --- /dev/null +++ b/ci/scripts/required_tests_to_run.json @@ -0,0 +1,11 @@ +{ + "unittest_GPU": + [ + "ctypes.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_integration_extract_from_bert_base", + "cython.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_integration_extract_from_bert_base", + "ctypes.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_dynamic_loop_extent", + "cython.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_dynamic_loop_extent", + "ctypes.tests.python.unittest.test_meta_schedule_integration#test_extract_task_arm_conv2d_nchwc", + "cython.tests.python.unittest.test_meta_schedule_integration#test_extract_task_arm_conv2d_nchwc" + ] +} diff --git a/tests/python/ci/test_ci.py b/tests/python/ci/test_ci.py index 4b8c5d9ad444..7b7b7298d8b3 100644 --- a/tests/python/ci/test_ci.py +++ b/tests/python/ci/test_ci.py @@ -46,7 +46,7 @@ def parameterize_named(**kwargs): # pylint: disable=line-too-long TEST_DATA_SKIPPED_BOT = { - "found-diff": { + "found-diff-no-additional": { "main_xml_file": "unittest/file1.xml", "main_xml_content": """ @@ -78,12 +78,61 @@ def parameterize_named(**kwargs): """, + "additional_tests_to_check": """{ + "unittest": ["dummy_class#dummy_test"], + "unittest_GPU": ["another_dummy_class#another_dummy_test"] + } + """, + "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect", + "s3_prefix": "tvm-jenkins-artifacts-prod", + "jenkins_prefix": "ci.tlcpack.ai", + "common_main_build": """{"build_number": "4115", "state": "success"}""", + "commit_sha": "sha1234", + "expected_body": "The list below shows tests that ran in main sha1234 but were skipped in the CI build of sha1234:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).", + }, + "found-diff-skipped-additional": { + "main_xml_file": "unittest/file1.xml", + "main_xml_content": """ + + + + + + + """, + "pr_xml_file": "unittest/file2.xml", + "pr_xml_content": """ + + + + + Skipped + + + + + Skipped + + + + + """, + "additional_tests_to_check": """{ + "unittest": ["ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner", "dummy_class#dummy_test"], + "unittest_GPU": ["another_dummy_class#another_dummy_test"] + } + """, "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect", "s3_prefix": "tvm-jenkins-artifacts-prod", "jenkins_prefix": "ci.tlcpack.ai", "common_main_build": """{"build_number": "4115", "state": "success"}""", "commit_sha": "sha1234", - "expected_body": "The list below shows some tests that ran in main sha1234 but were skipped in the CI build of sha1234:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).", + "expected_body": "The list below shows tests that ran in main sha1234 but were skipped in the CI build of sha1234:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\n\nAdditional tests that were skipped in the CI build and present in the [`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/required_tests_to_run.json) file:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).", }, "no-diff": { "main_xml_file": "unittest/file1.xml", @@ -114,12 +163,56 @@ def parameterize_named(**kwargs): """, + "additional_tests_to_check": """{ + } + """, + "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect", + "s3_prefix": "tvm-jenkins-artifacts-prod", + "jenkins_prefix": "ci.tlcpack.ai", + "common_main_build": """{"build_number": "4115", "state": "success"}""", + "commit_sha": "sha1234", + "expected_body": "No diff in skipped tests with main found in this branch for commit sha1234.\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).", + }, + "no-diff-skipped-additional": { + "main_xml_file": "unittest/file1.xml", + "main_xml_content": """ + + + + + Skipped + + + + + """, + "pr_xml_file": "unittest/file2.xml", + "pr_xml_content": """ + + + + + Skipped + + + + + """, + "additional_tests_to_check": """{ + "unittest": ["dummy_class#dummy_test", "ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner"], + "unittest_GPU": ["another_dummy_class#another_dummy_test"] + } + """, "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect", "s3_prefix": "tvm-jenkins-artifacts-prod", "jenkins_prefix": "ci.tlcpack.ai", "common_main_build": """{"build_number": "4115", "state": "success"}""", "commit_sha": "sha1234", - "expected_body": "No additional skipped tests found in this branch for commit sha1234.", + "expected_body": "No diff in skipped tests with main found in this branch for commit sha1234.\n\nAdditional tests that were skipped in the CI build and present in the [`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/required_tests_to_run.json) file:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).", }, "unable-to-run": { "main_xml_file": "unittest/file1.xml", @@ -132,6 +225,11 @@ def parameterize_named(**kwargs): """, + "additional_tests_to_check": """{ + "unittest": ["ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner", "dummy_class#dummy_test"], + "unittest_GPU": ["another_dummy_class#another_dummy_test"] + } + """, "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect", "s3_prefix": "tvm-jenkins-artifacts-prod", "jenkins_prefix": "ci.tlcpack.ai", @@ -153,6 +251,7 @@ def test_skipped_tests_comment( main_xml_content, pr_xml_file, pr_xml_content, + additional_tests_to_check, target_url, s3_prefix, jenkins_prefix, @@ -176,6 +275,8 @@ def write_xml_file(root_dir, xml_file, xml_content): write_xml_file(pr_test_report_dir, pr_xml_file, pr_xml_content) main_test_report_dir = Path(git.cwd) / "main-reports" write_xml_file(main_test_report_dir, main_xml_file, main_xml_content) + with open(Path(git.cwd) / "required_tests_to_run.json", "w") as f: + f.write(additional_tests_to_check) pr_data = { "commits": { @@ -208,6 +309,7 @@ def write_xml_file(root_dir, xml_file, xml_content): pr_test_report_dir=pr_test_report_dir, main_test_report_dir=main_test_report_dir, common_main_build=json.loads(common_main_build), + additional_tests_to_check_file=Path(git.cwd) / "required_tests_to_run.json", ) assert_in(expected_body, comment) assert_in(f"with target {target_url}", caplog.text)