diff --git a/ci/scripts/github_skipped_tests_comment.py b/ci/scripts/github_skipped_tests_comment.py
index 7a62f16a5b81..46e579105324 100755
--- a/ci/scripts/github_skipped_tests_comment.py
+++ b/ci/scripts/github_skipped_tests_comment.py
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+import inspect
+import json
import os
import logging
import subprocess
@@ -102,10 +104,30 @@ def to_node_name(dir_name: str):
return dir_name.replace("_", ": ", 1)
+def build_diff_comment_with_main(
+ common_commit_sha,
+ skipped_list,
+ commit_sha,
+):
+ if len(skipped_list) == 0:
+ return f"No diff in skipped tests with main found in this branch for commit {commit_sha}.\n"
+
+ text = (
+ f"The list below shows tests that ran in main {common_commit_sha} but were "
+ f"skipped in the CI build of {commit_sha}:\n"
+ f"```\n"
+ )
+ for skip in skipped_list:
+ text += skip + "\n"
+ text += f"```\n"
+ return text
+
+
def build_comment(
common_commit_sha,
common_main_build,
skipped_list,
+ additional_skipped_list,
pr_number,
build_number,
commit_sha,
@@ -114,18 +136,21 @@ def build_comment(
if common_main_build["state"] != "success":
return f"Unable to run tests bot because main failed to pass CI at {common_commit_sha}."
- if len(skipped_list) == 0:
- return f"No additional skipped tests found in this branch for commit {commit_sha}."
+ text = build_diff_comment_with_main(common_commit_sha, skipped_list, commit_sha)
+
+ if len(additional_skipped_list) != 0:
+ text += "\n"
+ text += (
+ f"Additional tests that were skipped in the CI build and present in the [`required_tests_to_run`]"
+ f"(https://github.com/apache/tvm/blob/main/ci/scripts/required_tests_to_run.json) file:"
+ f"\n```\n"
+ )
+ for skip in additional_skipped_list:
+ text += skip + "\n"
+ text += f"```\n"
- text = (
- f"The list below shows some tests that ran in main {common_commit_sha} but were "
- f"skipped in the CI build of {commit_sha}:\n"
- f"```\n"
- )
- for skip in skipped_list:
- text += skip + "\n"
text += (
- f"```\nA detailed report of ran tests is [here](https://{jenkins_prefix}/job/tvm/job/PR-{str(pr_number)}"
+ f"A detailed report of ran tests is [here](https://{jenkins_prefix}/job/tvm/job/PR-{str(pr_number)}"
f"/{str(build_number)}/testReport/)."
)
return text
@@ -148,6 +173,7 @@ def get_skipped_tests_comment(
main_test_report_dir: str = "main-reports",
common_commit_sha: Optional[str] = None,
common_main_build: Optional[Dict[str, Any]] = None,
+ additional_tests_to_check_file: str = "required_tests_to_run.json",
) -> str:
pr_head = pr["commits"]["nodes"][0]["commit"]
target_url = find_target_url(pr_head)
@@ -195,10 +221,41 @@ def get_skipped_tests_comment(
if len(skipped_list) == 0:
logging.info("No skipped tests found.")
+ if not is_dry_run:
+ current_file = Path(__file__).resolve()
+ additional_tests_to_check_file = Path(current_file).parent / "required_tests_to_run.json"
+
+ logging.info(
+ f"Checking additional tests in file {additional_tests_to_check_file} are not skipped."
+ )
+ try:
+ with open(additional_tests_to_check_file, "r") as f:
+ additional_tests_to_check = json.load(f)
+ except IOError:
+ logging.info(
+ f"Failed to read additional tests from file: {additional_tests_to_check_file}."
+ )
+ additional_tests_to_check = {}
+
+ # Assert that tests present in "required_tests_to_run.json" are not skipped.
+ additional_skipped_tests = []
+ for subdir, test_set in additional_tests_to_check.items():
+ if subdir not in build_tests.keys():
+ logging.warning(f"Could not find directory {subdir} in the build test set.")
+ continue
+
+ for test in test_set:
+ if test in build_tests[subdir]:
+ additional_skipped_tests.append(f"{to_node_name(subdir)} -> {test}")
+
+ if len(additional_skipped_tests) == 0:
+ logging.info("No skipped tests found in the additional list.")
+
body = build_comment(
common_commit_sha,
common_main_build,
skipped_list,
+ additional_skipped_tests,
pr_and_build["pr_number"],
pr_and_build["build_number"],
commit_sha,
diff --git a/ci/scripts/required_tests_to_run.json b/ci/scripts/required_tests_to_run.json
new file mode 100644
index 000000000000..8bd265c04fe0
--- /dev/null
+++ b/ci/scripts/required_tests_to_run.json
@@ -0,0 +1,11 @@
+{
+ "unittest_GPU":
+ [
+ "ctypes.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_integration_extract_from_bert_base",
+ "cython.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_integration_extract_from_bert_base",
+ "ctypes.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_dynamic_loop_extent",
+ "cython.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_dynamic_loop_extent",
+ "ctypes.tests.python.unittest.test_meta_schedule_integration#test_extract_task_arm_conv2d_nchwc",
+ "cython.tests.python.unittest.test_meta_schedule_integration#test_extract_task_arm_conv2d_nchwc"
+ ]
+}
diff --git a/tests/python/ci/test_ci.py b/tests/python/ci/test_ci.py
index 4b8c5d9ad444..7b7b7298d8b3 100644
--- a/tests/python/ci/test_ci.py
+++ b/tests/python/ci/test_ci.py
@@ -46,7 +46,7 @@ def parameterize_named(**kwargs):
# pylint: disable=line-too-long
TEST_DATA_SKIPPED_BOT = {
- "found-diff": {
+ "found-diff-no-additional": {
"main_xml_file": "unittest/file1.xml",
"main_xml_content": """
@@ -78,12 +78,61 @@ def parameterize_named(**kwargs):
""",
+ "additional_tests_to_check": """{
+ "unittest": ["dummy_class#dummy_test"],
+ "unittest_GPU": ["another_dummy_class#another_dummy_test"]
+ }
+ """,
+ "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
+ "s3_prefix": "tvm-jenkins-artifacts-prod",
+ "jenkins_prefix": "ci.tlcpack.ai",
+ "common_main_build": """{"build_number": "4115", "state": "success"}""",
+ "commit_sha": "sha1234",
+ "expected_body": "The list below shows tests that ran in main sha1234 but were skipped in the CI build of sha1234:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).",
+ },
+ "found-diff-skipped-additional": {
+ "main_xml_file": "unittest/file1.xml",
+ "main_xml_content": """
+
+
+
+
+
+
+ """,
+ "pr_xml_file": "unittest/file2.xml",
+ "pr_xml_content": """
+
+
+
+
+ Skipped
+
+
+
+
+ Skipped
+
+
+
+
+ """,
+ "additional_tests_to_check": """{
+ "unittest": ["ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner", "dummy_class#dummy_test"],
+ "unittest_GPU": ["another_dummy_class#another_dummy_test"]
+ }
+ """,
"target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
"s3_prefix": "tvm-jenkins-artifacts-prod",
"jenkins_prefix": "ci.tlcpack.ai",
"common_main_build": """{"build_number": "4115", "state": "success"}""",
"commit_sha": "sha1234",
- "expected_body": "The list below shows some tests that ran in main sha1234 but were skipped in the CI build of sha1234:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).",
+ "expected_body": "The list below shows tests that ran in main sha1234 but were skipped in the CI build of sha1234:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\n\nAdditional tests that were skipped in the CI build and present in the [`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/required_tests_to_run.json) file:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).",
},
"no-diff": {
"main_xml_file": "unittest/file1.xml",
@@ -114,12 +163,56 @@ def parameterize_named(**kwargs):
""",
+ "additional_tests_to_check": """{
+ }
+ """,
+ "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
+ "s3_prefix": "tvm-jenkins-artifacts-prod",
+ "jenkins_prefix": "ci.tlcpack.ai",
+ "common_main_build": """{"build_number": "4115", "state": "success"}""",
+ "commit_sha": "sha1234",
+ "expected_body": "No diff in skipped tests with main found in this branch for commit sha1234.\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).",
+ },
+ "no-diff-skipped-additional": {
+ "main_xml_file": "unittest/file1.xml",
+ "main_xml_content": """
+
+
+
+
+ Skipped
+
+
+
+
+ """,
+ "pr_xml_file": "unittest/file2.xml",
+ "pr_xml_content": """
+
+
+
+
+ Skipped
+
+
+
+
+ """,
+ "additional_tests_to_check": """{
+ "unittest": ["dummy_class#dummy_test", "ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner"],
+ "unittest_GPU": ["another_dummy_class#another_dummy_test"]
+ }
+ """,
"target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
"s3_prefix": "tvm-jenkins-artifacts-prod",
"jenkins_prefix": "ci.tlcpack.ai",
"common_main_build": """{"build_number": "4115", "state": "success"}""",
"commit_sha": "sha1234",
- "expected_body": "No additional skipped tests found in this branch for commit sha1234.",
+ "expected_body": "No diff in skipped tests with main found in this branch for commit sha1234.\n\nAdditional tests that were skipped in the CI build and present in the [`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/required_tests_to_run.json) file:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).",
},
"unable-to-run": {
"main_xml_file": "unittest/file1.xml",
@@ -132,6 +225,11 @@ def parameterize_named(**kwargs):
""",
+ "additional_tests_to_check": """{
+ "unittest": ["ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner", "dummy_class#dummy_test"],
+ "unittest_GPU": ["another_dummy_class#another_dummy_test"]
+ }
+ """,
"target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
"s3_prefix": "tvm-jenkins-artifacts-prod",
"jenkins_prefix": "ci.tlcpack.ai",
@@ -153,6 +251,7 @@ def test_skipped_tests_comment(
main_xml_content,
pr_xml_file,
pr_xml_content,
+ additional_tests_to_check,
target_url,
s3_prefix,
jenkins_prefix,
@@ -176,6 +275,8 @@ def write_xml_file(root_dir, xml_file, xml_content):
write_xml_file(pr_test_report_dir, pr_xml_file, pr_xml_content)
main_test_report_dir = Path(git.cwd) / "main-reports"
write_xml_file(main_test_report_dir, main_xml_file, main_xml_content)
+ with open(Path(git.cwd) / "required_tests_to_run.json", "w") as f:
+ f.write(additional_tests_to_check)
pr_data = {
"commits": {
@@ -208,6 +309,7 @@ def write_xml_file(root_dir, xml_file, xml_content):
pr_test_report_dir=pr_test_report_dir,
main_test_report_dir=main_test_report_dir,
common_main_build=json.loads(common_main_build),
+ additional_tests_to_check_file=Path(git.cwd) / "required_tests_to_run.json",
)
assert_in(expected_body, comment)
assert_in(f"with target {target_url}", caplog.text)