From 150ce469161655cc2c97f2feb0deea074e3ab78c Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Fri, 13 May 2022 09:04:37 -0500 Subject: [PATCH 1/8] [CI] Improved skip messages when using @tvm.testing.requires_* Previously, the same message was given regardless of why a test couldn't be run. This has been split up into separate checks for TVM cmake options in `config.cmake`, enabled targets in `TVM_TEST_TARGETS` environment variable, and checks for available hardware. --- python/tvm/testing/utils.py | 124 +++++++++++++++------- tests/python/driver/tvmc/test_compiler.py | 10 +- 2 files changed, 89 insertions(+), 45 deletions(-) diff --git a/python/tvm/testing/utils.py b/python/tvm/testing/utils.py index 8be5cc8ec471..b699c2a361ef 100644 --- a/python/tvm/testing/utils.py +++ b/python/tvm/testing/utils.py @@ -83,7 +83,7 @@ def test_something(): from tvm.contrib import nvcc, cudnn from tvm.error import TVMError -from tvm.relay.op.contrib.ethosn import ethosn_available +from tvm.relay.op.contrib import ethosn from tvm.relay.op.contrib import cmsisnn from tvm.relay.op.contrib import vitis_ai @@ -514,6 +514,64 @@ def enabled_targets(): return [(t["target"], tvm.device(t["target"])) for t in _get_targets() if t["is_runnable"]] +def _cmake_flag_enabled(flag): + flag = tvm.support.libinfo()[flag] + + # Because many of the flags can be library flags, we check if the + # flag is not disabled, rather than checking if it is enabled. + return flag.lower() not in ["off", "false", "0"] + + +def _cmake_flag_skipif(feature_name, cmake_flag): + return pytest.mark.skipif( + not _cmake_flag_enabled(cmake_flag), + reason=f"{feature_name} support not enabled. Set {cmake_flag}=ON in config.cmake to enable.", + ) + + +def _environment_var_enabled(target): + enabled_str = os.environ.get("TVM_TEST_TARGETS", "") + if enabled_str: + enabled_target_names = set(t.strip() for t in enabled_str.split(";")) + else: + enabled_target_names = DEFAULT_TEST_TARGETS + + target_kind = target.split()[0] + return any(enabled.split()[0] == target_kind for enabled in enabled_target_names) + + +def _environment_var_skipif(target_name, target=None): + if target is None: + target = target_name + + return pytest.mark.skipif( + not _environment_var_enabled(target), + reason=f"{target_name} tests disabled by TVM_TEST_TARGETS", + ) + + +def _device_exists(target): + target_kind = target.split()[0] + return tvm.runtime.enabled(target_kind) and tvm.device(target_kind).exist + + +def _device_skipif(target_name, target=None): + if target is None: + target = target_name + + return pytest.mark.skipif( + not _device_exists(target_kind), reason=f"{target_name} device not present" + ) + + +def _target_skipif_marks(target_name, cmake_flag, target_kind): + return [ + _cmake_flag_skipif(target_name, cmake_flag), + _environment_var_skipif(target_name, target_kind), + _device_skipif(target_name, target_kind), + ] + + def _compose(args, decs): """Helper to apply multiple markers""" if len(args) > 0: @@ -601,12 +659,14 @@ def requires_cuda(*args): f : function Function to mark """ - _requires_cuda = [ + libinfo = tvm.support.libinfo() + + marks = [ pytest.mark.cuda, - pytest.mark.skipif(not device_enabled("cuda"), reason="CUDA support not enabled"), + *_target_skipif_marks("CUDA", "USE_CUDA", "cuda"), *requires_gpu(), ] - return _compose(args, _requires_cuda) + return _compose(args, marks) def requires_cudnn(*args): @@ -621,9 +681,8 @@ def requires_cudnn(*args): """ requirements = [ - pytest.mark.skipif( - not cudnn.exists(), reason="cuDNN library not enabled, or not installed" - ), + _cmake_flag_skipif("cuDNN", "USE_CUDNN"), + pytest.mark.skipif(not cudnn.exists(), reason="cuDNN library not installed"), *requires_cuda(), ] return _compose(args, requirements) @@ -641,10 +700,7 @@ def requires_cublas(*args): """ requirements = [ - pytest.mark.skipif( - tvm.get_global_func("tvm.contrib.cublas.matmul", True), - reason="cuDNN library not enabled", - ), + _cmake_flag_skipif("cuBLAS", "USE_CUBLAS"), *requires_cuda(), ] return _compose(args, requirements) @@ -663,7 +719,7 @@ def requires_nvptx(*args): """ _requires_nvptx = [ - pytest.mark.skipif(not device_enabled("nvptx"), reason="NVPTX support not enabled"), + _device_skipif("nvptx"), *requires_llvm(), *requires_gpu(), ] @@ -754,7 +810,7 @@ def requires_opencl(*args): """ _requires_opencl = [ pytest.mark.opencl, - pytest.mark.skipif(not device_enabled("opencl"), reason="OpenCL support not enabled"), + *_target_skipif_marks("OpenCL", "USE_OPENCL", "opencl"), *requires_gpu(), ] return _compose(args, _requires_opencl) @@ -789,7 +845,7 @@ def requires_rocm(*args): """ _requires_rocm = [ pytest.mark.rocm, - pytest.mark.skipif(not device_enabled("rocm"), reason="rocm support not enabled"), + *_target_skipif_marks("ROCm", "USE_ROCM", "rocm"), *requires_gpu(), ] return _compose(args, _requires_rocm) @@ -807,7 +863,7 @@ def requires_metal(*args): """ _requires_metal = [ pytest.mark.metal, - pytest.mark.skipif(not device_enabled("metal"), reason="metal support not enabled"), + *_target_skipif_marks("metal", "USE_METAL", "metal"), *requires_gpu(), ] return _compose(args, _requires_metal) @@ -825,7 +881,7 @@ def requires_vulkan(*args): """ _requires_vulkan = [ pytest.mark.vulkan, - pytest.mark.skipif(not device_enabled("vulkan"), reason="vulkan support not enabled"), + *_target_skipif_marks("Vulkan", "USE_VULKAN", "vulkan"), *requires_gpu(), ] return _compose(args, _requires_vulkan) @@ -843,6 +899,7 @@ def requires_tensorcore(*args): """ _requires_tensorcore = [ pytest.mark.tensorcore, + *requires_cuda(), pytest.mark.skipif( not tvm.cuda().exist or not nvcc.have_tensorcore(tvm.cuda(0).compute_version), reason="No tensorcore present", @@ -862,7 +919,7 @@ def requires_llvm(*args): """ _requires_llvm = [ pytest.mark.llvm, - pytest.mark.skipif(not device_enabled("llvm"), reason="LLVM support not enabled"), + *_target_skipif_marks("LLVM", "USE_LLVM", "llvm"), ] return _compose(args, _requires_llvm) @@ -876,10 +933,7 @@ def requires_micro(*args): Function to mark """ _requires_micro = [ - pytest.mark.skipif( - tvm.support.libinfo().get("USE_MICRO", "OFF") != "ON", - reason="MicroTVM support not enabled. Set USE_MICRO=ON in config.cmake to enable.", - ) + _cmake_flag_skipif("MicroTVM", "USE_MICRO"), ] return _compose(args, _requires_micro) @@ -893,10 +947,7 @@ def requires_rpc(*args): Function to mark """ _requires_rpc = [ - pytest.mark.skipif( - tvm.support.libinfo().get("USE_RPC", "OFF") != "ON", - reason="RPC support not enabled. Set USE_RPC=ON in config.cmake to enable.", - ) + _cmake_flag_skipif("RPC", "USE_RPC"), ] return _compose(args, _requires_rpc) @@ -911,14 +962,8 @@ def requires_ethosn(*args): """ marks = [ pytest.mark.ethosn, - pytest.mark.skipif( - not ethosn_available(), - reason=( - "Arm(R) Ethos(TM)-N support not enabled. " - "Set USE_ETHOSN=ON in config.cmake to enable, " - "and ensure that hardware support is present." - ), - ), + _cmake_flag_skipif("Arm(R) Ethos(TM)-N", "USE_ETHOSN"), + _cmake_flag_skipif("Arm(R) Ethos(TM)-N Hardware", "USE_ETHOSN_HW"), ] return _compose(args, marks) @@ -933,10 +978,11 @@ def requires_hexagon(*args): """ _requires_hexagon = [ pytest.mark.hexagon, - pytest.mark.skipif(not device_enabled("hexagon"), reason="Hexagon support not enabled"), + *_target_skipif_marks("Hexagon", "USE_HEXAGON", "hexagon"), *requires_llvm(), pytest.mark.skipif( - tvm.target.codegen.llvm_version_major() < 7, reason="Hexagon requires LLVM 7 or later" + _cmake_flag_enabled("USE_LLVM") and tvm.target.codegen.llvm_version_major() < 7, + reason="Hexagon requires LLVM 7 or later", ), ] return _compose(args, _requires_hexagon) @@ -951,7 +997,9 @@ def requires_cmsisnn(*args): Function to mark """ - requirements = [pytest.mark.skipif(not cmsisnn.enabled(), reason="CMSIS NN not enabled")] + requirements = [ + _cmake_flag_skipif("CMSIS NN", "USE_CMSISNN"), + ] return _compose(args, requirements) @@ -964,7 +1012,9 @@ def requires_vitis_ai(*args): Function to mark """ - requirements = [pytest.mark.skipif(not vitis_ai.enabled(), reason="Vitis AI not enabled")] + requirements = [ + _cmake_flag_skipif("Vitis AI", "USE_VITIS_AI"), + ] return _compose(args, requirements) diff --git a/tests/python/driver/tvmc/test_compiler.py b/tests/python/driver/tvmc/test_compiler.py index bfbf9922e0a3..b7152423a107 100644 --- a/tests/python/driver/tvmc/test_compiler.py +++ b/tests/python/driver/tvmc/test_compiler.py @@ -412,10 +412,7 @@ def test_compile_tflite_module_with_external_codegen_cmsisnn( assert len(c_source_files) == 4 -@pytest.mark.skipif( - not ethosn_available(), - reason="--target=Ethos(TM)-N78 is not available. TVM built with 'USE_ETHOSN OFF'", -) +@tvm.testing.requires_ethosn def test_compile_tflite_module_with_external_codegen_ethos_n78(tflite_mobilenet_v1_1_quant): pytest.importorskip("tflite") tvmc_model = tvmc.load(tflite_mobilenet_v1_1_quant) @@ -430,10 +427,7 @@ def test_compile_tflite_module_with_external_codegen_ethos_n78(tflite_mobilenet_ assert os.path.exists(dumps_path) -@pytest.mark.skipif( - not vitis_ai_available(), - reason="--target=vitis-ai is not available. TVM built with 'USE_VITIS_AI OFF'", -) +@tvm.testing.requires_vitis_ai def test_compile_tflite_module_with_external_codegen_vitis_ai(tflite_mobilenet_v1_1_quant): pytest.importorskip("tflite") From 9659562d6eaff8662b5d089749e23900ea688125 Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Fri, 13 May 2022 12:47:01 -0500 Subject: [PATCH 2/8] Refactor to specify repeated feature marks, compile-only markers --- python/tvm/testing/plugin.py | 25 +- python/tvm/testing/utils.py | 827 ++++++++++++++++++----------------- 2 files changed, 445 insertions(+), 407 deletions(-) diff --git a/python/tvm/testing/plugin.py b/python/tvm/testing/plugin.py index e90bd5e6dbf5..1f4f983b7210 100644 --- a/python/tvm/testing/plugin.py +++ b/python/tvm/testing/plugin.py @@ -56,8 +56,8 @@ def pytest_configure(config): """Runs at pytest configure time, defines marks to be used later.""" - for markername, desc in MARKERS.items(): - config.addinivalue_line("markers", "{}: {}".format(markername, desc)) + for feature in utils.Feature._all_features.values(): + feature._register_marker(config) print("enabled targets:", "; ".join(map(lambda x: x[0], utils.enabled_targets()))) print("pytest marker:", config.option.markexpr) @@ -269,25 +269,26 @@ def _target_to_requirement(target): # mapping from target to decorator if target.kind.name == "cuda" and "cudnn" in target.attrs.get("libs", []): - return utils.requires_cudnn() + return utils.requires_cudnn.marks() if target.kind.name == "cuda" and "cublas" in target.attrs.get("libs", []): - return utils.requires_cublas() + return utils.requires_cublas.marks() if target.kind.name == "cuda": - return utils.requires_cuda() + return utils.requires_cuda.marks() if target.kind.name == "rocm": - return utils.requires_rocm() + return utils.requires_rocm.marks() if target.kind.name == "vulkan": - return utils.requires_vulkan() + return utils.requires_vulkan.marks() if target.kind.name == "nvptx": - return utils.requires_nvptx() + return utils.requires_nvptx.marks() if target.kind.name == "metal": - return utils.requires_metal() + return utils.requires_metal.marks() if target.kind.name == "opencl": - return utils.requires_opencl() + return utils.requires_opencl.marks() if target.kind.name == "llvm": - return utils.requires_llvm() + return utils.requires_llvm.marks() if target.kind.name == "hexagon": - return utils.requires_hexagon() + return utils.requires_hexagon.marks() + return [] diff --git a/python/tvm/testing/utils.py b/python/tvm/testing/utils.py index b699c2a361ef..9a31a23395a9 100644 --- a/python/tvm/testing/utils.py +++ b/python/tvm/testing/utils.py @@ -62,19 +62,25 @@ def test_something(): `TVM_TEST_TARGETS` environment variable in the CI. """ +import collections import copy import copyreg import ctypes import functools +import itertools import logging import os +import pickle import platform import shutil import sys import time -import pickle + +from typing import Optional, Callable, Union, List + import pytest import numpy as np + import tvm import tvm.arith import tvm.tir @@ -387,12 +393,9 @@ def _check_forward(constraints1, constraints2, varmap, backvarmap): ) -def _get_targets(target_str=None): - if target_str is None: - target_str = os.environ.get("TVM_TEST_TARGETS", "") - # Use dict instead of set for de-duplication so that the - # targets stay in the order specified. - target_names = list({t.strip(): None for t in target_str.split(";") if t.strip()}) +def _get_targets(target_names=None): + if target_names is None: + target_names = _tvm_test_targets() if not target_names: target_names = DEFAULT_TEST_TARGETS @@ -428,7 +431,7 @@ def _get_targets(target_str=None): " Try setting TVM_TEST_TARGETS to a supported target. Defaulting to llvm.", target_str, ) - return _get_targets("llvm") + return _get_targets(["llvm"]) raise TVMError( "None of the following targets are supported by this build of TVM: %s." @@ -514,508 +517,542 @@ def enabled_targets(): return [(t["target"], tvm.device(t["target"])) for t in _get_targets() if t["is_runnable"]] -def _cmake_flag_enabled(flag): - flag = tvm.support.libinfo()[flag] +class Feature: - # Because many of the flags can be library flags, we check if the - # flag is not disabled, rather than checking if it is enabled. - return flag.lower() not in ["off", "false", "0"] + """A feature that may be required to run a test. + Parameters + ---------- + name: str -def _cmake_flag_skipif(feature_name, cmake_flag): - return pytest.mark.skipif( - not _cmake_flag_enabled(cmake_flag), - reason=f"{feature_name} support not enabled. Set {cmake_flag}=ON in config.cmake to enable.", - ) + The short name of the feature. Should match the name in the + requires_* decorator. This is applied as a mark to all tests + using this feature, and can be used in pytests ``-m`` + argument. + long_name: Optional[str] -def _environment_var_enabled(target): - enabled_str = os.environ.get("TVM_TEST_TARGETS", "") - if enabled_str: - enabled_target_names = set(t.strip() for t in enabled_str.split(";")) - else: - enabled_target_names = DEFAULT_TEST_TARGETS + The long name of the feature, to be used in error messages. - target_kind = target.split()[0] - return any(enabled.split()[0] == target_kind for enabled in enabled_target_names) + If None, defaults to the short name. + cmake_flag: Optional[str] -def _environment_var_skipif(target_name, target=None): - if target is None: - target = target_name + The flag that must be enabled in the config.cmake in order to + use this feature. - return pytest.mark.skipif( - not _environment_var_enabled(target), - reason=f"{target_name} tests disabled by TVM_TEST_TARGETS", - ) + If None, no flag is required to use this feature. + target_kind_enabled: Optional[str] -def _device_exists(target): - target_kind = target.split()[0] - return tvm.runtime.enabled(target_kind) and tvm.device(target_kind).exist + The target kind that must be enabled to run tests using this + feature. If present, the target_kind must appear in the + TVM_TEST_TARGETS environment variable, or in + tvm.testing.DEFAULT_TEST_TARGETS if TVM_TEST_TARGETS is + undefined. + If None, this feature does not require a specific target to be + enabled. -def _device_skipif(target_name, target=None): - if target is None: - target = target_name + compile_time_check: Optional[Callable[[], Union[bool,str]]] - return pytest.mark.skipif( - not _device_exists(target_kind), reason=f"{target_name} device not present" - ) + A check that returns True if the feature can be used at + compile-time. (e.g. Validating the version number of the nvcc + compiler.) If the feature does not have support to perform + compile-time tests, the check should returns False to display + a generic error message, or a string to display a more + specific error message. + If None, no additional check is performed. -def _target_skipif_marks(target_name, cmake_flag, target_kind): - return [ - _cmake_flag_skipif(target_name, cmake_flag), - _environment_var_skipif(target_name, target_kind), - _device_skipif(target_name, target_kind), - ] + target_kind_hardware: Optional[str] + The target kind that must have available hardware in order to + run tests using this feature. This is checked using + tvm.device(target_kind_hardware).exist. If a feature requires + a different check, this should be implemented using + run_time_check. -def _compose(args, decs): - """Helper to apply multiple markers""" - if len(args) > 0: - f = args[0] - for d in reversed(decs): - f = d(f) - return f - return decs + If None, this feature does not require a specific + tvm.device to exist. + run_time_check: Optional[Callable[[], Union[bool,str]]] -def slow(fn): - @functools.wraps(fn) - def wrapper(*args, **kwargs): - if SKIP_SLOW_TESTS: - pytest.skip("Skipping slow test since RUN_SLOW_TESTS environment variables is 'true'") - else: - fn(*args, **kwargs) - - return wrapper - + A check that returns True if the feature can be used at + run-time. (e.g. Validating the compute version supported by a + GPU.) If the feature does not have support to perform + run-time tests, the check should returns False to display a + generic error message, or a string to display a more specific + error message. -def uses_gpu(*args): - """Mark to differentiate tests that use the GPU in some capacity. - - These tests will be run on CPU-only test nodes and on test nodes with GPUs. - To mark a test that must have a GPU present to run, use - :py:func:`tvm.testing.requires_gpu`. - - Parameters - ---------- - f : function - Function to mark - """ - _uses_gpu = [pytest.mark.gpu] - return _compose(args, _uses_gpu) + If None, no additional check is performed. + parent_features: Optional[Union[str,List[str]]] -def requires_x86(*args): - """Mark a test as requiring the x86 Architecture to run. + The short name of a feature or features that are required in + order to use this feature. (e.g. Using cuDNN requires using + CUDA) This feature should inherit all checks of the parent + feature, with the exception of the `target_kind_enabled` + checks. - Tests with this mark will not be run unless on an x86 platform. + If None, this feature does not require any other parent + features. - Parameters - ---------- - f : function - Function to mark """ - _requires_x86 = [ - pytest.mark.skipif(platform.machine() != "x86_64", reason="x86 Architecture Required"), - ] - return _compose(args, _requires_x86) - - -def requires_gpu(*args): - """Mark a test as requiring a GPU to run. - Tests with this mark will not be run unless a gpu is present. - - Parameters - ---------- - f : function - Function to mark - """ - _requires_gpu = [ - pytest.mark.skipif( - not tvm.cuda().exist - and not tvm.rocm().exist - and not tvm.opencl().exist - and not tvm.metal().exist - and not tvm.vulkan().exist, - reason="No GPU present", - ), - *uses_gpu(), - ] - return _compose(args, _requires_gpu) + _all_features = {} + + def __init__( + self, + name: str, + long_name: Optional[str] = None, + cmake_flag: Optional[str] = None, + target_kind_enabled: Optional[str] = None, + compile_time_check: Optional[Callable[[], Union[bool, str]]] = None, + target_kind_hardware: Optional[str] = None, + run_time_check: Optional[Callable[[], Union[bool, str]]] = None, + parent_features: Optional[Union[str, List[str]]] = None, + ): + self.name = name + self.long_name = long_name or name + self.cmake_flag = cmake_flag + self.target_kind_enabled = target_kind_enabled + self.compile_time_check = compile_time_check + self.target_kind_hardware = target_kind_hardware + self.run_time_check = run_time_check + + if parent_features is None: + self.parent_features = [] + elif isinstance(parent_features, str): + self.parent_features = [parent_features] + else: + self.parent_features = parent_features + self._all_features[self.name] = self -def requires_cuda(*args): - """Mark a test as requiring the CUDA runtime. + def _register_marker(self, config): + config.addinivalue_line("markers", f"{self.name}: Mark a test as using {self.long_name}") - This also marks the test as requiring a cuda gpu. + def _uses_marks(self): + for parent in self.parent_features: + yield from self._all_features[parent]._uses_marks() - Parameters - ---------- - f : function - Function to mark - """ - libinfo = tvm.support.libinfo() + yield getattr(pytest.mark, self.name) - marks = [ - pytest.mark.cuda, - *_target_skipif_marks("CUDA", "USE_CUDA", "cuda"), - *requires_gpu(), - ] - return _compose(args, marks) + def _compile_only_marks(self): + for parent in self.parent_features: + yield from self._all_features[parent]._compile_only_marks() + if self.compile_time_check is not None: + res = self.compile_time_check() + if isinstance(res, str): + yield pytest.mark.skipif(True, reason=res) + else: + yield pytest.mark.skipif( + res, reason=f"Compile-time support for {self.long_name} not present" + ) -def requires_cudnn(*args): - """Mark a test as requiring the cuDNN library. + if self.target_kind_enabled is not None: + target_kind = self.target_kind_enabled.split()[0] + yield pytest.mark.skipif( + all(enabled.split()[0] != target_kind for enabled in _tvm_test_targets()), + reason=( + f"{self.target_kind_enabled} tests disabled by TVM_TEST_TARGETS environment variable" + ), + ) - This also marks the test as requiring a cuda gpu. + if self.cmake_flag is not None: + yield pytest.mark.skipif( + not _cmake_flag_enabled(self.cmake_flag), + reason=( + f"{self.long_name} support not enabled. " + f"Set {self.cmake_flag} in config.cmake to enable." + ), + ) - Parameters - ---------- - f : function - Function to mark - """ + def _run_only_marks(self): + for parent in self.parent_features: + yield from self._all_features[parent]._run_only_marks() + + if self.run_time_check is not None: + res = self.run_time_check() + if isinstance(res, str): + yield pytest.mark.skipif(True, reason=res) + else: + yield pytest.mark.skipif( + res, reason=f"Run-time support for {self.long_name} not present" + ) - requirements = [ - _cmake_flag_skipif("cuDNN", "USE_CUDNN"), - pytest.mark.skipif(not cudnn.exists(), reason="cuDNN library not installed"), - *requires_cuda(), - ] - return _compose(args, requirements) + if self.target_kind_hardware is not None: + yield pytest.mark.skipif( + not tvm.device(self.target_kind_hardware).exist, + reason=f"No device exists for target {self.target_kind_hardware}", + ) + def marks(self, support_required="compile-and-run"): + """Return a list of marks to be used -def requires_cublas(*args): - """Mark a test as requiring the cuBLAS library. + Parameters + ---------- - This also marks the test as requiring a cuda gpu. + support_required: str - Parameters - ---------- - f : function - Function to mark - """ + Allowed values: "compile-and-run" (default), + "compile-only", or "optional". - requirements = [ - _cmake_flag_skipif("cuBLAS", "USE_CUBLAS"), - *requires_cuda(), - ] - return _compose(args, requirements) + See Feature.__call__ for details. + """ + if support_required not in ["compile-and-run", "compile-only", "optional"]: + raise ValueError(f"Unknown feature support type: {support_required}") + if support_required == "compile-and-run": + marks = itertools.chain( + self._run_only_marks(), self._compile_only_marks(), self._uses_marks() + ) + elif support_required == "compile-only": + marks = itertools.chain(self._compile_only_marks(), self._uses_marks()) + elif support_required == "optional": + marks = self._uses_marks() + else: + raise ValueError(f"Unknown feature support type: {support_required}") -def requires_nvptx(*args): - """Mark a test as requiring the NVPTX compilation on the CUDA runtime + return list(marks) - This also marks the test as requiring a cuda gpu, and requiring - LLVM support. + def __call__(self, func=None, *, support_required="compile-and-run"): + """Mark a pytest function as requiring this feature - Parameters - ---------- - f : function - Function to mark + Can be used either as a bare decorator, or as a decorator with + arguments. - """ - _requires_nvptx = [ - _device_skipif("nvptx"), - *requires_llvm(), - *requires_gpu(), - ] - return _compose(args, _requires_nvptx) + Parameters + ---------- + func: Callable -def requires_nvcc_version(major_version, minor_version=0, release_version=0): - """Mark a test as requiring at least a specific version of nvcc. + The pytest test function to be marked - Unit test marked with this decorator will run only if the - installed version of NVCC is at least `(major_version, - minor_version, release_version)`. + support_required: str - This also marks the test as requiring a cuda support. + Allowed values: "compile-and-run" (default), + "compile-only", or "optional". - Parameters - ---------- - major_version: int + If "compile-and-run", the test case is marked as using the + feature, and is skipped if the environment lacks either + compile-time or run-time support for the feature. - The major version of the (major,minor,release) version tuple. + If "compile-only", the test case is marked as using the + feature, and is skipped if the environment lacks + compile-time support. - minor_version: int + If "optional", the test case is marked as using the + feature, but isn't skipped. This is kept for backwards + compatibility for tests that use `enabled_targets()`, and + should be avoided in new test code. Instead, prefer + parametrizing over the target using the `target` fixture. - The minor version of the (major,minor,release) version tuple. + Examples + -------- - release_version: int + .. code-block:: python - The release version of the (major,minor,release) version tuple. + @feature + def test_compile_and_run(): + ... - """ + @feature(compile_only=True) + def test_compile_only(): + ... - try: - nvcc_version = nvcc.get_cuda_version() - except RuntimeError: - nvcc_version = (0, 0, 0) + """ - min_version = (major_version, minor_version, release_version) - version_str = ".".join(str(v) for v in min_version) - requires = [ - pytest.mark.skipif(nvcc_version < min_version, reason=f"Requires NVCC >= {version_str}"), - *requires_cuda(), - ] + if support_required not in ["compile-and-run", "compile-only", "optional"]: + raise ValueError(f"Unknown feature support type: {support_required}") - def inner(func): - return _compose([func], requires) + def wrapper(func): + for mark in self.marks(support_required=support_required): + func = mark(func) + return func - return inner + if func is None: + return wrapper + return wrapper(func) -def skip_if_32bit(reason): - def decorator(*args): - if "32bit" in platform.architecture()[0]: - return _compose(args, [pytest.mark.skip(reason=reason)]) + @classmethod + def require(cls, name, support_required="compile-and-run"): + """Returns a decorator that marks a test as requiring a feature - return _compose(args, []) + Parameters + ---------- - return decorator + name: str + The name of the feature that is used by the test -def requires_cudagraph(*args): - """Mark a test as requiring the CUDA Graph Feature + support_required: str - This also marks the test as requiring cuda + Allowed values: "compile-and-run" (default), + "compile-only", or "optional". - Parameters - ---------- - f : function - Function to mark - """ - _requires_cudagraph = [ - pytest.mark.skipif( - not nvcc.have_cudagraph(), reason="CUDA Graph is not supported in this environment" - ), - *requires_cuda(), - ] - return _compose(args, _requires_cudagraph) + See Feature.__call__ for details. + Examples + -------- -def requires_opencl(*args): - """Mark a test as requiring the OpenCL runtime. + .. code-block:: python - This also marks the test as requiring a gpu. - - Parameters - ---------- - f : function - Function to mark - """ - _requires_opencl = [ - pytest.mark.opencl, - *_target_skipif_marks("OpenCL", "USE_OPENCL", "opencl"), - *requires_gpu(), - ] - return _compose(args, _requires_opencl) + @Feature.require("cuda") + def test_compile_and_run(): + ... + @Feature.require("cuda", compile_only=True) + def test_compile_only(): + ... + """ + return cls._all_features[name](support_required=support_required) -def requires_corstone300(*args): - """Mark a test as requiring the corstone300 FVP - - Parameters - ---------- - f : function - Function to mark - """ - _requires_corstone300 = [ - pytest.mark.corstone300, - pytest.mark.skipif( - shutil.which("arm-none-eabi-gcc") is None, reason="ARM embedded toolchain unavailable" - ), - ] - return _compose(args, _requires_corstone300) +def _any_gpu_exists(): + return ( + tvm.cuda().exist + or tvm.rocm().exist + or tvm.opencl().exist + or tvm.metal().exist + or tvm.vulkan().exist + ) -def requires_rocm(*args): - """Mark a test as requiring the rocm runtime. - This also marks the test as requiring a gpu. +# Mark a test as requiring llvm to run +requires_llvm = Feature( + "llvm", "LLVM", cmake_flag="USE_LLVM", target_kind_enabled="llvm", target_kind_hardware="llvm" +) - Parameters - ---------- - f : function - Function to mark - """ - _requires_rocm = [ - pytest.mark.rocm, - *_target_skipif_marks("ROCm", "USE_ROCM", "rocm"), - *requires_gpu(), - ] - return _compose(args, _requires_rocm) +# Mark a test as requiring a GPU to run. +requires_gpu = Feature("gpu", run_time_check=_any_gpu_exists) +# Mark to differentiate tests that use the GPU in some capacity. +# +# These tests will be run on CPU-only test nodes and on test nodes with GPUs. +# To mark a test that must have a GPU present to run, use +# :py:func:`tvm.testing.requires_gpu`. +uses_gpu = requires_gpu(support_required="optional") + +# Mark a test as requiring the x86 Architecture to run. +requires_x86 = Feature( + "x86", "x86 Architecture", run_time_check=lambda: platform.machine() == "x86_64" +) + +# Mark a test as requiring the CUDA runtime. +requires_cuda = Feature( + "cuda", + "CUDA", + cmake_flag="USE_CUDA", + target_kind_enabled="cuda", + target_kind_hardware="cuda", + parent_features="gpu", +) + +# Mark a test as requiring a tensorcore to run +requires_tensorcore = Feature( + "tensorcore", + "NVIDIA Tensor Core", + run_time_check=lambda: tvm.cuda().exist and nvcc.have_tensorcore(tvm.cuda().compute_version), + parent_features="cuda", +) + +# Mark a test as requiring the cuDNN library. +requires_cudnn = Feature("cudnn", "cuDNN", cmake_flag="USE_CUDNN", parent_features="cuda") + +# Mark a test as requiring the cuBLAS library. +requires_cublas = Feature("cublas", "cuBLAS", cmake_flag="USE_CUBLAS", parent_features="cuda") + +# Mark a test as requiring the NVPTX compilation on the CUDA runtime +requires_nvptx = Feature( + "nvptx", + "NVPTX", + target_kind_enabled="nvptx", + target_kind_hardware="nvptx", + parent_features=["llvm", "cuda"], +) + +# Mark a test as requiring the CUDA Graph Feature +requires_cudagraph = Feature( + "cudagraph", + "CUDA Graph", + target_kind_enabled="cuda", + compile_time_check=nvcc.have_cudagraph, + parent_features="cuda", +) + +# Mark a test as requiring the OpenCL runtime +requires_opencl = Feature( + "opencl", + "OpenCL", + cmake_flag="USE_OPENCL", + target_kind_enabled="opencl", + target_kind_hardware="opencl", + parent_features="gpu", +) + +# Mark a test as requiring the corstone300 FVP +requires_corstone300 = Feature( + "corstone300", + "Corstone-300", + compile_time_check=lambda: ( + (shutil.which("arm-none-eabi-gcc") is None) or "ARM embedded toolchain unavailable" + ), +) + +# Mark a test as requiring the rocm runtime +requires_rocm = Feature( + "rocm", + "ROCm", + cmake_flag="USE_ROCM", + target_kind_enabled="rocm", + target_kind_hardware="rocm", + parent_features="gpu", +) + +# Mark a test as requiring the metal runtime +requires_metal = Feature( + "metal", + "Metal", + cmake_flag="USE_METAL", + target_kind_enabled="metal", + target_kind_hardware="metal", + parent_features="gpu", +) + +# Mark a test as requiring the vulkan runtime +requires_vulkan = Feature( + "vulkan", + "Vulkan", + cmake_flag="USE_VULKAN", + target_kind_enabled="vulkan", + target_kind_hardware="vulkan", + parent_features="gpu", +) + +# Mark a test as requiring microTVM to run +requires_micro = Feature("micro", "MicroTVM", cmake_flag="USE_MICRO") + +# Mark a test as requiring rpc to run +requires_rpc = Feature("rpc", "RPC", cmake_flag="USE_RPC") + +# Mark a test as requiring Arm(R) Ethos(TM)-N to run +requires_ethosn = Feature("ethosn", "Arm(R) Ethos(TM)-N", cmake_flag="USE_ETHOSN") + +# Mark a test as requiring Hexagon to run +requires_hexagon = Feature( + "hexagon", + "Hexagon", + cmake_flag="USE_HEXAGON", + target_kind_enabled="hexagon", + compile_time_check=lambda: ( + (_cmake_flag_enabled("USE_LLVM") and tvm.target.codegen.llvm_version_major() >= 7) + or "Hexagon requires LLVM 7 or later" + ), + target_kind_hardware="hexagon", + parent_features="llvm", +) -def requires_metal(*args): - """Mark a test as requiring the metal runtime. +# Mark a test as requiring the CMSIS NN library +requires_cmsisnn = Feature("cmsisnn", "CMSIS NN", cmake_flag="USE_CMSISNN") - This also marks the test as requiring a gpu. +# Mark a test as requiring Vitis AI to run +requires_vitis_ai = Feature("vitis_ai", "Vitis AI", cmake_flag="USE_VITIS_AI") - Parameters - ---------- - f : function - Function to mark - """ - _requires_metal = [ - pytest.mark.metal, - *_target_skipif_marks("metal", "USE_METAL", "metal"), - *requires_gpu(), - ] - return _compose(args, _requires_metal) +def _cmake_flag_enabled(flag): + flag = tvm.support.libinfo()[flag] -def requires_vulkan(*args): - """Mark a test as requiring the vulkan runtime. + # Because many of the flags can be library flags, we check if the + # flag is not disabled, rather than checking if it is enabled. + return flag.lower() not in ["off", "false", "0"] - This also marks the test as requiring a gpu. - Parameters - ---------- - f : function - Function to mark - """ - _requires_vulkan = [ - pytest.mark.vulkan, - *_target_skipif_marks("Vulkan", "USE_VULKAN", "vulkan"), - *requires_gpu(), - ] - return _compose(args, _requires_vulkan) +def _tvm_test_targets(): + target_str = os.environ.get("TVM_TEST_TARGETS", "").strip() + if target_str: + # Use dict instead of set for de-duplication so that the + # targets stay in the order specified. + return list({t.strip(): None for t in target_str.split(";") if t.strip()}) + else: + return DEFAULT_TEST_TARGETS -def requires_tensorcore(*args): - """Mark a test as requiring a tensorcore to run. +def _compose(args, decs): + """Helper to apply multiple markers""" + if len(args) > 0: + f = args[0] + for d in reversed(decs): + f = d(f) + return f + return decs - Tests with this mark will not be run unless a tensorcore is present. - Parameters - ---------- - f : function - Function to mark - """ - _requires_tensorcore = [ - pytest.mark.tensorcore, - *requires_cuda(), - pytest.mark.skipif( - not tvm.cuda().exist or not nvcc.have_tensorcore(tvm.cuda(0).compute_version), - reason="No tensorcore present", - ), - *requires_gpu(), - ] - return _compose(args, _requires_tensorcore) +def slow(fn): + @functools.wraps(fn) + def wrapper(*args, **kwargs): + if SKIP_SLOW_TESTS: + pytest.skip("Skipping slow test since RUN_SLOW_TESTS environment variables is 'true'") + else: + fn(*args, **kwargs) + return wrapper -def requires_llvm(*args): - """Mark a test as requiring llvm to run. - Parameters - ---------- - f : function - Function to mark - """ - _requires_llvm = [ - pytest.mark.llvm, - *_target_skipif_marks("LLVM", "USE_LLVM", "llvm"), - ] - return _compose(args, _requires_llvm) +def requires_nvcc_version(major_version, minor_version=0, release_version=0): + """Mark a test as requiring at least a specific version of nvcc. + Unit test marked with this decorator will run only if the + installed version of NVCC is at least `(major_version, + minor_version, release_version)`. -def requires_micro(*args): - """Mark a test as requiring microTVM to run. + This also marks the test as requiring a cuda support. Parameters ---------- - f : function - Function to mark - """ - _requires_micro = [ - _cmake_flag_skipif("MicroTVM", "USE_MICRO"), - ] - return _compose(args, _requires_micro) + major_version: int + The major version of the (major,minor,release) version tuple. -def requires_rpc(*args): - """Mark a test as requiring rpc to run. + minor_version: int - Parameters - ---------- - f : function - Function to mark - """ - _requires_rpc = [ - _cmake_flag_skipif("RPC", "USE_RPC"), - ] - return _compose(args, _requires_rpc) + The minor version of the (major,minor,release) version tuple. + release_version: int -def requires_ethosn(*args): - """Mark a test as requiring Arm(R) Ethos(TM)-N to run. + The release version of the (major,minor,release) version tuple. - Parameters - ---------- - f : function - Function to mark """ - marks = [ - pytest.mark.ethosn, - _cmake_flag_skipif("Arm(R) Ethos(TM)-N", "USE_ETHOSN"), - _cmake_flag_skipif("Arm(R) Ethos(TM)-N Hardware", "USE_ETHOSN_HW"), - ] - return _compose(args, marks) - -def requires_hexagon(*args): - """Mark a test as requiring Hexagon to run. + try: + nvcc_version = nvcc.get_cuda_version() + except RuntimeError: + nvcc_version = (0, 0, 0) - Parameters - ---------- - f : function - Function to mark - """ - _requires_hexagon = [ - pytest.mark.hexagon, - *_target_skipif_marks("Hexagon", "USE_HEXAGON", "hexagon"), - *requires_llvm(), - pytest.mark.skipif( - _cmake_flag_enabled("USE_LLVM") and tvm.target.codegen.llvm_version_major() < 7, - reason="Hexagon requires LLVM 7 or later", - ), + min_version = (major_version, minor_version, release_version) + version_str = ".".join(str(v) for v in min_version) + requires = [ + pytest.mark.skipif(nvcc_version < min_version, reason=f"Requires NVCC >= {version_str}"), + *requires_cuda(), ] - return _compose(args, _requires_hexagon) - -def requires_cmsisnn(*args): - """Mark a test as requiring the CMSIS NN library. - - Parameters - ---------- - f : function - Function to mark - """ + def inner(func): + return _compose([func], requires) - requirements = [ - _cmake_flag_skipif("CMSIS NN", "USE_CMSISNN"), - ] - return _compose(args, requirements) + return inner -def requires_vitis_ai(*args): - """Mark a test as requiring Vitis AI to run. +def skip_if_32bit(reason): + def decorator(*args): + if "32bit" in platform.architecture()[0]: + return _compose(args, [pytest.mark.skip(reason=reason)]) - Parameters - ---------- - f : function - Function to mark - """ + return _compose(args, []) - requirements = [ - _cmake_flag_skipif("Vitis AI", "USE_VITIS_AI"), - ] - return _compose(args, requirements) + return decorator def requires_package(*packages): From 2cd06543f7512cf584306bfdcbd1f66c7669158c Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Mon, 16 May 2022 08:30:40 -0500 Subject: [PATCH 3/8] Fixed lint errors --- python/tvm/testing/utils.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/python/tvm/testing/utils.py b/python/tvm/testing/utils.py index 9a31a23395a9..3c25f699724a 100644 --- a/python/tvm/testing/utils.py +++ b/python/tvm/testing/utils.py @@ -62,7 +62,6 @@ def test_something(): `TVM_TEST_TARGETS` environment variable in the CI. """ -import collections import copy import copyreg import ctypes @@ -89,9 +88,6 @@ def test_something(): from tvm.contrib import nvcc, cudnn from tvm.error import TVMError -from tvm.relay.op.contrib import ethosn -from tvm.relay.op.contrib import cmsisnn -from tvm.relay.op.contrib import vitis_ai SKIP_SLOW_TESTS = os.getenv("SKIP_SLOW_TESTS", "").lower() in {"true", "1", "yes"} @@ -657,7 +653,8 @@ def _compile_only_marks(self): yield pytest.mark.skipif( all(enabled.split()[0] != target_kind for enabled in _tvm_test_targets()), reason=( - f"{self.target_kind_enabled} tests disabled by TVM_TEST_TARGETS environment variable" + f"{self.target_kind_enabled} tests disabled " + f"by TVM_TEST_TARGETS environment variable" ), ) @@ -977,8 +974,8 @@ def _tvm_test_targets(): # Use dict instead of set for de-duplication so that the # targets stay in the order specified. return list({t.strip(): None for t in target_str.split(";") if t.strip()}) - else: - return DEFAULT_TEST_TARGETS + + return DEFAULT_TEST_TARGETS def _compose(args, decs): From fff040494a1991a47334bbbca82a7cbcd524022e Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Tue, 17 May 2022 15:21:07 -0500 Subject: [PATCH 4/8] Import from contrib, not from a different import --- tests/python/driver/tvmc/test_compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/driver/tvmc/test_compiler.py b/tests/python/driver/tvmc/test_compiler.py index b7152423a107..e557d310eb75 100644 --- a/tests/python/driver/tvmc/test_compiler.py +++ b/tests/python/driver/tvmc/test_compiler.py @@ -25,7 +25,7 @@ import tvm import tvm.testing -from tvm.testing.utils import ethosn_available +from tvm.relay.op.contrib.ethosn import ethosn_available from tvm.relay.backend import Runtime, Executor from tvm.contrib.target.vitis_ai import vitis_ai_available From ced22eed406316233e996da7f84acf40c7bafabc Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Tue, 17 May 2022 15:21:28 -0500 Subject: [PATCH 5/8] Removed use of requires_llvm() as a list of marks --- python/tvm/testing/utils.py | 2 +- tests/python/contrib/test_dnnl.py | 4 ++-- tests/python/contrib/test_tensorrt.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/tvm/testing/utils.py b/python/tvm/testing/utils.py index 3c25f699724a..d261d142e653 100644 --- a/python/tvm/testing/utils.py +++ b/python/tvm/testing/utils.py @@ -1033,7 +1033,7 @@ def requires_nvcc_version(major_version, minor_version=0, release_version=0): version_str = ".".join(str(v) for v in min_version) requires = [ pytest.mark.skipif(nvcc_version < min_version, reason=f"Requires NVCC >= {version_str}"), - *requires_cuda(), + *requires_cuda.marks(), ] def inner(func): diff --git a/tests/python/contrib/test_dnnl.py b/tests/python/contrib/test_dnnl.py index df3f64946f75..43debeaa56aa 100755 --- a/tests/python/contrib/test_dnnl.py +++ b/tests/python/contrib/test_dnnl.py @@ -32,8 +32,8 @@ ) run_module = tvm.testing.parameter( - pytest.param(False, marks=[has_dnnl_codegen, *tvm.testing.requires_llvm()]), - pytest.param(True, marks=[has_dnnl_codegen, *tvm.testing.requires_llvm()]), + pytest.param(False, marks=[has_dnnl_codegen, *tvm.testing.requires_llvm.marks()]), + pytest.param(True, marks=[has_dnnl_codegen, *tvm.testing.requires_llvm.marks()]), ids=["compile", "run"], ) diff --git a/tests/python/contrib/test_tensorrt.py b/tests/python/contrib/test_tensorrt.py index 4e6aab14c06c..e3a97102e481 100644 --- a/tests/python/contrib/test_tensorrt.py +++ b/tests/python/contrib/test_tensorrt.py @@ -44,9 +44,9 @@ ) run_module = tvm.testing.parameter( - pytest.param(False, marks=[has_tensorrt_codegen, *tvm.testing.requires_cuda()]), + pytest.param(False, marks=[has_tensorrt_codegen, *tvm.testing.requires_cuda.marks()]), pytest.param( - True, marks=[has_tensorrt_runtime, has_tensorrt_codegen, *tvm.testing.requires_cuda()] + True, marks=[has_tensorrt_runtime, has_tensorrt_codegen, *tvm.testing.requires_cuda.marks()] ), ids=["compile", "run"], ) From d8090faddb67d2fda054bd5e9c6c2c0420775ff2 Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Tue, 17 May 2022 15:22:24 -0500 Subject: [PATCH 6/8] Corrected mark from requires_gpu to requires_cuda --- tests/python/integration/test_reduce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/integration/test_reduce.py b/tests/python/integration/test_reduce.py index a40164ded941..f3886374ccb6 100644 --- a/tests/python/integration/test_reduce.py +++ b/tests/python/integration/test_reduce.py @@ -528,7 +528,7 @@ def check_target(device): check_target("rocm") -@tvm.testing.requires_gpu +@tvm.testing.requires_cuda def test_reduce_storage_reuse(): target = tvm.target.Target("cuda") From 7bd82115f244e9964ec1bf104044441907395573 Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Wed, 18 May 2022 10:50:11 -0500 Subject: [PATCH 7/8] Adding missing "not" --- python/tvm/testing/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/tvm/testing/utils.py b/python/tvm/testing/utils.py index d261d142e653..0be3bb2f5e67 100644 --- a/python/tvm/testing/utils.py +++ b/python/tvm/testing/utils.py @@ -645,7 +645,7 @@ def _compile_only_marks(self): yield pytest.mark.skipif(True, reason=res) else: yield pytest.mark.skipif( - res, reason=f"Compile-time support for {self.long_name} not present" + not res, reason=f"Compile-time support for {self.long_name} not present" ) if self.target_kind_enabled is not None: @@ -677,7 +677,7 @@ def _run_only_marks(self): yield pytest.mark.skipif(True, reason=res) else: yield pytest.mark.skipif( - res, reason=f"Run-time support for {self.long_name} not present" + not res, reason=f"Run-time support for {self.long_name} not present" ) if self.target_kind_hardware is not None: From 3dd37a405d0562150a02709f4a9bff720c8a05c5 Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Wed, 18 May 2022 16:08:47 -0500 Subject: [PATCH 8/8] Added USE_CMSISNN as a requirement for corstone300. --- python/tvm/testing/utils.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/python/tvm/testing/utils.py b/python/tvm/testing/utils.py index 0be3bb2f5e67..b04e16eb408f 100644 --- a/python/tvm/testing/utils.py +++ b/python/tvm/testing/utils.py @@ -891,15 +891,6 @@ def _any_gpu_exists(): parent_features="gpu", ) -# Mark a test as requiring the corstone300 FVP -requires_corstone300 = Feature( - "corstone300", - "Corstone-300", - compile_time_check=lambda: ( - (shutil.which("arm-none-eabi-gcc") is None) or "ARM embedded toolchain unavailable" - ), -) - # Mark a test as requiring the rocm runtime requires_rocm = Feature( "rocm", @@ -956,6 +947,16 @@ def _any_gpu_exists(): # Mark a test as requiring the CMSIS NN library requires_cmsisnn = Feature("cmsisnn", "CMSIS NN", cmake_flag="USE_CMSISNN") +# Mark a test as requiring the corstone300 FVP +requires_corstone300 = Feature( + "corstone300", + "Corstone-300", + compile_time_check=lambda: ( + (shutil.which("arm-none-eabi-gcc") is None) or "ARM embedded toolchain unavailable" + ), + parent_features="cmsisnn", +) + # Mark a test as requiring Vitis AI to run requires_vitis_ai = Feature("vitis_ai", "Vitis AI", cmake_flag="USE_VITIS_AI")