From 215cf69bcb0956a8e92db1b889f978be902a7c50 Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Mon, 4 Oct 2021 09:14:12 -0700 Subject: [PATCH 1/2] [Pytest] Sort unit tests before running. By default, pytest will sort tests to maximize the re-use of fixtures. However, this assumes that all fixtures have an equal cost to generate, and no caches outside of those managed by pytest. A fixture for a `tvm.testing.parameter` is effectively free, while a fixture maintaining a cache of reference data `tvm.testing.utils._fixture_cache` be quite large. Since most of the TVM fixtures are specific to a python function, sort the test ordering by python function, so that tvm.testing.utils._fixture_cache can be cleared sooner rather than later. --- python/tvm/testing/plugin.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/python/tvm/testing/plugin.py b/python/tvm/testing/plugin.py index 0413c44208b0..2cb228c357e5 100644 --- a/python/tvm/testing/plugin.py +++ b/python/tvm/testing/plugin.py @@ -74,6 +74,7 @@ def pytest_collection_modifyitems(config, items): # pylint: disable=unused-argument _count_num_fixture_uses(items) _remove_global_fixture_definitions(items) + _sort_tests(items) @pytest.fixture @@ -236,6 +237,25 @@ def _remove_global_fixture_definitions(items): delattr(module, name) +def _sort_tests(items): + """Sort tests by file/function. + + By default, pytest will sort tests to maximize the re-use of + fixtures. However, this assumes that all fixtures have an equal + cost to generate, and no caches outside of those managed by + pytest. A tvm.testing.parameter is effectively free, while + reference data for testing may be quite large. Since most of the + TVM fixtures are specific to a python function, sort the test + ordering by python function, so that + tvm.testing.utils._fixture_cache can be cleared sooner rather than + later. + + Should be called from pytest_collection_modifyitems. + + """ + items.sort(key=lambda item: item.location) + + def _target_to_requirement(target): if isinstance(target, str): target = tvm.target.Target(target) From ebac8511929e2268f0b8f21599a0b6a53f33c807 Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Mon, 4 Oct 2021 15:44:30 -0500 Subject: [PATCH 2/2] Updated TestTargetAutoParametrization When sorting the tests, the order of parametrizations may change. Therefore, the tests checking for automatic target parametrization shouldn't depend on order. --- .../python/unittest/test_tvm_testing_features.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/python/unittest/test_tvm_testing_features.py b/tests/python/unittest/test_tvm_testing_features.py index cbcdc4356250..c00fc02c4331 100644 --- a/tests/python/unittest/test_tvm_testing_features.py +++ b/tests/python/unittest/test_tvm_testing_features.py @@ -46,8 +46,11 @@ def test_device_parametrization(self, dev): self.devices_used.append(dev) def test_all_targets_used(self): - assert self.targets_used == self.enabled_targets - assert self.devices_used == self.enabled_devices + assert sorted(self.targets_used) == sorted(self.enabled_targets) + + def test_all_devices_used(self): + sort_key = lambda dev: (dev.device_type, dev.device_id) + assert sorted(self.devices_used, key=sort_key) == sorted(self.enabled_devices, key=sort_key) targets_with_explicit_list = [] @@ -70,9 +73,9 @@ def test_exclude_target(self, target): self.targets_with_exclusion.append(target) def test_all_nonexcluded_targets_ran(self): - assert self.targets_with_exclusion == [ - target for target in self.enabled_targets if not target.startswith("llvm") - ] + assert sorted(self.targets_with_exclusion) == sorted( + [target for target in self.enabled_targets if not target.startswith("llvm")] + ) run_targets_with_known_failure = [] @@ -85,7 +88,7 @@ def test_known_failing_target(self, target): assert "llvm" not in target def test_all_targets_ran(self): - assert self.run_targets_with_known_failure == self.enabled_targets + assert sorted(self.run_targets_with_known_failure) == sorted(self.enabled_targets) @tvm.testing.known_failing_targets("llvm") @tvm.testing.parametrize_targets("llvm")