From 0b6e33aa19ce7d8040e2b40d64caaab5db174e40 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Wed, 27 Jul 2022 12:07:36 -0700 Subject: [PATCH 1/8] Refactor micro test fixtures --- python/tvm/micro/testing/evaluation.py | 12 +- python/tvm/micro/testing/pytest_plugin.py | 129 ++++++++++++++++++ tests/micro/arduino/conftest.py | 44 ++---- .../arduino/test_arduino_error_detection.py | 5 - .../micro/arduino/test_arduino_rpc_server.py | 5 - tests/micro/arduino/test_arduino_workflow.py | 22 +-- tests/micro/arduino/test_utils.py | 17 --- tests/micro/common/conftest.py | 31 +---- tests/micro/zephyr/conftest.py | 71 +--------- tests/micro/zephyr/test_zephyr.py | 38 +++--- tests/micro/zephyr/test_zephyr_aot_exec.py | 12 +- .../zephyr/test_zephyr_aot_exec_standalone.py | 16 ++- tests/micro/zephyr/test_zephyr_armv7m.py | 14 +- tests/scripts/task_python_microtvm.sh | 14 +- 14 files changed, 216 insertions(+), 214 deletions(-) create mode 100644 python/tvm/micro/testing/pytest_plugin.py diff --git a/python/tvm/micro/testing/evaluation.py b/python/tvm/micro/testing/evaluation.py index 5407fcbabf10..6e426944f1fd 100644 --- a/python/tvm/micro/testing/evaluation.py +++ b/python/tvm/micro/testing/evaluation.py @@ -28,6 +28,7 @@ import tempfile import tvm +from tvm.relay.op.contrib import cmsisnn def tune_model( @@ -94,6 +95,15 @@ def create_aot_session( executor = tvm.relay.backend.Executor("aot") crt_runtime = tvm.relay.backend.Runtime("crt", {"system-lib": True}) + if use_cmsis_nn: + with tvm.transform.PassContext( + config={ + "tir.disable_vectorize": True, + "relay.ext.cmsisnn.options": {"mcpu": target.mcpu}, + } + ): + mod = cmsisnn.partition_for_cmsisnn(mod, params, mcpu=target.mcpu) + with ExitStack() as stack: config = {"tir.disable_vectorize": True} if use_cmsis_nn: @@ -153,4 +163,4 @@ def evaluate_model_accuracy(session, aot_executor, input_data, true_labels, runs num_correct = sum(u == v for u, v in zip(true_labels, predicted_labels)) average_time = sum(aot_runtimes) / len(aot_runtimes) accuracy = num_correct / len(predicted_labels) - return average_time, accuracy + return average_time, accuracy, predicted_labels diff --git a/python/tvm/micro/testing/pytest_plugin.py b/python/tvm/micro/testing/pytest_plugin.py new file mode 100644 index 000000000000..54a83b7983f2 --- /dev/null +++ b/python/tvm/micro/testing/pytest_plugin.py @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: disable=invalid-name,redefined-outer-name +""" microTVM testing fixtures used to deduce testing argument + values from testing parameters """ + +import pathlib +import json +import os +import datetime +import pytest + +import tvm +from tvm.contrib.utils import tempdir + + +def zephyr_boards() -> dict: + """Returns a dict mapping board to target model""" + with open( + pathlib.Path(tvm.micro.get_microtvm_template_projects("zephyr")) / "boards.json" + ) as f: + board_properties = json.load(f) + + boards_model = {board: info["model"] for board, info in board_properties.items()} + return boards_model + + +def get_boards(platform: str) -> dict: + """Returns a dict mapping board to target model""" + with open( + pathlib.Path(tvm.micro.get_microtvm_template_projects(platform)) / "boards.json" + ) as f: + board_properties = json.load(f) + + boards_model = {board: info["model"] for board, info in board_properties.items()} + return boards_model + + +def pytest_addoption(parser): + """Adds more pytest arguments""" + parser.addoption( + "--board", + required=True, + choices=list(get_boards("zephyr").keys()) + list(get_boards("arduino").keys()), + help="microTVM boards for tests", + ) + parser.addoption( + "--test-build-only", + action="store_true", + help="Only run tests that don't require physical hardware.", + ) + parser.addoption( + "--tvm-debug", + action="store_true", + default=False, + help="If set true, it will keep the project directory for debugging.", + ) + + +@pytest.fixture +def board(request): + return request.config.getoption("--board") + + +@pytest.fixture +def tvm_debug(request): + return request.config.getoption("--tvm-debug") + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--test-build-only"): + skip_hardware_tests = pytest.mark.skip(reason="--test-build-only was passed") + for item in items: + if "requires_hardware" in item.keywords: + item.add_marker(skip_hardware_tests) + + +@pytest.fixture +def workspace_dir(board, tvm_debug): + """Creates workspace directory for each test.""" + parent_dir = pathlib.Path(os.path.dirname(__file__)) + filename = os.path.splitext(os.path.basename(__file__))[0] + board_workspace = ( + parent_dir + / f"workspace_{filename}_{board}" + / datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + ) + board_workspace_base = str(board_workspace) + number = 1 + while board_workspace.exists(): + board_workspace = pathlib.Path(board_workspace_base + f"-{number}") + number += 1 + + if not os.path.exists(board_workspace.parent): + os.makedirs(board_workspace.parent) + + keep_for_debug = tvm_debug if tvm_debug else None + test_temp_dir = tempdir(custom_path=board_workspace, keep_for_debug=keep_for_debug) + return test_temp_dir + + +@pytest.fixture(autouse=True) +def skip_by_board(request, board): + """Skip test if board is in the list.""" + if request.node.get_closest_marker("skip_boards"): + if board in request.node.get_closest_marker("skip_boards").args[0]: + pytest.skip("skipped on this board: {}".format(board)) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "skip_boards(board): skip test for the given board", + ) diff --git a/tests/micro/arduino/conftest.py b/tests/micro/arduino/conftest.py index bb4db18f7886..f8eff85a6053 100644 --- a/tests/micro/arduino/conftest.py +++ b/tests/micro/arduino/conftest.py @@ -15,35 +15,19 @@ # specific language governing permissions and limitations # under the License. -import pytest +pytest_plugins = [ + "tvm.micro.testing.pytest_plugin", +] -from test_utils import ARDUINO_BOARDS +import pytest def pytest_addoption(parser): - parser.addoption( - "--arduino-board", - nargs="+", - required=True, - choices=ARDUINO_BOARDS.keys(), - help="Arduino board for tests.", - ) parser.addoption( "--arduino-cli-cmd", default="arduino-cli", help="Path to `arduino-cli` command for flashing device.", ) - parser.addoption( - "--test-build-only", - action="store_true", - help="Only run tests that don't require physical hardware.", - ) - parser.addoption( - "--tvm-debug", - action="store_true", - default=False, - help="If given, enable a debug session while the test is running. Before running the test, in a separate shell, you should run: ", - ) def pytest_configure(config): @@ -52,20 +36,12 @@ def pytest_configure(config): ) -def pytest_collection_modifyitems(config, items): - if config.getoption("--test-build-only"): - skip_hardware_tests = pytest.mark.skip(reason="--test-build-only was passed") - for item in items: - if "requires_hardware" in item.keywords: - item.add_marker(skip_hardware_tests) - - # We might do project generation differently for different boards in the future # (to take advantage of multiple cores / external memory / etc.), so all tests # are parameterized by board -def pytest_generate_tests(metafunc): - board = metafunc.config.getoption("arduino_board") - metafunc.parametrize("board", board, scope="session") +# def pytest_generate_tests(metafunc): +# board = metafunc.config.getoption("arduino_board") +# metafunc.parametrize("board", board, scope="session") @pytest.fixture(scope="session") @@ -73,6 +49,6 @@ def arduino_cli_cmd(request): return request.config.getoption("--arduino-cli-cmd") -@pytest.fixture(scope="session") -def tvm_debug(request): - return request.config.getoption("--tvm-debug") +@pytest.fixture() +def project_dir(workspace_dir): + return workspace_dir / "project" diff --git a/tests/micro/arduino/test_arduino_error_detection.py b/tests/micro/arduino/test_arduino_error_detection.py index 583f8283ca4c..c04edf37b74f 100644 --- a/tests/micro/arduino/test_arduino_error_detection.py +++ b/tests/micro/arduino/test_arduino_error_detection.py @@ -23,11 +23,6 @@ import test_utils import tvm.testing -# A new project and workspace dir is created for EVERY test -@pytest.fixture -def workspace_dir(request, board): - return test_utils.make_workspace_dir("arduino_error_detection", board) - @pytest.fixture def project(board, arduino_cli_cmd, tvm_debug, workspace_dir): diff --git a/tests/micro/arduino/test_arduino_rpc_server.py b/tests/micro/arduino/test_arduino_rpc_server.py index acd4186940ac..278bb5516b71 100644 --- a/tests/micro/arduino/test_arduino_rpc_server.py +++ b/tests/micro/arduino/test_arduino_rpc_server.py @@ -37,11 +37,6 @@ import test_utils -# # A new project and workspace dir is created for EVERY test -@pytest.fixture -def workspace_dir(board): - return test_utils.make_workspace_dir("arduino_rpc_server", board) - def _make_session(model, arduino_board, arduino_cli_cmd, workspace_dir, mod, build_config): project = tvm.micro.generate_project( diff --git a/tests/micro/arduino/test_arduino_workflow.py b/tests/micro/arduino/test_arduino_workflow.py index d4f151845290..6131b6dfe4c8 100644 --- a/tests/micro/arduino/test_arduino_workflow.py +++ b/tests/micro/arduino/test_arduino_workflow.py @@ -37,20 +37,8 @@ """ -# Since these tests are sequential, we'll use the same project/workspace -# directory for all tests in this file -@pytest.fixture(scope="module") -def workspace_dir(request, board): - return test_utils.make_workspace_dir("arduino_workflow", board) - - -@pytest.fixture(scope="module") -def project_dir(workspace_dir): - return workspace_dir / "project" - - # We MUST pass workspace_dir, not project_dir, or the workspace will be dereferenced too soon -@pytest.fixture(scope="module") +@pytest.fixture() def project(board, arduino_cli_cmd, tvm_debug, workspace_dir): return test_utils.make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir) @@ -108,7 +96,7 @@ def test_import_rerouting(project_dir, project): # Build on top of the generated project by replacing the # top-level .ino fileand adding data input files, much # like a user would -@pytest.fixture(scope="module") +@pytest.fixture() def modified_project(project_dir, project): this_dir = pathlib.Path(__file__).parent kws_testdata_dir = this_dir.parent / "testdata" / "kws" @@ -124,7 +112,7 @@ def modified_project(project_dir, project): return project -@pytest.fixture(scope="module") +@pytest.fixture() def compiled_project(modified_project): modified_project.build() return modified_project @@ -142,7 +130,7 @@ def test_compile_yes_no_project(project_dir, project, compiled_project): ------------------------------------------------------------""" -@pytest.fixture(scope="module") +@pytest.fixture() def uploaded_project(compiled_project): compiled_project.flash() return compiled_project @@ -159,7 +147,7 @@ def uploaded_project(compiled_project): SERIAL_OUTPUT_HEADERS = "category,runtime,yes,no,silence,unknown" -@pytest.fixture(scope="module") +@pytest.fixture() def serial_output(uploaded_project): transport = uploaded_project.transport() transport.open() diff --git a/tests/micro/arduino/test_utils.py b/tests/micro/arduino/test_utils.py index 20e7d9e75001..73516ff28bd6 100644 --- a/tests/micro/arduino/test_utils.py +++ b/tests/micro/arduino/test_utils.py @@ -44,23 +44,6 @@ def arduino_boards() -> dict: ARDUINO_BOARDS = arduino_boards() -def make_workspace_dir(test_name, board): - filepath = pathlib.Path(__file__) - board_workspace = ( - filepath.parent - / f"workspace_{test_name}_{board}" - / datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S") - ) - - number = 0 - while board_workspace.exists(): - number += 1 - board_workspace = pathlib.Path(str(board_workspace) + f"-{number}") - board_workspace.parent.mkdir(exist_ok=True, parents=True) - t = tvm.contrib.utils.tempdir(board_workspace) - return t - - def make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir): this_dir = pathlib.Path(__file__).parent model = ARDUINO_BOARDS[board] diff --git a/tests/micro/common/conftest.py b/tests/micro/common/conftest.py index 10dda8774bca..0bf70ed06138 100644 --- a/tests/micro/common/conftest.py +++ b/tests/micro/common/conftest.py @@ -14,10 +14,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations -import pytest +pytest_plugins = [ + "tvm.micro.testing.pytest_plugin", +] -from ..zephyr.test_utils import ZEPHYR_BOARDS -from ..arduino.test_utils import ARDUINO_BOARDS +import pytest def pytest_addoption(parser): @@ -27,32 +28,8 @@ def pytest_addoption(parser): choices=["arduino", "zephyr"], help="Platform to run tests with", ) - parser.addoption( - "--board", - required=True, - choices=list(ARDUINO_BOARDS.keys()) + list(ZEPHYR_BOARDS.keys()), - help="microTVM boards for tests", - ) - parser.addoption( - "--test-build-only", - action="store_true", - help="Only run tests that don't require physical hardware.", - ) @pytest.fixture def platform(request): return request.config.getoption("--platform") - - -@pytest.fixture -def board(request): - return request.config.getoption("--board") - - -def pytest_collection_modifyitems(config, items): - if config.getoption("--test-build-only"): - skip_hardware_tests = pytest.mark.skip(reason="--test-build-only was passed") - for item in items: - if "requires_hardware" in item.keywords: - item.add_marker(skip_hardware_tests) diff --git a/tests/micro/zephyr/conftest.py b/tests/micro/zephyr/conftest.py index c4de48a0a47a..c72fd4f0c937 100644 --- a/tests/micro/zephyr/conftest.py +++ b/tests/micro/zephyr/conftest.py @@ -14,83 +14,20 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import datetime -import os -import pathlib -import pytest - -import test_utils +pytest_plugins = [ + "tvm.micro.testing.pytest_plugin", +] -from tvm.contrib.utils import tempdir +import pytest def pytest_addoption(parser): - parser.addoption( - "--zephyr-board", - required=True, - choices=test_utils.ZEPHYR_BOARDS.keys(), - help="Zephyr board for test.", - ) parser.addoption( "--west-cmd", default="west", help="Path to `west` command for flashing device." ) - parser.addoption( - "--tvm-debug", - action="store_true", - default=False, - help="If set true, enable a debug session while the test is running. Before running the test, in a separate shell, you should run: ", - ) - - -def pytest_generate_tests(metafunc): - if "board" in metafunc.fixturenames: - metafunc.parametrize("board", [metafunc.config.getoption("zephyr_board")]) @pytest.fixture def west_cmd(request): return request.config.getoption("--west-cmd") - - -@pytest.fixture -def tvm_debug(request): - return request.config.getoption("--tvm-debug") - - -@pytest.fixture -def temp_dir(board, tvm_debug): - parent_dir = pathlib.Path(os.path.dirname(__file__)) - filename = os.path.splitext(os.path.basename(__file__))[0] - board_workspace = ( - parent_dir - / f"workspace_{filename}_{board}" - / datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S") - ) - board_workspace_base = str(board_workspace) - number = 1 - while board_workspace.exists(): - board_workspace = pathlib.Path(board_workspace_base + f"-{number}") - number += 1 - - if not os.path.exists(board_workspace.parent): - os.makedirs(board_workspace.parent) - - keep_for_debug = tvm_debug if tvm_debug else None - test_temp_dir = tempdir(custom_path=board_workspace, keep_for_debug=keep_for_debug) - return test_temp_dir - - -@pytest.fixture(autouse=True) -def skip_by_board(request, board): - """Skip test if board is in the list.""" - if request.node.get_closest_marker("skip_boards"): - if board in request.node.get_closest_marker("skip_boards").args[0]: - pytest.skip("skipped on this board: {}".format(board)) - - -def pytest_configure(config): - config.addinivalue_line( - "markers", - "skip_boards(board): skip test for the given board", - ) diff --git a/tests/micro/zephyr/test_zephyr.py b/tests/micro/zephyr/test_zephyr.py index 05c8daa20c21..e8e0c3cd824d 100644 --- a/tests/micro/zephyr/test_zephyr.py +++ b/tests/micro/zephyr/test_zephyr.py @@ -89,7 +89,7 @@ def _make_add_sess(temp_dir, model, zephyr_board, west_cmd, build_config, dtype= # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_add_uint(temp_dir, board, west_cmd, tvm_debug): +def test_add_uint(workspace_dir, board, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" model = test_utils.ZEPHYR_BOARDS[board] @@ -108,14 +108,14 @@ def test_basic_add(sess): system_lib.get_function("add")(A_data, B_data, C_data) assert (C_data.numpy() == np.array([6, 7])).all() - with _make_add_sess(temp_dir, model, board, west_cmd, build_config) as sess: + with _make_add_sess(workspace_dir, model, board, west_cmd, build_config) as sess: test_basic_add(sess) # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_add_float(temp_dir, board, west_cmd, tvm_debug): +def test_add_float(workspace_dir, board, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" model = test_utils.ZEPHYR_BOARDS[board] if not test_utils.has_fpu(board): @@ -136,13 +136,15 @@ def test_basic_add(sess): system_lib.get_function("add")(A_data, B_data, C_data) assert (C_data.numpy() == np.array([7, 8])).all() - with _make_add_sess(temp_dir, model, board, west_cmd, build_config, dtype="float32") as sess: + with _make_add_sess( + workspace_dir, model, board, west_cmd, build_config, dtype="float32" + ) as sess: test_basic_add(sess) @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_platform_timer(temp_dir, board, west_cmd, tvm_debug): +def test_platform_timer(workspace_dir, board, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" model = test_utils.ZEPHYR_BOARDS[board] @@ -166,13 +168,13 @@ def test_basic_add(sess): assert result.mean > 0 assert len(result.results) == 3 - with _make_add_sess(temp_dir, model, board, west_cmd, build_config) as sess: + with _make_add_sess(workspace_dir, model, board, west_cmd, build_config) as sess: test_basic_add(sess) @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_relay(temp_dir, board, west_cmd, tvm_debug): +def test_relay(workspace_dir, board, west_cmd, tvm_debug): """Testing a simple relay graph""" model = test_utils.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} @@ -191,7 +193,7 @@ def test_relay(temp_dir, board, west_cmd, tvm_debug): with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.relay.build(ir_mod, target=target, runtime=runtime) - with _make_session(temp_dir, board, west_cmd, mod, build_config) as session: + with _make_session(workspace_dir, board, west_cmd, mod, build_config) as session: graph_mod = tvm.micro.create_local_graph_executor( mod.get_graph_json(), session.get_system_lib(), session.device ) @@ -205,7 +207,7 @@ def test_relay(temp_dir, board, west_cmd, tvm_debug): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_onnx(temp_dir, board, west_cmd, tvm_debug): +def test_onnx(workspace_dir, board, west_cmd, tvm_debug): """Testing a simple ONNX model.""" model = test_utils.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} @@ -237,7 +239,7 @@ def test_onnx(temp_dir, board, west_cmd, tvm_debug): lowered = relay.build(relay_mod, target, params=params, executor=executor, runtime=runtime) graph = lowered.get_graph_json() - with _make_session(temp_dir, board, west_cmd, lowered, build_config) as session: + with _make_session(workspace_dir, board, west_cmd, lowered, build_config) as session: graph_mod = tvm.micro.create_local_graph_executor( graph, session.get_system_lib(), session.device ) @@ -286,7 +288,7 @@ def check_result( @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_byoc_microtvm(temp_dir, board, west_cmd, tvm_debug): +def test_byoc_microtvm(workspace_dir, board, west_cmd, tvm_debug): """This is a simple test case to check BYOC capabilities of microTVM""" model = test_utils.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} @@ -329,7 +331,7 @@ def test_byoc_microtvm(temp_dir, board, west_cmd, tvm_debug): map_inputs = {"w{}".format(i): w_data[i] for i in range(8)} map_inputs["x"] = x_data check_result( - temp_dir=temp_dir, + temp_dir=workspace_dir, relay_mod=mod, map_inputs=map_inputs, out_shape=(30, 10), @@ -367,7 +369,7 @@ def _make_add_sess_with_shape(temp_dir, model, zephyr_board, west_cmd, shape, bu ) @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_rpc_large_array(temp_dir, board, west_cmd, tvm_debug, shape): +def test_rpc_large_array(workspace_dir, board, west_cmd, tvm_debug, shape): """Test large RPC array transfer.""" model = test_utils.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} @@ -381,13 +383,15 @@ def test_tensors(sess): C_data = tvm.nd.array(np.zeros(shape, dtype="int8"), device=sess.device) assert (C_data.numpy() == np.zeros(shape)).all() - with _make_add_sess_with_shape(temp_dir, model, board, west_cmd, shape, build_config) as sess: + with _make_add_sess_with_shape( + workspace_dir, model, board, west_cmd, shape, build_config + ) as sess: test_tensors(sess) @pytest.mark.xfail(strict=False, reason="See https://github.com/apache/tvm/issues/10297") @tvm.testing.requires_micro -def test_autotune_conv2d(temp_dir, board, west_cmd, tvm_debug): +def test_autotune_conv2d(workspace_dir, board, west_cmd, tvm_debug): """Test AutoTune for microTVM Zephyr""" if board != "qemu_x86": pytest.xfail(f"Autotune fails on {board}.") @@ -513,7 +517,7 @@ def test_autotune_conv2d(temp_dir, board, west_cmd, tvm_debug): @tvm.testing.requires_micro -def test_schedule_build_with_cmsis_dependency(temp_dir, board, west_cmd, tvm_debug): +def test_schedule_build_with_cmsis_dependency(workspace_dir, board, west_cmd, tvm_debug): """Test Relay schedule with CMSIS dependency. This test shows if microTVM Auto tuning with Zephyr breaks if CMSIS dependency was required for a schedule. """ @@ -555,7 +559,7 @@ def test_schedule_build_with_cmsis_dependency(temp_dir, board, west_cmd, tvm_deb "cmsis_path": os.getenv("CMSIS_PATH"), } - project_dir = temp_dir / "project" + project_dir = workspace_dir / "project" project = tvm.micro.generate_project( str(test_utils.TEMPLATE_PROJECT_DIR), mod, diff --git a/tests/micro/zephyr/test_zephyr_aot_exec.py b/tests/micro/zephyr/test_zephyr_aot_exec.py index 9bf1a7fc0608..a5a1e416b0c3 100644 --- a/tests/micro/zephyr/test_zephyr_aot_exec.py +++ b/tests/micro/zephyr/test_zephyr_aot_exec.py @@ -38,7 +38,7 @@ import test_utils -def _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config): +def _make_session(workspace_dir, zephyr_board, west_cmd, mod, build_config): config_main_stack_size = None if test_utils.qemu_boards(zephyr_board): # fyi: qemu_riscv64 seems to be the greediest stack user @@ -59,7 +59,7 @@ def _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config): project = tvm.micro.generate_project( str(test_utils.TEMPLATE_PROJECT_DIR), mod, - temp_dir / "project", + workspace_dir / "project", project_options, ) project.build() @@ -69,7 +69,7 @@ def _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_relay(temp_dir, board, west_cmd, tvm_debug): +def test_relay(workspace_dir, board, west_cmd, tvm_debug): """Testing a simple relay graph""" model = test_utils.ZEPHYR_BOARDS[board] @@ -90,7 +90,7 @@ def test_relay(temp_dir, board, west_cmd, tvm_debug): with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.relay.build(ir_mod, target=target, runtime=runtime, executor=executor) - with _make_session(temp_dir, board, west_cmd, mod, build_config) as session: + with _make_session(workspace_dir, board, west_cmd, mod, build_config) as session: aot_executor = tvm.runtime.executor.aot_executor.AotModule(session.create_aot_executor()) @@ -103,7 +103,7 @@ def test_relay(temp_dir, board, west_cmd, tvm_debug): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_aot_executor(temp_dir, board, west_cmd, tvm_debug): +def test_aot_executor(workspace_dir, board, west_cmd, tvm_debug): """Test use of the AOT executor with microTVM.""" model = test_utils.ZEPHYR_BOARDS[board] @@ -154,7 +154,7 @@ def do_test(): aot_executor.set_input("b", B_np_new) assert (B_data.numpy() == B_np_new).all() - with _make_session(temp_dir, board, west_cmd, mod, build_config) as session: + with _make_session(workspace_dir, board, west_cmd, mod, build_config) as session: do_test() diff --git a/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py b/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py index 3d509f100d6e..0770aa5584da 100644 --- a/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py +++ b/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py @@ -39,7 +39,7 @@ @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_tflite(temp_dir, board, west_cmd, tvm_debug): +def test_tflite(workspace_dir, board, west_cmd, tvm_debug): """Testing a TFLite model.""" model = test_utils.ZEPHYR_BOARDS[board] input_shape = (1, 49, 10, 1) @@ -78,7 +78,7 @@ def test_tflite(temp_dir, board, west_cmd, tvm_debug): sample = np.load(sample_path) project, _ = test_utils.generate_project( - temp_dir, + workspace_dir, board, west_cmd, lowered, @@ -95,7 +95,7 @@ def test_tflite(temp_dir, board, west_cmd, tvm_debug): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_qemu_make_fail(temp_dir, board, west_cmd, tvm_debug): +def test_qemu_make_fail(workspace_dir, board, west_cmd, tvm_debug): """Testing QEMU make fail.""" if board not in ["qemu_x86", "mps2_an521", "mps3_an547"]: pytest.skip(msg="Only for QEMU targets.") @@ -120,7 +120,15 @@ def test_qemu_make_fail(temp_dir, board, west_cmd, tvm_debug): sample = np.zeros(shape=shape, dtype=dtype) project, project_dir = test_utils.generate_project( - temp_dir, board, west_cmd, lowered, build_config, sample, shape, dtype, load_cmsis=False + workspace_dir, + board, + west_cmd, + lowered, + build_config, + sample, + shape, + dtype, + load_cmsis=False, ) file_path = ( diff --git a/tests/micro/zephyr/test_zephyr_armv7m.py b/tests/micro/zephyr/test_zephyr_armv7m.py index c629403ced82..6fb94543caaa 100644 --- a/tests/micro/zephyr/test_zephyr_armv7m.py +++ b/tests/micro/zephyr/test_zephyr_armv7m.py @@ -104,7 +104,7 @@ def _apply_desired_layout_no_simd(relay_mod): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_armv7m_intrinsic(temp_dir, board, west_cmd, tvm_debug): +def test_armv7m_intrinsic(workspace_dir, board, west_cmd, tvm_debug): """Testing a ARM v7m SIMD extension.""" if board not in [ @@ -137,11 +137,11 @@ def test_armv7m_intrinsic(temp_dir, board, west_cmd, tvm_debug): executor = Executor("aot", {"unpacked-api": True, "interface-api": "c"}) runtime = Runtime("crt") - temp_dir_simd = temp_dir / "simd" - temp_dir_no_simd = temp_dir / "nosimd" + workspace_dir_simd = workspace_dir / "simd" + workspace_dir_no_simd = workspace_dir / "nosimd" - os.makedirs(temp_dir_simd, exist_ok=True) - os.makedirs(temp_dir_no_simd, exist_ok=True) + os.makedirs(workspace_dir_simd, exist_ok=True) + os.makedirs(workspace_dir_no_simd, exist_ok=True) with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): lowered_simd = relay.build( @@ -152,7 +152,7 @@ def test_armv7m_intrinsic(temp_dir, board, west_cmd, tvm_debug): ) simd_project, _ = test_utils.generate_project( - temp_dir_simd, + workspace_dir_simd, board, west_cmd, lowered_simd, @@ -165,7 +165,7 @@ def test_armv7m_intrinsic(temp_dir, board, west_cmd, tvm_debug): result_simd, time_simd = test_utils.run_model(simd_project) no_simd_project, _ = test_utils.generate_project( - temp_dir_no_simd, + workspace_dir_no_simd, board, west_cmd, lowered_no_simd, diff --git a/tests/scripts/task_python_microtvm.sh b/tests/scripts/task_python_microtvm.sh index e057883776bb..a7632a626712 100755 --- a/tests/scripts/task_python_microtvm.sh +++ b/tests/scripts/task_python_microtvm.sh @@ -24,22 +24,22 @@ source tests/scripts/setup-pytest-env.sh make cython3 # Zephyr -run_pytest ctypes python-microtvm-zephyr-qemu_x86 tests/micro/zephyr --zephyr-board=qemu_x86 -run_pytest ctypes python-microtvm-zephyr-qemu_riscv32 tests/micro/zephyr --zephyr-board=qemu_riscv32 -run_pytest ctypes python-microtvm-zephyr-qemu_riscv64 tests/micro/zephyr --zephyr-board=qemu_riscv64 -run_pytest ctypes python-microtvm-zephyr-mps2_an521 tests/micro/zephyr --zephyr-board=mps2_an521 +run_pytest ctypes python-microtvm-zephyr-qemu_x86 tests/micro/zephyr --board=qemu_x86 +run_pytest ctypes python-microtvm-zephyr-qemu_riscv32 tests/micro/zephyr --board=qemu_riscv32 +run_pytest ctypes python-microtvm-zephyr-qemu_riscv64 tests/micro/zephyr --board=qemu_riscv64 +run_pytest ctypes python-microtvm-zephyr-mps2_an521 tests/micro/zephyr --board=mps2_an521 # Arduino run_pytest ctypes python-microtvm-arduino apps/microtvm/arduino/template_project/tests -run_pytest ctypes python-microtvm-arduino-nano33ble tests/micro/arduino --test-build-only --arduino-board=nano33ble -run_pytest ctypes python-microtvm-arduino-due tests/micro/arduino --test-build-only --arduino-board=due +run_pytest ctypes python-microtvm-arduino-nano33ble tests/micro/arduino --board=nano33ble --test-build-only +run_pytest ctypes python-microtvm-arduino-due tests/micro/arduino --board=due --test-build-only # STM32 run_pytest ctypes python-microtvm-stm32 tests/micro/stm32 # Common Tests run_pytest ctypes python-microtvm-common-qemu_x86 tests/micro/common --platform=zephyr --board=qemu_x86 -run_pytest ctypes python-microtvm-common-due tests/micro/common --platform=arduino --test-build-only --board=due +run_pytest ctypes python-microtvm-common-due tests/micro/common --platform=arduino --board=due --test-build-only # Tutorials python3 gallery/how_to/work_with_microtvm/micro_tflite.py From 2cfd8786af566154db52813c002ea507ecc0dead Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Thu, 28 Jul 2022 17:21:39 -0700 Subject: [PATCH 2/8] fix error --- tests/micro/common/test_autotune.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/micro/common/test_autotune.py b/tests/micro/common/test_autotune.py index 1575036bf726..d62c95934aa3 100644 --- a/tests/micro/common/test_autotune.py +++ b/tests/micro/common/test_autotune.py @@ -75,7 +75,7 @@ def test_kws_autotune_workflow(platform, board, tmp_path): labels = [0, 0, 0] # Validate perforance across random runs - time, _ = tvm.micro.testing.evaluate_model_accuracy( + time, _, _ = tvm.micro.testing.evaluate_model_accuracy( session, aot_executor, samples, labels, runs_per_sample=20 ) # `time` is the average time taken to execute model inference on the From b416bd3c47a13ff89ac24ad1beeb98979e97dfcf Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Fri, 29 Jul 2022 10:58:54 -0700 Subject: [PATCH 3/8] fix scope --- python/tvm/micro/testing/pytest_plugin.py | 13 ++++------ tests/micro/arduino/conftest.py | 13 ---------- tests/micro/arduino/test_arduino_workflow.py | 25 ++++++++++++++------ tests/micro/arduino/test_utils.py | 17 +++++++++++++ 4 files changed, 40 insertions(+), 28 deletions(-) diff --git a/python/tvm/micro/testing/pytest_plugin.py b/python/tvm/micro/testing/pytest_plugin.py index 54a83b7983f2..53c93b111498 100644 --- a/python/tvm/micro/testing/pytest_plugin.py +++ b/python/tvm/micro/testing/pytest_plugin.py @@ -72,12 +72,12 @@ def pytest_addoption(parser): ) -@pytest.fixture +@pytest.fixture(scope="module") def board(request): return request.config.getoption("--board") -@pytest.fixture +@pytest.fixture(scope="module") def tvm_debug(request): return request.config.getoption("--tvm-debug") @@ -91,14 +91,11 @@ def pytest_collection_modifyitems(config, items): @pytest.fixture -def workspace_dir(board, tvm_debug): +def workspace_dir(request, board, tvm_debug): """Creates workspace directory for each test.""" - parent_dir = pathlib.Path(os.path.dirname(__file__)) - filename = os.path.splitext(os.path.basename(__file__))[0] + parent_dir = pathlib.Path(os.path.dirname(request.module.__file__)) board_workspace = ( - parent_dir - / f"workspace_{filename}_{board}" - / datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + parent_dir / f"workspace_{board}" / datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S") ) board_workspace_base = str(board_workspace) number = 1 diff --git a/tests/micro/arduino/conftest.py b/tests/micro/arduino/conftest.py index f8eff85a6053..a5ce8127c0bb 100644 --- a/tests/micro/arduino/conftest.py +++ b/tests/micro/arduino/conftest.py @@ -36,19 +36,6 @@ def pytest_configure(config): ) -# We might do project generation differently for different boards in the future -# (to take advantage of multiple cores / external memory / etc.), so all tests -# are parameterized by board -# def pytest_generate_tests(metafunc): -# board = metafunc.config.getoption("arduino_board") -# metafunc.parametrize("board", board, scope="session") - - @pytest.fixture(scope="session") def arduino_cli_cmd(request): return request.config.getoption("--arduino-cli-cmd") - - -@pytest.fixture() -def project_dir(workspace_dir): - return workspace_dir / "project" diff --git a/tests/micro/arduino/test_arduino_workflow.py b/tests/micro/arduino/test_arduino_workflow.py index 6131b6dfe4c8..6c3df669086b 100644 --- a/tests/micro/arduino/test_arduino_workflow.py +++ b/tests/micro/arduino/test_arduino_workflow.py @@ -36,11 +36,22 @@ 6. Use serial connection to ensure model behaves correctly """ +# Since these tests are sequential, we'll use the same project/workspace +# directory for all tests in this file +@pytest.fixture(scope="module") +def workflow_workspace_dir(request, board): + return test_utils.make_workspace_dir("arduino_workflow", board) + + +@pytest.fixture(scope="module") +def project_dir(workflow_workspace_dir): + return workflow_workspace_dir / "project" + # We MUST pass workspace_dir, not project_dir, or the workspace will be dereferenced too soon -@pytest.fixture() -def project(board, arduino_cli_cmd, tvm_debug, workspace_dir): - return test_utils.make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir) +@pytest.fixture(scope="module") +def project(board, arduino_cli_cmd, tvm_debug, workflow_workspace_dir): + return test_utils.make_kws_project(board, arduino_cli_cmd, tvm_debug, workflow_workspace_dir) def _get_directory_elements(directory): @@ -96,7 +107,7 @@ def test_import_rerouting(project_dir, project): # Build on top of the generated project by replacing the # top-level .ino fileand adding data input files, much # like a user would -@pytest.fixture() +@pytest.fixture(scope="module") def modified_project(project_dir, project): this_dir = pathlib.Path(__file__).parent kws_testdata_dir = this_dir.parent / "testdata" / "kws" @@ -112,7 +123,7 @@ def modified_project(project_dir, project): return project -@pytest.fixture() +@pytest.fixture(scope="module") def compiled_project(modified_project): modified_project.build() return modified_project @@ -130,7 +141,7 @@ def test_compile_yes_no_project(project_dir, project, compiled_project): ------------------------------------------------------------""" -@pytest.fixture() +@pytest.fixture(scope="module") def uploaded_project(compiled_project): compiled_project.flash() return compiled_project @@ -147,7 +158,7 @@ def uploaded_project(compiled_project): SERIAL_OUTPUT_HEADERS = "category,runtime,yes,no,silence,unknown" -@pytest.fixture() +@pytest.fixture(scope="module") def serial_output(uploaded_project): transport = uploaded_project.transport() transport.open() diff --git a/tests/micro/arduino/test_utils.py b/tests/micro/arduino/test_utils.py index 73516ff28bd6..20e7d9e75001 100644 --- a/tests/micro/arduino/test_utils.py +++ b/tests/micro/arduino/test_utils.py @@ -44,6 +44,23 @@ def arduino_boards() -> dict: ARDUINO_BOARDS = arduino_boards() +def make_workspace_dir(test_name, board): + filepath = pathlib.Path(__file__) + board_workspace = ( + filepath.parent + / f"workspace_{test_name}_{board}" + / datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + ) + + number = 0 + while board_workspace.exists(): + number += 1 + board_workspace = pathlib.Path(str(board_workspace) + f"-{number}") + board_workspace.parent.mkdir(exist_ok=True, parents=True) + t = tvm.contrib.utils.tempdir(board_workspace) + return t + + def make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir): this_dir = pathlib.Path(__file__).parent model = ARDUINO_BOARDS[board] From d9c774cf5e1ce5ebc234a81ed36a921a5bd52a27 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Fri, 29 Jul 2022 14:51:24 -0700 Subject: [PATCH 4/8] address @guberti comments --- python/tvm/micro/testing/evaluation.py | 11 ++----- python/tvm/micro/testing/pytest_plugin.py | 38 ++++++----------------- tests/micro/zephyr/conftest.py | 2 +- 3 files changed, 13 insertions(+), 38 deletions(-) diff --git a/python/tvm/micro/testing/evaluation.py b/python/tvm/micro/testing/evaluation.py index 6e426944f1fd..26d3c3432712 100644 --- a/python/tvm/micro/testing/evaluation.py +++ b/python/tvm/micro/testing/evaluation.py @@ -95,20 +95,13 @@ def create_aot_session( executor = tvm.relay.backend.Executor("aot") crt_runtime = tvm.relay.backend.Runtime("crt", {"system-lib": True}) - if use_cmsis_nn: - with tvm.transform.PassContext( - config={ - "tir.disable_vectorize": True, - "relay.ext.cmsisnn.options": {"mcpu": target.mcpu}, - } - ): - mod = cmsisnn.partition_for_cmsisnn(mod, params, mcpu=target.mcpu) - with ExitStack() as stack: config = {"tir.disable_vectorize": True} if use_cmsis_nn: config["relay.ext.cmsisnn.options"] = {"mcpu": target.mcpu} stack.enter_context(tvm.transform.PassContext(opt_level=3, config=config)) + if use_cmsis_nn: + mod = cmsisnn.partition_for_cmsisnn(mod, params, mcpu=target.mcpu) if tune_logs is not None: stack.enter_context(tvm.autotvm.apply_history_best(tune_logs)) diff --git a/python/tvm/micro/testing/pytest_plugin.py b/python/tvm/micro/testing/pytest_plugin.py index 53c93b111498..c652dd7ae07f 100644 --- a/python/tvm/micro/testing/pytest_plugin.py +++ b/python/tvm/micro/testing/pytest_plugin.py @@ -20,35 +20,13 @@ values from testing parameters """ import pathlib -import json import os import datetime import pytest -import tvm from tvm.contrib.utils import tempdir - -def zephyr_boards() -> dict: - """Returns a dict mapping board to target model""" - with open( - pathlib.Path(tvm.micro.get_microtvm_template_projects("zephyr")) / "boards.json" - ) as f: - board_properties = json.load(f) - - boards_model = {board: info["model"] for board, info in board_properties.items()} - return boards_model - - -def get_boards(platform: str) -> dict: - """Returns a dict mapping board to target model""" - with open( - pathlib.Path(tvm.micro.get_microtvm_template_projects(platform)) / "boards.json" - ) as f: - board_properties = json.load(f) - - boards_model = {board: info["model"] for board, info in board_properties.items()} - return boards_model +from .utils import get_supported_boards def pytest_addoption(parser): @@ -56,8 +34,12 @@ def pytest_addoption(parser): parser.addoption( "--board", required=True, - choices=list(get_boards("zephyr").keys()) + list(get_boards("arduino").keys()), - help="microTVM boards for tests", + choices=list(get_supported_boards("zephyr").keys()) + + list(get_supported_boards("arduino").keys()), + help=( + "microTVM boards for tests. Board refers to instances" + "of microcontrollers/emulators defined in a platform." + ), ) parser.addoption( "--test-build-only", @@ -72,12 +54,12 @@ def pytest_addoption(parser): ) -@pytest.fixture(scope="module") +@pytest.fixture(scope="session") def board(request): return request.config.getoption("--board") -@pytest.fixture(scope="module") +@pytest.fixture(scope="session") def tvm_debug(request): return request.config.getoption("--tvm-debug") @@ -111,7 +93,7 @@ def workspace_dir(request, board, tvm_debug): return test_temp_dir -@pytest.fixture(autouse=True) +@pytest.fixture(autouse=True, scope="session") def skip_by_board(request, board): """Skip test if board is in the list.""" if request.node.get_closest_marker("skip_boards"): diff --git a/tests/micro/zephyr/conftest.py b/tests/micro/zephyr/conftest.py index c72fd4f0c937..9f3b56769075 100644 --- a/tests/micro/zephyr/conftest.py +++ b/tests/micro/zephyr/conftest.py @@ -28,6 +28,6 @@ def pytest_addoption(parser): ) -@pytest.fixture +@pytest.fixture(scope="session") def west_cmd(request): return request.config.getoption("--west-cmd") From 989facda97cf7f2d356bff6ad589473673fbe8bc Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Mon, 1 Aug 2022 08:20:47 -0700 Subject: [PATCH 5/8] fix help message --- python/tvm/micro/testing/pytest_plugin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/tvm/micro/testing/pytest_plugin.py b/python/tvm/micro/testing/pytest_plugin.py index c652dd7ae07f..d5e47d855996 100644 --- a/python/tvm/micro/testing/pytest_plugin.py +++ b/python/tvm/micro/testing/pytest_plugin.py @@ -50,7 +50,10 @@ def pytest_addoption(parser): "--tvm-debug", action="store_true", default=False, - help="If set true, it will keep the project directory for debugging.", + help=( + "If set true, it will keep the project directory for debugging." + "Also, it will enable debug level logging in project generation." + ), ) From d5c0ba5d8f0714fa238f4fe82224ceb1528415cb Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Mon, 1 Aug 2022 09:51:19 -0700 Subject: [PATCH 6/8] rename tvm_debug and added .gitignore --- python/tvm/micro/testing/pytest_plugin.py | 8 ++--- tests/micro/.gitignore | 2 ++ .../arduino/test_arduino_error_detection.py | 4 +-- .../micro/arduino/test_arduino_rpc_server.py | 24 ++++++------- tests/micro/arduino/test_arduino_workflow.py | 6 ++-- tests/micro/arduino/test_utils.py | 4 +-- tests/micro/zephyr/test_zephyr.py | 36 +++++++++---------- tests/micro/zephyr/test_zephyr_aot_exec.py | 8 ++--- .../zephyr/test_zephyr_aot_exec_standalone.py | 8 ++--- tests/micro/zephyr/test_zephyr_armv7m.py | 4 +-- 10 files changed, 54 insertions(+), 50 deletions(-) create mode 100644 tests/micro/.gitignore diff --git a/python/tvm/micro/testing/pytest_plugin.py b/python/tvm/micro/testing/pytest_plugin.py index d5e47d855996..13d9ca87f810 100644 --- a/python/tvm/micro/testing/pytest_plugin.py +++ b/python/tvm/micro/testing/pytest_plugin.py @@ -47,7 +47,7 @@ def pytest_addoption(parser): help="Only run tests that don't require physical hardware.", ) parser.addoption( - "--tvm-debug", + "--microtvm-debug", action="store_true", default=False, help=( @@ -63,7 +63,7 @@ def board(request): @pytest.fixture(scope="session") -def tvm_debug(request): +def microtvm_debug(request): return request.config.getoption("--tvm-debug") @@ -76,7 +76,7 @@ def pytest_collection_modifyitems(config, items): @pytest.fixture -def workspace_dir(request, board, tvm_debug): +def workspace_dir(request, board, microtvm_debug): """Creates workspace directory for each test.""" parent_dir = pathlib.Path(os.path.dirname(request.module.__file__)) board_workspace = ( @@ -91,7 +91,7 @@ def workspace_dir(request, board, tvm_debug): if not os.path.exists(board_workspace.parent): os.makedirs(board_workspace.parent) - keep_for_debug = tvm_debug if tvm_debug else None + keep_for_debug = microtvm_debug if microtvm_debug else None test_temp_dir = tempdir(custom_path=board_workspace, keep_for_debug=keep_for_debug) return test_temp_dir diff --git a/tests/micro/.gitignore b/tests/micro/.gitignore new file mode 100644 index 000000000000..88d8d834ed2c --- /dev/null +++ b/tests/micro/.gitignore @@ -0,0 +1,2 @@ +# test workspaces +*/workspace_*/ diff --git a/tests/micro/arduino/test_arduino_error_detection.py b/tests/micro/arduino/test_arduino_error_detection.py index c04edf37b74f..de5e5bb56c94 100644 --- a/tests/micro/arduino/test_arduino_error_detection.py +++ b/tests/micro/arduino/test_arduino_error_detection.py @@ -25,8 +25,8 @@ @pytest.fixture -def project(board, arduino_cli_cmd, tvm_debug, workspace_dir): - return test_utils.make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir) +def project(board, arduino_cli_cmd, microtvm_debug, workspace_dir): + return test_utils.make_kws_project(board, arduino_cli_cmd, microtvm_debug, workspace_dir) def test_blank_project_compiles(workspace_dir, project): diff --git a/tests/micro/arduino/test_arduino_rpc_server.py b/tests/micro/arduino/test_arduino_rpc_server.py index 278bb5516b71..3440fde8f7e6 100644 --- a/tests/micro/arduino/test_arduino_rpc_server.py +++ b/tests/micro/arduino/test_arduino_rpc_server.py @@ -79,11 +79,11 @@ def _make_add_sess(model, arduino_board, arduino_cli_cmd, workspace_dir, build_c # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_compile_runtime(board, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_compile_runtime(board, arduino_cli_cmd, microtvm_debug, workspace_dir): """Test compiling the on-device runtime.""" model = test_utils.ARDUINO_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_basic_add(sess): @@ -104,11 +104,11 @@ def test_basic_add(sess): @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_platform_timer(board, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_platform_timer(board, arduino_cli_cmd, microtvm_debug, workspace_dir): """Test compiling the on-device runtime.""" model = test_utils.ARDUINO_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_basic_add(sess): @@ -134,10 +134,10 @@ def test_basic_add(sess): @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_relay(board, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_relay(board, arduino_cli_cmd, microtvm_debug, workspace_dir): """Testing a simple relay graph""" model = test_utils.ARDUINO_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} shape = (10,) dtype = "int8" @@ -167,10 +167,10 @@ def test_relay(board, arduino_cli_cmd, tvm_debug, workspace_dir): @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_onnx(board, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_onnx(board, arduino_cli_cmd, microtvm_debug, workspace_dir): """Testing a simple ONNX model.""" model = test_utils.ARDUINO_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # Load test images. this_dir = pathlib.Path(__file__).parent @@ -258,10 +258,10 @@ def check_result( @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_byoc_microtvm(board, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_byoc_microtvm(board, arduino_cli_cmd, microtvm_debug, workspace_dir): """This is a simple test case to check BYOC capabilities of microTVM""" model = test_utils.ARDUINO_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} x = relay.var("x", shape=(10, 10)) w0 = relay.var("w0", shape=(10, 10)) @@ -342,10 +342,10 @@ def _make_add_sess_with_shape( ) @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_rpc_large_array(board, arduino_cli_cmd, tvm_debug, workspace_dir, shape): +def test_rpc_large_array(board, arduino_cli_cmd, microtvm_debug, workspace_dir, shape): """Test large RPC array transfer.""" model = test_utils.ARDUINO_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_tensors(sess): diff --git a/tests/micro/arduino/test_arduino_workflow.py b/tests/micro/arduino/test_arduino_workflow.py index 6c3df669086b..35bfa2556231 100644 --- a/tests/micro/arduino/test_arduino_workflow.py +++ b/tests/micro/arduino/test_arduino_workflow.py @@ -50,8 +50,10 @@ def project_dir(workflow_workspace_dir): # We MUST pass workspace_dir, not project_dir, or the workspace will be dereferenced too soon @pytest.fixture(scope="module") -def project(board, arduino_cli_cmd, tvm_debug, workflow_workspace_dir): - return test_utils.make_kws_project(board, arduino_cli_cmd, tvm_debug, workflow_workspace_dir) +def project(board, arduino_cli_cmd, microtvm_debug, workflow_workspace_dir): + return test_utils.make_kws_project( + board, arduino_cli_cmd, microtvm_debug, workflow_workspace_dir + ) def _get_directory_elements(directory): diff --git a/tests/micro/arduino/test_utils.py b/tests/micro/arduino/test_utils.py index 20e7d9e75001..42ad7d40f35b 100644 --- a/tests/micro/arduino/test_utils.py +++ b/tests/micro/arduino/test_utils.py @@ -61,10 +61,10 @@ def make_workspace_dir(test_name, board): return t -def make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir): +def make_kws_project(board, arduino_cli_cmd, microtvm_debug, workspace_dir): this_dir = pathlib.Path(__file__).parent model = ARDUINO_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} mod, params = fetch_model_from_url( url="https://github.com/tensorflow/tflite-micro/raw/main/tensorflow/lite/micro/examples/micro_speech/micro_speech.tflite", diff --git a/tests/micro/zephyr/test_zephyr.py b/tests/micro/zephyr/test_zephyr.py index e8e0c3cd824d..1f53e4baa8c3 100644 --- a/tests/micro/zephyr/test_zephyr.py +++ b/tests/micro/zephyr/test_zephyr.py @@ -89,11 +89,11 @@ def _make_add_sess(temp_dir, model, zephyr_board, west_cmd, build_config, dtype= # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_add_uint(workspace_dir, board, west_cmd, tvm_debug): +def test_add_uint(workspace_dir, board, west_cmd, microtvm_debug): """Test compiling the on-device runtime.""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_basic_add(sess): @@ -115,13 +115,13 @@ def test_basic_add(sess): # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_add_float(workspace_dir, board, west_cmd, tvm_debug): +def test_add_float(workspace_dir, board, west_cmd, microtvm_debug): """Test compiling the on-device runtime.""" model = test_utils.ZEPHYR_BOARDS[board] if not test_utils.has_fpu(board): pytest.skip(f"FPU not enabled for {board}") - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_basic_add(sess): @@ -144,11 +144,11 @@ def test_basic_add(sess): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_platform_timer(workspace_dir, board, west_cmd, tvm_debug): +def test_platform_timer(workspace_dir, board, west_cmd, microtvm_debug): """Test compiling the on-device runtime.""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_basic_add(sess): @@ -174,10 +174,10 @@ def test_basic_add(sess): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_relay(workspace_dir, board, west_cmd, tvm_debug): +def test_relay(workspace_dir, board, west_cmd, microtvm_debug): """Testing a simple relay graph""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} shape = (10,) dtype = "int8" @@ -207,10 +207,10 @@ def test_relay(workspace_dir, board, west_cmd, tvm_debug): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_onnx(workspace_dir, board, west_cmd, tvm_debug): +def test_onnx(workspace_dir, board, west_cmd, microtvm_debug): """Testing a simple ONNX model.""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} this_dir = pathlib.Path(os.path.dirname(__file__)) mnist_testdata = this_dir.parent / "testdata" / "mnist" @@ -288,10 +288,10 @@ def check_result( @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_byoc_microtvm(workspace_dir, board, west_cmd, tvm_debug): +def test_byoc_microtvm(workspace_dir, board, west_cmd, microtvm_debug): """This is a simple test case to check BYOC capabilities of microTVM""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} x = relay.var("x", shape=(10, 10)) w0 = relay.var("w0", shape=(10, 10)) w1 = relay.var("w1", shape=(10, 10)) @@ -369,10 +369,10 @@ def _make_add_sess_with_shape(temp_dir, model, zephyr_board, west_cmd, shape, bu ) @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_rpc_large_array(workspace_dir, board, west_cmd, tvm_debug, shape): +def test_rpc_large_array(workspace_dir, board, west_cmd, microtvm_debug, shape): """Test large RPC array transfer.""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_tensors(sess): @@ -391,14 +391,14 @@ def test_tensors(sess): @pytest.mark.xfail(strict=False, reason="See https://github.com/apache/tvm/issues/10297") @tvm.testing.requires_micro -def test_autotune_conv2d(workspace_dir, board, west_cmd, tvm_debug): +def test_autotune_conv2d(workspace_dir, board, west_cmd, microtvm_debug): """Test AutoTune for microTVM Zephyr""" if board != "qemu_x86": pytest.xfail(f"Autotune fails on {board}.") runtime = Runtime("crt", {"system-lib": True}) model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # Create a Relay model data_shape = (1, 3, 16, 16) @@ -517,12 +517,12 @@ def test_autotune_conv2d(workspace_dir, board, west_cmd, tvm_debug): @tvm.testing.requires_micro -def test_schedule_build_with_cmsis_dependency(workspace_dir, board, west_cmd, tvm_debug): +def test_schedule_build_with_cmsis_dependency(workspace_dir, board, west_cmd, microtvm_debug): """Test Relay schedule with CMSIS dependency. This test shows if microTVM Auto tuning with Zephyr breaks if CMSIS dependency was required for a schedule. """ model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} target = tvm.target.target.micro(model, options=["-keys=arm_cpu,cpu"]) isa = arm_isa.IsaAnalyzer(target) diff --git a/tests/micro/zephyr/test_zephyr_aot_exec.py b/tests/micro/zephyr/test_zephyr_aot_exec.py index a5a1e416b0c3..054a06aa020b 100644 --- a/tests/micro/zephyr/test_zephyr_aot_exec.py +++ b/tests/micro/zephyr/test_zephyr_aot_exec.py @@ -69,11 +69,11 @@ def _make_session(workspace_dir, zephyr_board, west_cmd, mod, build_config): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_relay(workspace_dir, board, west_cmd, tvm_debug): +def test_relay(workspace_dir, board, west_cmd, microtvm_debug): """Testing a simple relay graph""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} shape = (10,) dtype = "int8" @@ -103,11 +103,11 @@ def test_relay(workspace_dir, board, west_cmd, tvm_debug): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_aot_executor(workspace_dir, board, west_cmd, tvm_debug): +def test_aot_executor(workspace_dir, board, west_cmd, microtvm_debug): """Test use of the AOT executor with microTVM.""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} shape = (10,) dtype = "int8" diff --git a/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py b/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py index 0770aa5584da..da9d63216f15 100644 --- a/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py +++ b/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py @@ -39,12 +39,12 @@ @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_tflite(workspace_dir, board, west_cmd, tvm_debug): +def test_tflite(workspace_dir, board, west_cmd, microtvm_debug): """Testing a TFLite model.""" model = test_utils.ZEPHYR_BOARDS[board] input_shape = (1, 49, 10, 1) output_shape = (1, 12) - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} model_url = "https://github.com/tlc-pack/web-data/raw/25fe99fb00329a26bd37d3dca723da94316fd34c/testdata/microTVM/model/keyword_spotting_quant.tflite" model_path = download_testdata(model_url, "keyword_spotting_quant.tflite", module="model") @@ -95,13 +95,13 @@ def test_tflite(workspace_dir, board, west_cmd, tvm_debug): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_qemu_make_fail(workspace_dir, board, west_cmd, tvm_debug): +def test_qemu_make_fail(workspace_dir, board, west_cmd, microtvm_debug): """Testing QEMU make fail.""" if board not in ["qemu_x86", "mps2_an521", "mps3_an547"]: pytest.skip(msg="Only for QEMU targets.") model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} shape = (10,) dtype = "float32" diff --git a/tests/micro/zephyr/test_zephyr_armv7m.py b/tests/micro/zephyr/test_zephyr_armv7m.py index 6fb94543caaa..1f6a1a1bc13e 100644 --- a/tests/micro/zephyr/test_zephyr_armv7m.py +++ b/tests/micro/zephyr/test_zephyr_armv7m.py @@ -104,7 +104,7 @@ def _apply_desired_layout_no_simd(relay_mod): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_armv7m_intrinsic(workspace_dir, board, west_cmd, tvm_debug): +def test_armv7m_intrinsic(workspace_dir, board, west_cmd, microtvm_debug): """Testing a ARM v7m SIMD extension.""" if board not in [ @@ -118,7 +118,7 @@ def test_armv7m_intrinsic(workspace_dir, board, west_cmd, tvm_debug): model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} this_dir = pathlib.Path(os.path.dirname(__file__)) testdata_dir = this_dir.parent / "testdata" / "mnist" From 1c51152f64f0263fd5f205a8916d6b57d4192ad9 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Mon, 1 Aug 2022 13:13:05 -0700 Subject: [PATCH 7/8] fix bug --- python/tvm/micro/testing/pytest_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tvm/micro/testing/pytest_plugin.py b/python/tvm/micro/testing/pytest_plugin.py index 13d9ca87f810..7324142b6ea7 100644 --- a/python/tvm/micro/testing/pytest_plugin.py +++ b/python/tvm/micro/testing/pytest_plugin.py @@ -64,7 +64,7 @@ def board(request): @pytest.fixture(scope="session") def microtvm_debug(request): - return request.config.getoption("--tvm-debug") + return request.config.getoption("--microtvm-debug") def pytest_collection_modifyitems(config, items): From 44931125d838c58bafd58ff10a944dad208305a0 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Mon, 1 Aug 2022 17:07:44 -0700 Subject: [PATCH 8/8] fix bug --- python/tvm/micro/testing/pytest_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tvm/micro/testing/pytest_plugin.py b/python/tvm/micro/testing/pytest_plugin.py index 7324142b6ea7..b677588172b7 100644 --- a/python/tvm/micro/testing/pytest_plugin.py +++ b/python/tvm/micro/testing/pytest_plugin.py @@ -96,7 +96,7 @@ def workspace_dir(request, board, microtvm_debug): return test_temp_dir -@pytest.fixture(autouse=True, scope="session") +@pytest.fixture(autouse=True) def skip_by_board(request, board): """Skip test if board is in the list.""" if request.node.get_closest_marker("skip_boards"):