diff --git a/python/tvm/micro/testing/evaluation.py b/python/tvm/micro/testing/evaluation.py index 5407fcbabf10..26d3c3432712 100644 --- a/python/tvm/micro/testing/evaluation.py +++ b/python/tvm/micro/testing/evaluation.py @@ -28,6 +28,7 @@ import tempfile import tvm +from tvm.relay.op.contrib import cmsisnn def tune_model( @@ -99,6 +100,8 @@ def create_aot_session( if use_cmsis_nn: config["relay.ext.cmsisnn.options"] = {"mcpu": target.mcpu} stack.enter_context(tvm.transform.PassContext(opt_level=3, config=config)) + if use_cmsis_nn: + mod = cmsisnn.partition_for_cmsisnn(mod, params, mcpu=target.mcpu) if tune_logs is not None: stack.enter_context(tvm.autotvm.apply_history_best(tune_logs)) @@ -153,4 +156,4 @@ def evaluate_model_accuracy(session, aot_executor, input_data, true_labels, runs num_correct = sum(u == v for u, v in zip(true_labels, predicted_labels)) average_time = sum(aot_runtimes) / len(aot_runtimes) accuracy = num_correct / len(predicted_labels) - return average_time, accuracy + return average_time, accuracy, predicted_labels diff --git a/python/tvm/micro/testing/pytest_plugin.py b/python/tvm/micro/testing/pytest_plugin.py new file mode 100644 index 000000000000..b677588172b7 --- /dev/null +++ b/python/tvm/micro/testing/pytest_plugin.py @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: disable=invalid-name,redefined-outer-name +""" microTVM testing fixtures used to deduce testing argument + values from testing parameters """ + +import pathlib +import os +import datetime +import pytest + +from tvm.contrib.utils import tempdir + +from .utils import get_supported_boards + + +def pytest_addoption(parser): + """Adds more pytest arguments""" + parser.addoption( + "--board", + required=True, + choices=list(get_supported_boards("zephyr").keys()) + + list(get_supported_boards("arduino").keys()), + help=( + "microTVM boards for tests. Board refers to instances" + "of microcontrollers/emulators defined in a platform." + ), + ) + parser.addoption( + "--test-build-only", + action="store_true", + help="Only run tests that don't require physical hardware.", + ) + parser.addoption( + "--microtvm-debug", + action="store_true", + default=False, + help=( + "If set true, it will keep the project directory for debugging." + "Also, it will enable debug level logging in project generation." + ), + ) + + +@pytest.fixture(scope="session") +def board(request): + return request.config.getoption("--board") + + +@pytest.fixture(scope="session") +def microtvm_debug(request): + return request.config.getoption("--microtvm-debug") + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--test-build-only"): + skip_hardware_tests = pytest.mark.skip(reason="--test-build-only was passed") + for item in items: + if "requires_hardware" in item.keywords: + item.add_marker(skip_hardware_tests) + + +@pytest.fixture +def workspace_dir(request, board, microtvm_debug): + """Creates workspace directory for each test.""" + parent_dir = pathlib.Path(os.path.dirname(request.module.__file__)) + board_workspace = ( + parent_dir / f"workspace_{board}" / datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + ) + board_workspace_base = str(board_workspace) + number = 1 + while board_workspace.exists(): + board_workspace = pathlib.Path(board_workspace_base + f"-{number}") + number += 1 + + if not os.path.exists(board_workspace.parent): + os.makedirs(board_workspace.parent) + + keep_for_debug = microtvm_debug if microtvm_debug else None + test_temp_dir = tempdir(custom_path=board_workspace, keep_for_debug=keep_for_debug) + return test_temp_dir + + +@pytest.fixture(autouse=True) +def skip_by_board(request, board): + """Skip test if board is in the list.""" + if request.node.get_closest_marker("skip_boards"): + if board in request.node.get_closest_marker("skip_boards").args[0]: + pytest.skip("skipped on this board: {}".format(board)) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "skip_boards(board): skip test for the given board", + ) diff --git a/tests/micro/.gitignore b/tests/micro/.gitignore new file mode 100644 index 000000000000..88d8d834ed2c --- /dev/null +++ b/tests/micro/.gitignore @@ -0,0 +1,2 @@ +# test workspaces +*/workspace_*/ diff --git a/tests/micro/arduino/conftest.py b/tests/micro/arduino/conftest.py index bb4db18f7886..a5ce8127c0bb 100644 --- a/tests/micro/arduino/conftest.py +++ b/tests/micro/arduino/conftest.py @@ -15,35 +15,19 @@ # specific language governing permissions and limitations # under the License. -import pytest +pytest_plugins = [ + "tvm.micro.testing.pytest_plugin", +] -from test_utils import ARDUINO_BOARDS +import pytest def pytest_addoption(parser): - parser.addoption( - "--arduino-board", - nargs="+", - required=True, - choices=ARDUINO_BOARDS.keys(), - help="Arduino board for tests.", - ) parser.addoption( "--arduino-cli-cmd", default="arduino-cli", help="Path to `arduino-cli` command for flashing device.", ) - parser.addoption( - "--test-build-only", - action="store_true", - help="Only run tests that don't require physical hardware.", - ) - parser.addoption( - "--tvm-debug", - action="store_true", - default=False, - help="If given, enable a debug session while the test is running. Before running the test, in a separate shell, you should run: ", - ) def pytest_configure(config): @@ -52,27 +36,6 @@ def pytest_configure(config): ) -def pytest_collection_modifyitems(config, items): - if config.getoption("--test-build-only"): - skip_hardware_tests = pytest.mark.skip(reason="--test-build-only was passed") - for item in items: - if "requires_hardware" in item.keywords: - item.add_marker(skip_hardware_tests) - - -# We might do project generation differently for different boards in the future -# (to take advantage of multiple cores / external memory / etc.), so all tests -# are parameterized by board -def pytest_generate_tests(metafunc): - board = metafunc.config.getoption("arduino_board") - metafunc.parametrize("board", board, scope="session") - - @pytest.fixture(scope="session") def arduino_cli_cmd(request): return request.config.getoption("--arduino-cli-cmd") - - -@pytest.fixture(scope="session") -def tvm_debug(request): - return request.config.getoption("--tvm-debug") diff --git a/tests/micro/arduino/test_arduino_error_detection.py b/tests/micro/arduino/test_arduino_error_detection.py index 583f8283ca4c..de5e5bb56c94 100644 --- a/tests/micro/arduino/test_arduino_error_detection.py +++ b/tests/micro/arduino/test_arduino_error_detection.py @@ -23,15 +23,10 @@ import test_utils import tvm.testing -# A new project and workspace dir is created for EVERY test -@pytest.fixture -def workspace_dir(request, board): - return test_utils.make_workspace_dir("arduino_error_detection", board) - @pytest.fixture -def project(board, arduino_cli_cmd, tvm_debug, workspace_dir): - return test_utils.make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir) +def project(board, arduino_cli_cmd, microtvm_debug, workspace_dir): + return test_utils.make_kws_project(board, arduino_cli_cmd, microtvm_debug, workspace_dir) def test_blank_project_compiles(workspace_dir, project): diff --git a/tests/micro/arduino/test_arduino_rpc_server.py b/tests/micro/arduino/test_arduino_rpc_server.py index acd4186940ac..3440fde8f7e6 100644 --- a/tests/micro/arduino/test_arduino_rpc_server.py +++ b/tests/micro/arduino/test_arduino_rpc_server.py @@ -37,11 +37,6 @@ import test_utils -# # A new project and workspace dir is created for EVERY test -@pytest.fixture -def workspace_dir(board): - return test_utils.make_workspace_dir("arduino_rpc_server", board) - def _make_session(model, arduino_board, arduino_cli_cmd, workspace_dir, mod, build_config): project = tvm.micro.generate_project( @@ -84,11 +79,11 @@ def _make_add_sess(model, arduino_board, arduino_cli_cmd, workspace_dir, build_c # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_compile_runtime(board, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_compile_runtime(board, arduino_cli_cmd, microtvm_debug, workspace_dir): """Test compiling the on-device runtime.""" model = test_utils.ARDUINO_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_basic_add(sess): @@ -109,11 +104,11 @@ def test_basic_add(sess): @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_platform_timer(board, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_platform_timer(board, arduino_cli_cmd, microtvm_debug, workspace_dir): """Test compiling the on-device runtime.""" model = test_utils.ARDUINO_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_basic_add(sess): @@ -139,10 +134,10 @@ def test_basic_add(sess): @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_relay(board, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_relay(board, arduino_cli_cmd, microtvm_debug, workspace_dir): """Testing a simple relay graph""" model = test_utils.ARDUINO_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} shape = (10,) dtype = "int8" @@ -172,10 +167,10 @@ def test_relay(board, arduino_cli_cmd, tvm_debug, workspace_dir): @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_onnx(board, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_onnx(board, arduino_cli_cmd, microtvm_debug, workspace_dir): """Testing a simple ONNX model.""" model = test_utils.ARDUINO_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # Load test images. this_dir = pathlib.Path(__file__).parent @@ -263,10 +258,10 @@ def check_result( @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_byoc_microtvm(board, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_byoc_microtvm(board, arduino_cli_cmd, microtvm_debug, workspace_dir): """This is a simple test case to check BYOC capabilities of microTVM""" model = test_utils.ARDUINO_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} x = relay.var("x", shape=(10, 10)) w0 = relay.var("w0", shape=(10, 10)) @@ -347,10 +342,10 @@ def _make_add_sess_with_shape( ) @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_rpc_large_array(board, arduino_cli_cmd, tvm_debug, workspace_dir, shape): +def test_rpc_large_array(board, arduino_cli_cmd, microtvm_debug, workspace_dir, shape): """Test large RPC array transfer.""" model = test_utils.ARDUINO_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_tensors(sess): diff --git a/tests/micro/arduino/test_arduino_workflow.py b/tests/micro/arduino/test_arduino_workflow.py index d4f151845290..35bfa2556231 100644 --- a/tests/micro/arduino/test_arduino_workflow.py +++ b/tests/micro/arduino/test_arduino_workflow.py @@ -36,23 +36,24 @@ 6. Use serial connection to ensure model behaves correctly """ - # Since these tests are sequential, we'll use the same project/workspace # directory for all tests in this file @pytest.fixture(scope="module") -def workspace_dir(request, board): +def workflow_workspace_dir(request, board): return test_utils.make_workspace_dir("arduino_workflow", board) @pytest.fixture(scope="module") -def project_dir(workspace_dir): - return workspace_dir / "project" +def project_dir(workflow_workspace_dir): + return workflow_workspace_dir / "project" # We MUST pass workspace_dir, not project_dir, or the workspace will be dereferenced too soon @pytest.fixture(scope="module") -def project(board, arduino_cli_cmd, tvm_debug, workspace_dir): - return test_utils.make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir) +def project(board, arduino_cli_cmd, microtvm_debug, workflow_workspace_dir): + return test_utils.make_kws_project( + board, arduino_cli_cmd, microtvm_debug, workflow_workspace_dir + ) def _get_directory_elements(directory): diff --git a/tests/micro/arduino/test_utils.py b/tests/micro/arduino/test_utils.py index 20e7d9e75001..42ad7d40f35b 100644 --- a/tests/micro/arduino/test_utils.py +++ b/tests/micro/arduino/test_utils.py @@ -61,10 +61,10 @@ def make_workspace_dir(test_name, board): return t -def make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir): +def make_kws_project(board, arduino_cli_cmd, microtvm_debug, workspace_dir): this_dir = pathlib.Path(__file__).parent model = ARDUINO_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} mod, params = fetch_model_from_url( url="https://github.com/tensorflow/tflite-micro/raw/main/tensorflow/lite/micro/examples/micro_speech/micro_speech.tflite", diff --git a/tests/micro/common/conftest.py b/tests/micro/common/conftest.py index 10dda8774bca..0bf70ed06138 100644 --- a/tests/micro/common/conftest.py +++ b/tests/micro/common/conftest.py @@ -14,10 +14,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations -import pytest +pytest_plugins = [ + "tvm.micro.testing.pytest_plugin", +] -from ..zephyr.test_utils import ZEPHYR_BOARDS -from ..arduino.test_utils import ARDUINO_BOARDS +import pytest def pytest_addoption(parser): @@ -27,32 +28,8 @@ def pytest_addoption(parser): choices=["arduino", "zephyr"], help="Platform to run tests with", ) - parser.addoption( - "--board", - required=True, - choices=list(ARDUINO_BOARDS.keys()) + list(ZEPHYR_BOARDS.keys()), - help="microTVM boards for tests", - ) - parser.addoption( - "--test-build-only", - action="store_true", - help="Only run tests that don't require physical hardware.", - ) @pytest.fixture def platform(request): return request.config.getoption("--platform") - - -@pytest.fixture -def board(request): - return request.config.getoption("--board") - - -def pytest_collection_modifyitems(config, items): - if config.getoption("--test-build-only"): - skip_hardware_tests = pytest.mark.skip(reason="--test-build-only was passed") - for item in items: - if "requires_hardware" in item.keywords: - item.add_marker(skip_hardware_tests) diff --git a/tests/micro/common/test_autotune.py b/tests/micro/common/test_autotune.py index 1575036bf726..d62c95934aa3 100644 --- a/tests/micro/common/test_autotune.py +++ b/tests/micro/common/test_autotune.py @@ -75,7 +75,7 @@ def test_kws_autotune_workflow(platform, board, tmp_path): labels = [0, 0, 0] # Validate perforance across random runs - time, _ = tvm.micro.testing.evaluate_model_accuracy( + time, _, _ = tvm.micro.testing.evaluate_model_accuracy( session, aot_executor, samples, labels, runs_per_sample=20 ) # `time` is the average time taken to execute model inference on the diff --git a/tests/micro/zephyr/conftest.py b/tests/micro/zephyr/conftest.py index c4de48a0a47a..9f3b56769075 100644 --- a/tests/micro/zephyr/conftest.py +++ b/tests/micro/zephyr/conftest.py @@ -14,83 +14,20 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import datetime -import os -import pathlib -import pytest - -import test_utils +pytest_plugins = [ + "tvm.micro.testing.pytest_plugin", +] -from tvm.contrib.utils import tempdir +import pytest def pytest_addoption(parser): - parser.addoption( - "--zephyr-board", - required=True, - choices=test_utils.ZEPHYR_BOARDS.keys(), - help="Zephyr board for test.", - ) parser.addoption( "--west-cmd", default="west", help="Path to `west` command for flashing device." ) - parser.addoption( - "--tvm-debug", - action="store_true", - default=False, - help="If set true, enable a debug session while the test is running. Before running the test, in a separate shell, you should run: ", - ) - - -def pytest_generate_tests(metafunc): - if "board" in metafunc.fixturenames: - metafunc.parametrize("board", [metafunc.config.getoption("zephyr_board")]) -@pytest.fixture +@pytest.fixture(scope="session") def west_cmd(request): return request.config.getoption("--west-cmd") - - -@pytest.fixture -def tvm_debug(request): - return request.config.getoption("--tvm-debug") - - -@pytest.fixture -def temp_dir(board, tvm_debug): - parent_dir = pathlib.Path(os.path.dirname(__file__)) - filename = os.path.splitext(os.path.basename(__file__))[0] - board_workspace = ( - parent_dir - / f"workspace_{filename}_{board}" - / datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S") - ) - board_workspace_base = str(board_workspace) - number = 1 - while board_workspace.exists(): - board_workspace = pathlib.Path(board_workspace_base + f"-{number}") - number += 1 - - if not os.path.exists(board_workspace.parent): - os.makedirs(board_workspace.parent) - - keep_for_debug = tvm_debug if tvm_debug else None - test_temp_dir = tempdir(custom_path=board_workspace, keep_for_debug=keep_for_debug) - return test_temp_dir - - -@pytest.fixture(autouse=True) -def skip_by_board(request, board): - """Skip test if board is in the list.""" - if request.node.get_closest_marker("skip_boards"): - if board in request.node.get_closest_marker("skip_boards").args[0]: - pytest.skip("skipped on this board: {}".format(board)) - - -def pytest_configure(config): - config.addinivalue_line( - "markers", - "skip_boards(board): skip test for the given board", - ) diff --git a/tests/micro/zephyr/test_zephyr.py b/tests/micro/zephyr/test_zephyr.py index 05c8daa20c21..1f53e4baa8c3 100644 --- a/tests/micro/zephyr/test_zephyr.py +++ b/tests/micro/zephyr/test_zephyr.py @@ -89,11 +89,11 @@ def _make_add_sess(temp_dir, model, zephyr_board, west_cmd, build_config, dtype= # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_add_uint(temp_dir, board, west_cmd, tvm_debug): +def test_add_uint(workspace_dir, board, west_cmd, microtvm_debug): """Test compiling the on-device runtime.""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_basic_add(sess): @@ -108,20 +108,20 @@ def test_basic_add(sess): system_lib.get_function("add")(A_data, B_data, C_data) assert (C_data.numpy() == np.array([6, 7])).all() - with _make_add_sess(temp_dir, model, board, west_cmd, build_config) as sess: + with _make_add_sess(workspace_dir, model, board, west_cmd, build_config) as sess: test_basic_add(sess) # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_add_float(temp_dir, board, west_cmd, tvm_debug): +def test_add_float(workspace_dir, board, west_cmd, microtvm_debug): """Test compiling the on-device runtime.""" model = test_utils.ZEPHYR_BOARDS[board] if not test_utils.has_fpu(board): pytest.skip(f"FPU not enabled for {board}") - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_basic_add(sess): @@ -136,17 +136,19 @@ def test_basic_add(sess): system_lib.get_function("add")(A_data, B_data, C_data) assert (C_data.numpy() == np.array([7, 8])).all() - with _make_add_sess(temp_dir, model, board, west_cmd, build_config, dtype="float32") as sess: + with _make_add_sess( + workspace_dir, model, board, west_cmd, build_config, dtype="float32" + ) as sess: test_basic_add(sess) @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_platform_timer(temp_dir, board, west_cmd, tvm_debug): +def test_platform_timer(workspace_dir, board, west_cmd, microtvm_debug): """Test compiling the on-device runtime.""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_basic_add(sess): @@ -166,16 +168,16 @@ def test_basic_add(sess): assert result.mean > 0 assert len(result.results) == 3 - with _make_add_sess(temp_dir, model, board, west_cmd, build_config) as sess: + with _make_add_sess(workspace_dir, model, board, west_cmd, build_config) as sess: test_basic_add(sess) @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_relay(temp_dir, board, west_cmd, tvm_debug): +def test_relay(workspace_dir, board, west_cmd, microtvm_debug): """Testing a simple relay graph""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} shape = (10,) dtype = "int8" @@ -191,7 +193,7 @@ def test_relay(temp_dir, board, west_cmd, tvm_debug): with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.relay.build(ir_mod, target=target, runtime=runtime) - with _make_session(temp_dir, board, west_cmd, mod, build_config) as session: + with _make_session(workspace_dir, board, west_cmd, mod, build_config) as session: graph_mod = tvm.micro.create_local_graph_executor( mod.get_graph_json(), session.get_system_lib(), session.device ) @@ -205,10 +207,10 @@ def test_relay(temp_dir, board, west_cmd, tvm_debug): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_onnx(temp_dir, board, west_cmd, tvm_debug): +def test_onnx(workspace_dir, board, west_cmd, microtvm_debug): """Testing a simple ONNX model.""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} this_dir = pathlib.Path(os.path.dirname(__file__)) mnist_testdata = this_dir.parent / "testdata" / "mnist" @@ -237,7 +239,7 @@ def test_onnx(temp_dir, board, west_cmd, tvm_debug): lowered = relay.build(relay_mod, target, params=params, executor=executor, runtime=runtime) graph = lowered.get_graph_json() - with _make_session(temp_dir, board, west_cmd, lowered, build_config) as session: + with _make_session(workspace_dir, board, west_cmd, lowered, build_config) as session: graph_mod = tvm.micro.create_local_graph_executor( graph, session.get_system_lib(), session.device ) @@ -286,10 +288,10 @@ def check_result( @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_byoc_microtvm(temp_dir, board, west_cmd, tvm_debug): +def test_byoc_microtvm(workspace_dir, board, west_cmd, microtvm_debug): """This is a simple test case to check BYOC capabilities of microTVM""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} x = relay.var("x", shape=(10, 10)) w0 = relay.var("w0", shape=(10, 10)) w1 = relay.var("w1", shape=(10, 10)) @@ -329,7 +331,7 @@ def test_byoc_microtvm(temp_dir, board, west_cmd, tvm_debug): map_inputs = {"w{}".format(i): w_data[i] for i in range(8)} map_inputs["x"] = x_data check_result( - temp_dir=temp_dir, + temp_dir=workspace_dir, relay_mod=mod, map_inputs=map_inputs, out_shape=(30, 10), @@ -367,10 +369,10 @@ def _make_add_sess_with_shape(temp_dir, model, zephyr_board, west_cmd, shape, bu ) @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_rpc_large_array(temp_dir, board, west_cmd, tvm_debug, shape): +def test_rpc_large_array(workspace_dir, board, west_cmd, microtvm_debug, shape): """Test large RPC array transfer.""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. def test_tensors(sess): @@ -381,20 +383,22 @@ def test_tensors(sess): C_data = tvm.nd.array(np.zeros(shape, dtype="int8"), device=sess.device) assert (C_data.numpy() == np.zeros(shape)).all() - with _make_add_sess_with_shape(temp_dir, model, board, west_cmd, shape, build_config) as sess: + with _make_add_sess_with_shape( + workspace_dir, model, board, west_cmd, shape, build_config + ) as sess: test_tensors(sess) @pytest.mark.xfail(strict=False, reason="See https://github.com/apache/tvm/issues/10297") @tvm.testing.requires_micro -def test_autotune_conv2d(temp_dir, board, west_cmd, tvm_debug): +def test_autotune_conv2d(workspace_dir, board, west_cmd, microtvm_debug): """Test AutoTune for microTVM Zephyr""" if board != "qemu_x86": pytest.xfail(f"Autotune fails on {board}.") runtime = Runtime("crt", {"system-lib": True}) model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} # Create a Relay model data_shape = (1, 3, 16, 16) @@ -513,12 +517,12 @@ def test_autotune_conv2d(temp_dir, board, west_cmd, tvm_debug): @tvm.testing.requires_micro -def test_schedule_build_with_cmsis_dependency(temp_dir, board, west_cmd, tvm_debug): +def test_schedule_build_with_cmsis_dependency(workspace_dir, board, west_cmd, microtvm_debug): """Test Relay schedule with CMSIS dependency. This test shows if microTVM Auto tuning with Zephyr breaks if CMSIS dependency was required for a schedule. """ model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} target = tvm.target.target.micro(model, options=["-keys=arm_cpu,cpu"]) isa = arm_isa.IsaAnalyzer(target) @@ -555,7 +559,7 @@ def test_schedule_build_with_cmsis_dependency(temp_dir, board, west_cmd, tvm_deb "cmsis_path": os.getenv("CMSIS_PATH"), } - project_dir = temp_dir / "project" + project_dir = workspace_dir / "project" project = tvm.micro.generate_project( str(test_utils.TEMPLATE_PROJECT_DIR), mod, diff --git a/tests/micro/zephyr/test_zephyr_aot_exec.py b/tests/micro/zephyr/test_zephyr_aot_exec.py index 9bf1a7fc0608..054a06aa020b 100644 --- a/tests/micro/zephyr/test_zephyr_aot_exec.py +++ b/tests/micro/zephyr/test_zephyr_aot_exec.py @@ -38,7 +38,7 @@ import test_utils -def _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config): +def _make_session(workspace_dir, zephyr_board, west_cmd, mod, build_config): config_main_stack_size = None if test_utils.qemu_boards(zephyr_board): # fyi: qemu_riscv64 seems to be the greediest stack user @@ -59,7 +59,7 @@ def _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config): project = tvm.micro.generate_project( str(test_utils.TEMPLATE_PROJECT_DIR), mod, - temp_dir / "project", + workspace_dir / "project", project_options, ) project.build() @@ -69,11 +69,11 @@ def _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_relay(temp_dir, board, west_cmd, tvm_debug): +def test_relay(workspace_dir, board, west_cmd, microtvm_debug): """Testing a simple relay graph""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} shape = (10,) dtype = "int8" @@ -90,7 +90,7 @@ def test_relay(temp_dir, board, west_cmd, tvm_debug): with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.relay.build(ir_mod, target=target, runtime=runtime, executor=executor) - with _make_session(temp_dir, board, west_cmd, mod, build_config) as session: + with _make_session(workspace_dir, board, west_cmd, mod, build_config) as session: aot_executor = tvm.runtime.executor.aot_executor.AotModule(session.create_aot_executor()) @@ -103,11 +103,11 @@ def test_relay(temp_dir, board, west_cmd, tvm_debug): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_aot_executor(temp_dir, board, west_cmd, tvm_debug): +def test_aot_executor(workspace_dir, board, west_cmd, microtvm_debug): """Test use of the AOT executor with microTVM.""" model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} shape = (10,) dtype = "int8" @@ -154,7 +154,7 @@ def do_test(): aot_executor.set_input("b", B_np_new) assert (B_data.numpy() == B_np_new).all() - with _make_session(temp_dir, board, west_cmd, mod, build_config) as session: + with _make_session(workspace_dir, board, west_cmd, mod, build_config) as session: do_test() diff --git a/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py b/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py index 3d509f100d6e..da9d63216f15 100644 --- a/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py +++ b/tests/micro/zephyr/test_zephyr_aot_exec_standalone.py @@ -39,12 +39,12 @@ @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_tflite(temp_dir, board, west_cmd, tvm_debug): +def test_tflite(workspace_dir, board, west_cmd, microtvm_debug): """Testing a TFLite model.""" model = test_utils.ZEPHYR_BOARDS[board] input_shape = (1, 49, 10, 1) output_shape = (1, 12) - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} model_url = "https://github.com/tlc-pack/web-data/raw/25fe99fb00329a26bd37d3dca723da94316fd34c/testdata/microTVM/model/keyword_spotting_quant.tflite" model_path = download_testdata(model_url, "keyword_spotting_quant.tflite", module="model") @@ -78,7 +78,7 @@ def test_tflite(temp_dir, board, west_cmd, tvm_debug): sample = np.load(sample_path) project, _ = test_utils.generate_project( - temp_dir, + workspace_dir, board, west_cmd, lowered, @@ -95,13 +95,13 @@ def test_tflite(temp_dir, board, west_cmd, tvm_debug): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_qemu_make_fail(temp_dir, board, west_cmd, tvm_debug): +def test_qemu_make_fail(workspace_dir, board, west_cmd, microtvm_debug): """Testing QEMU make fail.""" if board not in ["qemu_x86", "mps2_an521", "mps3_an547"]: pytest.skip(msg="Only for QEMU targets.") model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} shape = (10,) dtype = "float32" @@ -120,7 +120,15 @@ def test_qemu_make_fail(temp_dir, board, west_cmd, tvm_debug): sample = np.zeros(shape=shape, dtype=dtype) project, project_dir = test_utils.generate_project( - temp_dir, board, west_cmd, lowered, build_config, sample, shape, dtype, load_cmsis=False + workspace_dir, + board, + west_cmd, + lowered, + build_config, + sample, + shape, + dtype, + load_cmsis=False, ) file_path = ( diff --git a/tests/micro/zephyr/test_zephyr_armv7m.py b/tests/micro/zephyr/test_zephyr_armv7m.py index c629403ced82..1f6a1a1bc13e 100644 --- a/tests/micro/zephyr/test_zephyr_armv7m.py +++ b/tests/micro/zephyr/test_zephyr_armv7m.py @@ -104,7 +104,7 @@ def _apply_desired_layout_no_simd(relay_mod): @tvm.testing.requires_micro @pytest.mark.skip_boards(["mps2_an521"]) -def test_armv7m_intrinsic(temp_dir, board, west_cmd, tvm_debug): +def test_armv7m_intrinsic(workspace_dir, board, west_cmd, microtvm_debug): """Testing a ARM v7m SIMD extension.""" if board not in [ @@ -118,7 +118,7 @@ def test_armv7m_intrinsic(temp_dir, board, west_cmd, tvm_debug): model = test_utils.ZEPHYR_BOARDS[board] - build_config = {"debug": tvm_debug} + build_config = {"debug": microtvm_debug} this_dir = pathlib.Path(os.path.dirname(__file__)) testdata_dir = this_dir.parent / "testdata" / "mnist" @@ -137,11 +137,11 @@ def test_armv7m_intrinsic(temp_dir, board, west_cmd, tvm_debug): executor = Executor("aot", {"unpacked-api": True, "interface-api": "c"}) runtime = Runtime("crt") - temp_dir_simd = temp_dir / "simd" - temp_dir_no_simd = temp_dir / "nosimd" + workspace_dir_simd = workspace_dir / "simd" + workspace_dir_no_simd = workspace_dir / "nosimd" - os.makedirs(temp_dir_simd, exist_ok=True) - os.makedirs(temp_dir_no_simd, exist_ok=True) + os.makedirs(workspace_dir_simd, exist_ok=True) + os.makedirs(workspace_dir_no_simd, exist_ok=True) with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): lowered_simd = relay.build( @@ -152,7 +152,7 @@ def test_armv7m_intrinsic(temp_dir, board, west_cmd, tvm_debug): ) simd_project, _ = test_utils.generate_project( - temp_dir_simd, + workspace_dir_simd, board, west_cmd, lowered_simd, @@ -165,7 +165,7 @@ def test_armv7m_intrinsic(temp_dir, board, west_cmd, tvm_debug): result_simd, time_simd = test_utils.run_model(simd_project) no_simd_project, _ = test_utils.generate_project( - temp_dir_no_simd, + workspace_dir_no_simd, board, west_cmd, lowered_no_simd, diff --git a/tests/scripts/task_python_microtvm.sh b/tests/scripts/task_python_microtvm.sh index e057883776bb..a7632a626712 100755 --- a/tests/scripts/task_python_microtvm.sh +++ b/tests/scripts/task_python_microtvm.sh @@ -24,22 +24,22 @@ source tests/scripts/setup-pytest-env.sh make cython3 # Zephyr -run_pytest ctypes python-microtvm-zephyr-qemu_x86 tests/micro/zephyr --zephyr-board=qemu_x86 -run_pytest ctypes python-microtvm-zephyr-qemu_riscv32 tests/micro/zephyr --zephyr-board=qemu_riscv32 -run_pytest ctypes python-microtvm-zephyr-qemu_riscv64 tests/micro/zephyr --zephyr-board=qemu_riscv64 -run_pytest ctypes python-microtvm-zephyr-mps2_an521 tests/micro/zephyr --zephyr-board=mps2_an521 +run_pytest ctypes python-microtvm-zephyr-qemu_x86 tests/micro/zephyr --board=qemu_x86 +run_pytest ctypes python-microtvm-zephyr-qemu_riscv32 tests/micro/zephyr --board=qemu_riscv32 +run_pytest ctypes python-microtvm-zephyr-qemu_riscv64 tests/micro/zephyr --board=qemu_riscv64 +run_pytest ctypes python-microtvm-zephyr-mps2_an521 tests/micro/zephyr --board=mps2_an521 # Arduino run_pytest ctypes python-microtvm-arduino apps/microtvm/arduino/template_project/tests -run_pytest ctypes python-microtvm-arduino-nano33ble tests/micro/arduino --test-build-only --arduino-board=nano33ble -run_pytest ctypes python-microtvm-arduino-due tests/micro/arduino --test-build-only --arduino-board=due +run_pytest ctypes python-microtvm-arduino-nano33ble tests/micro/arduino --board=nano33ble --test-build-only +run_pytest ctypes python-microtvm-arduino-due tests/micro/arduino --board=due --test-build-only # STM32 run_pytest ctypes python-microtvm-stm32 tests/micro/stm32 # Common Tests run_pytest ctypes python-microtvm-common-qemu_x86 tests/micro/common --platform=zephyr --board=qemu_x86 -run_pytest ctypes python-microtvm-common-due tests/micro/common --platform=arduino --test-build-only --board=due +run_pytest ctypes python-microtvm-common-due tests/micro/common --platform=arduino --board=due --test-build-only # Tutorials python3 gallery/how_to/work_with_microtvm/micro_tflite.py