From e2352a16ebf921de07cdfd01da654ac4ccd39441 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Mon, 6 Sep 2021 10:52:03 -0700 Subject: [PATCH 01/12] refactor zephyr --- .../template_project/microtvm_api_server.py | 12 +++++ tests/micro/zephyr/conftest.py | 47 ++++++++++-------- tests/micro/zephyr/test_zephyr.py | 48 +++++++------------ tests/micro/zephyr/test_zephyr_aot.py | 14 +++--- tests/scripts/task_python_microtvm.sh | 4 +- 5 files changed, 65 insertions(+), 60 deletions(-) diff --git a/apps/microtvm/zephyr/template_project/microtvm_api_server.py b/apps/microtvm/zephyr/template_project/microtvm_api_server.py index f267648a83f9..0ddd66f05b56 100644 --- a/apps/microtvm/zephyr/template_project/microtvm_api_server.py +++ b/apps/microtvm/zephyr/template_project/microtvm_api_server.py @@ -57,6 +57,18 @@ IS_TEMPLATE = not (API_SERVER_DIR / MODEL_LIBRARY_FORMAT_RELPATH).exists() +# Maps a short, identifying microtvm device string to (target, zephyr_board). +MICRO_DEVICES = { + "qemu_x86": ("host", "qemu_x86"), + "qemu_riscv32": ("host", "qemu_riscv32"), + "qemu_riscv64": ("host", "qemu_riscv64"), + "mps2_an521": ("mps2_an521", "mps2_an521"), + "nrf5340dk": ("nrf5340dk", "nrf5340dk_nrf5340_cpuapp"), + "stm32f746xx_disco": ("stm32f746xx", "stm32f746g_disco"), + "stm32f746xx_nucleo": ("stm32f746xx", "nucleo_f746zg"), + "stm32l4r5zi_nucleo": ("stm32l4r5zi", "nucleo_l4r5zi"), + "zynq_mp_r5": ("zynq_mp_r5", "qemu_cortex_r5"), +} def check_call(cmd_args, *args, **kwargs): cwd_str = "" if "cwd" not in kwargs else f" (in cwd: {kwargs['cwd']})" diff --git a/tests/micro/zephyr/conftest.py b/tests/micro/zephyr/conftest.py index cfdb208c92b8..8ee576f1b363 100644 --- a/tests/micro/zephyr/conftest.py +++ b/tests/micro/zephyr/conftest.py @@ -17,35 +17,40 @@ import datetime import os import pathlib +import sys import pytest import tvm.contrib.utils import tvm.target.target -# The models that should pass this configuration. Maps a short, identifying platform string to -# (model, zephyr_board). -PLATFORMS = { - "qemu_x86": ("host", "qemu_x86"), - "qemu_riscv32": ("host", "qemu_riscv32"), - "qemu_riscv64": ("host", "qemu_riscv64"), - "mps2_an521": ("mps2_an521", "mps2_an521"), - "nrf5340dk": ("nrf5340dk", "nrf5340dk_nrf5340_cpuapp"), - "stm32f746xx_disco": ("stm32f746xx", "stm32f746g_disco"), - "stm32f746xx_nucleo": ("stm32f746xx", "nucleo_f746zg"), - "stm32l4r5zi_nucleo": ("stm32l4r5zi", "nucleo_l4r5zi"), - "zynq_mp_r5": ("zynq_mp_r5", "qemu_cortex_r5"), -} +TEMPLATE_PROJECT_DIR = ( + pathlib.Path(__file__).parent + / ".." + / ".." + / ".." + / "apps" + / "microtvm" + / "zephyr" + / "template_project" +).resolve() +def zephyr_micro_devices() -> dict: + sys.path.insert(0, str(TEMPLATE_PROJECT_DIR)) + try: + import microtvm_api_server + finally: + sys.path.pop(0) + + return microtvm_api_server.MICRO_DEVICES def pytest_addoption(parser): parser.addoption( - "--microtvm-platforms", + "--microtvm-device", default="qemu_x86", - choices=PLATFORMS.keys(), + choices=zephyr_micro_devices().keys(), help=( - "Specify a comma-separated list of test models (i.e. as passed to tvm.target.micro()) " - "for microTVM tests." + "Specify a microtvm device (i.e. as passed to tvm.target.micro()) for microTVM tests." ), ) parser.addoption( @@ -60,8 +65,8 @@ def pytest_addoption(parser): def pytest_generate_tests(metafunc): - if "platform" in metafunc.fixturenames: - metafunc.parametrize("platform", metafunc.config.getoption("microtvm_platforms").split(",")) + if "device" in metafunc.fixturenames: + metafunc.parametrize("device", metafunc.config.getoption("microtvm_device")) @pytest.fixture @@ -75,8 +80,8 @@ def tvm_debug(request): @pytest.fixture -def temp_dir(platform): - _, zephyr_board = PLATFORMS[platform] +def temp_dir(device): + _, zephyr_board = zephyr_micro_devices()[device] parent_dir = pathlib.Path(os.path.dirname(__file__)) filename = os.path.splitext(os.path.basename(__file__))[0] board_workspace = ( diff --git a/tests/micro/zephyr/test_zephyr.py b/tests/micro/zephyr/test_zephyr.py index 5a7e69e3c7f9..b4309f7fdf14 100644 --- a/tests/micro/zephyr/test_zephyr.py +++ b/tests/micro/zephyr/test_zephyr.py @@ -46,7 +46,7 @@ _LOG = logging.getLogger(__name__) -PLATFORMS = conftest.PLATFORMS +MICRO_DEVICES = conftest.zephyr_micro_devices() def _make_sess_from_op( @@ -60,21 +60,9 @@ def _make_sess_from_op( return _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config) -TEMPLATE_PROJECT_DIR = ( - pathlib.Path(__file__).parent - / ".." - / ".." - / ".." - / "apps" - / "microtvm" - / "zephyr" - / "template_project" -).resolve() - - def _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config): project = tvm.micro.generate_project( - str(TEMPLATE_PROJECT_DIR), + str(conftest.TEMPLATE_PROJECT_DIR), mod, temp_dir / "project", { @@ -101,10 +89,10 @@ def _make_add_sess(temp_dir, model, zephyr_board, west_cmd, build_config, dtype= # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro -def test_add_uint(temp_dir, platform, west_cmd, tvm_debug): +def test_add_uint(temp_dir, device, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" - model, zephyr_board = PLATFORMS[platform] + model, zephyr_board = MICRO_DEVICES[device] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -125,7 +113,7 @@ def test_basic_add(sess): def has_fpu(zephyr_board): - sys.path.insert(0, str(TEMPLATE_PROJECT_DIR)) + sys.path.insert(0, str(conftest.TEMPLATE_PROJECT_DIR)) try: import microtvm_api_server finally: @@ -136,11 +124,11 @@ def has_fpu(zephyr_board): # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro -def test_add_float(temp_dir, platform, west_cmd, tvm_debug): +def test_add_float(temp_dir, device, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" - model, zephyr_board = PLATFORMS[platform] + model, zephyr_board = MICRO_DEVICES[device] if not has_fpu(zephyr_board): - pytest.skip(f"FPU not enabled for {platform}") + pytest.skip(f"FPU not enabled for {device}") build_config = {"debug": tvm_debug} @@ -164,10 +152,10 @@ def test_basic_add(sess): @tvm.testing.requires_micro -def test_platform_timer(temp_dir, platform, west_cmd, tvm_debug): +def test_platform_timer(temp_dir, device, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" - model, zephyr_board = PLATFORMS[platform] + model, zephyr_board = MICRO_DEVICES[device] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -193,9 +181,9 @@ def test_basic_add(sess): @tvm.testing.requires_micro -def test_relay(temp_dir, platform, west_cmd, tvm_debug): +def test_relay(temp_dir, device, west_cmd, tvm_debug): """Testing a simple relay graph""" - model, zephyr_board = PLATFORMS[platform] + model, zephyr_board = MICRO_DEVICES[device] build_config = {"debug": tvm_debug} shape = (10,) dtype = "int8" @@ -224,9 +212,9 @@ def test_relay(temp_dir, platform, west_cmd, tvm_debug): @tvm.testing.requires_micro -def test_onnx(temp_dir, platform, west_cmd, tvm_debug): +def test_onnx(temp_dir, device, west_cmd, tvm_debug): """Testing a simple ONNX model.""" - model, zephyr_board = PLATFORMS[platform] + model, zephyr_board = MICRO_DEVICES[device] build_config = {"debug": tvm_debug} this_dir = pathlib.Path(os.path.dirname(__file__)) @@ -301,9 +289,9 @@ def check_result( @tvm.testing.requires_micro -def test_byoc_microtvm(temp_dir, platform, west_cmd, tvm_debug): +def test_byoc_microtvm(temp_dir, device, west_cmd, tvm_debug): """This is a simple test case to check BYOC capabilities of microTVM""" - model, zephyr_board = PLATFORMS[platform] + model, zephyr_board = MICRO_DEVICES[device] build_config = {"debug": tvm_debug} x = relay.var("x", shape=(10, 10)) w0 = relay.var("w0", shape=(10, 10)) @@ -381,9 +369,9 @@ def _make_add_sess_with_shape(temp_dir, model, zephyr_board, west_cmd, shape, bu ], ) @tvm.testing.requires_micro -def test_rpc_large_array(temp_dir, platform, west_cmd, tvm_debug, shape): +def test_rpc_large_array(temp_dir, device, west_cmd, tvm_debug, shape): """Test large RPC array transfer.""" - model, zephyr_board = PLATFORMS[platform] + model, zephyr_board = MICRO_DEVICES[device] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. diff --git a/tests/micro/zephyr/test_zephyr_aot.py b/tests/micro/zephyr/test_zephyr_aot.py index 37aa0f76a852..719b5a9a0ae1 100644 --- a/tests/micro/zephyr/test_zephyr_aot.py +++ b/tests/micro/zephyr/test_zephyr_aot.py @@ -42,7 +42,7 @@ _LOG = logging.getLogger(__name__) -PLATFORMS = conftest.PLATFORMS +MICRO_DEVICES = conftest.zephyr_micro_devices() def _build_project(temp_dir, zephyr_board, west_cmd, mod, build_config, extra_files_tar=None): @@ -135,13 +135,13 @@ def _get_message(fd, expr: str, timeout_sec: int): @tvm.testing.requires_micro -def test_tflite(temp_dir, platform, west_cmd, tvm_debug): +def test_tflite(temp_dir, device, west_cmd, tvm_debug): """Testing a TFLite model.""" - if platform not in ["qemu_x86", "mps2_an521", "nrf5340dk", "stm32l4r5zi_nucleo", "zynq_mp_r5"]: + if device not in ["qemu_x86", "mps2_an521", "nrf5340dk", "stm32l4r5zi_nucleo", "zynq_mp_r5"]: pytest.skip(msg="Model does not fit.") - model, zephyr_board = PLATFORMS[platform] + model, zephyr_board = MICRO_DEVICES[device] input_shape = (1, 32, 32, 3) output_shape = (1, 10) build_config = {"debug": tvm_debug} @@ -218,12 +218,12 @@ def test_tflite(temp_dir, platform, west_cmd, tvm_debug): @tvm.testing.requires_micro -def test_qemu_make_fail(temp_dir, platform, west_cmd, tvm_debug): +def test_qemu_make_fail(temp_dir, device, west_cmd, tvm_debug): """Testing QEMU make fail.""" - if platform not in ["qemu_x86", "mps2_an521"]: + if device not in ["qemu_x86", "mps2_an521"]: pytest.skip(msg="Only for QEMU targets.") - model, zephyr_board = PLATFORMS[platform] + model, zephyr_board = MICRO_DEVICES[device] build_config = {"debug": tvm_debug} shape = (10,) dtype = "float32" diff --git a/tests/scripts/task_python_microtvm.sh b/tests/scripts/task_python_microtvm.sh index ff1b50e61d41..49ef3b461581 100755 --- a/tests/scripts/task_python_microtvm.sh +++ b/tests/scripts/task_python_microtvm.sh @@ -23,10 +23,10 @@ set -x # NOTE(areusch): Adding to diagnose flaky timeouts source tests/scripts/setup-pytest-env.sh make cython3 -run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-platforms=qemu_x86 +run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-device=qemu_x86 # Temporarily removing mps2_an512 from CI due to issue 8728: # https://github.com/apache/tvm/issues/8728 -# run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-platforms=mps2_an521 +# run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-device=mps2_an521 run_pytest ctypes python-microtvm-arduino apps/microtvm/arduino/template_project/tests run_pytest ctypes python-microtvm-arduino-nano33ble tests/micro/arduino --test-build-only --microtvm-platforms=nano33ble From ae7d111fe63e3d69ef338d6775251d0dc8d4304d Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Mon, 6 Sep 2021 11:02:56 -0700 Subject: [PATCH 02/12] refactor arduino --- .../template_project/microtvm_api_server.py | 12 ++++++ tests/micro/arduino/conftest.py | 41 ++++++++----------- .../arduino/test_arduino_error_detection.py | 8 ++-- .../micro/arduino/test_arduino_rpc_server.py | 29 ++++++------- tests/micro/arduino/test_arduino_workflow.py | 10 ++--- tests/scripts/task_python_microtvm.sh | 4 +- 6 files changed, 56 insertions(+), 48 deletions(-) diff --git a/apps/microtvm/arduino/template_project/microtvm_api_server.py b/apps/microtvm/arduino/template_project/microtvm_api_server.py index 38fc73c5e53a..e88f81f4dd80 100644 --- a/apps/microtvm/arduino/template_project/microtvm_api_server.py +++ b/apps/microtvm/arduino/template_project/microtvm_api_server.py @@ -43,6 +43,18 @@ IS_TEMPLATE = not (API_SERVER_DIR / MODEL_LIBRARY_FORMAT_RELPATH).exists() +# Maps a short, identifying microtvm device string to (target, arduino_board). +MICRO_DEVICES = { + "due": ("sam3x8e", "due"), + "feathers2": ("esp32", "feathers2"), + "metrom4": ("atsamd51", "metrom4"), + "nano33ble": ("nrf52840", "nano33ble"), + "pybadge": ("atsamd51", "pybadge"), + "spresense": ("cxd5602gg", "spresense"), + "teensy40": ("imxrt1060", "teensy40"), + "teensy41": ("imxrt1060", "teensy41"), + "wioterminal": ("atsamd51", "wioterminal"), +} class BoardAutodetectFailed(Exception): """Raised when no attached hardware is found matching the requested board""" diff --git a/tests/micro/arduino/conftest.py b/tests/micro/arduino/conftest.py index 38870b6b4dfe..fabe3fdd157c 100644 --- a/tests/micro/arduino/conftest.py +++ b/tests/micro/arduino/conftest.py @@ -22,20 +22,6 @@ import tvm.target.target from tvm import micro, relay -# The models that should pass this configuration. Maps a short, identifying platform string to -# (model, zephyr_board). -PLATFORMS = { - "due": ("sam3x8e", "due"), - "feathers2": ("esp32", "feathers2"), - "metrom4": ("atsamd51", "metrom4"), - "nano33ble": ("nrf52840", "nano33ble"), - "pybadge": ("atsamd51", "pybadge"), - "spresense": ("cxd5602gg", "spresense"), - "teensy40": ("imxrt1060", "teensy40"), - "teensy41": ("imxrt1060", "teensy41"), - "wioterminal": ("atsamd51", "wioterminal"), -} - TEMPLATE_PROJECT_DIR = ( pathlib.Path(__file__).parent / ".." @@ -47,14 +33,23 @@ / "template_project" ).resolve() +def arduino_micro_devices() -> dict: + sys.path.insert(0, str(TEMPLATE_PROJECT_DIR)) + try: + import microtvm_api_server + finally: + sys.path.pop(0) + + return microtvm_api_server.MICRO_DEVICES + def pytest_addoption(parser): parser.addoption( - "--microtvm-platforms", + "--microtvm-device", nargs="+", required=True, - choices=PLATFORMS.keys(), - help="Target platforms for microTVM tests.", + choices=arduino_micro_devices().keys(), + help="MicroTVM device for tests.", ) parser.addoption( "--arduino-cli-cmd", @@ -92,8 +87,8 @@ def pytest_collection_modifyitems(config, items): # (to take advantage of multiple cores / external memory / etc.), so all tests # are parameterized by board def pytest_generate_tests(metafunc): - platforms = metafunc.config.getoption("microtvm_platforms") - metafunc.parametrize("platform", platforms, scope="session") + device = metafunc.config.getoption("microtvm_device") + metafunc.parametrize("device", device, scope="session") @pytest.fixture(scope="session") @@ -106,8 +101,8 @@ def tvm_debug(request): return request.config.getoption("--tvm-debug") -def make_workspace_dir(test_name, platform): - _, arduino_board = PLATFORMS[platform] +def make_workspace_dir(test_name, device): + _, arduino_board = arduino_micro_devices()[device] filepath = pathlib.Path(__file__) board_workspace = ( filepath.parent @@ -125,9 +120,9 @@ def make_workspace_dir(test_name, platform): return t -def make_kws_project(platform, arduino_cli_cmd, tvm_debug, workspace_dir): +def make_kws_project(device, arduino_cli_cmd, tvm_debug, workspace_dir): this_dir = pathlib.Path(__file__).parent - model, arduino_board = PLATFORMS[platform] + model, arduino_board = arduino_micro_devices()[device] build_config = {"debug": tvm_debug} with open(this_dir.parent / "testdata" / "kws" / "yes_no.tflite", "rb") as f: diff --git a/tests/micro/arduino/test_arduino_error_detection.py b/tests/micro/arduino/test_arduino_error_detection.py index 1789fff2e7bb..cdccf91a92e1 100644 --- a/tests/micro/arduino/test_arduino_error_detection.py +++ b/tests/micro/arduino/test_arduino_error_detection.py @@ -27,13 +27,13 @@ # A new project and workspace dir is created for EVERY test @pytest.fixture -def workspace_dir(request, platform): - return conftest.make_workspace_dir("arduino_error_detection", platform) +def workspace_dir(request, device): + return conftest.make_workspace_dir("arduino_error_detection", device) @pytest.fixture -def project(platform, arduino_cli_cmd, tvm_debug, workspace_dir): - return conftest.make_kws_project(platform, arduino_cli_cmd, tvm_debug, workspace_dir) +def project(device, arduino_cli_cmd, tvm_debug, workspace_dir): + return conftest.make_kws_project(device, arduino_cli_cmd, tvm_debug, workspace_dir) def test_blank_project_compiles(workspace_dir, project): diff --git a/tests/micro/arduino/test_arduino_rpc_server.py b/tests/micro/arduino/test_arduino_rpc_server.py index 57ebbc605197..154bda62e24e 100644 --- a/tests/micro/arduino/test_arduino_rpc_server.py +++ b/tests/micro/arduino/test_arduino_rpc_server.py @@ -36,11 +36,12 @@ import conftest +MICRO_DEVICES = conftest.arduino_micro_devices() # # A new project and workspace dir is created for EVERY test @pytest.fixture -def workspace_dir(platform): - return conftest.make_workspace_dir("arduino_rpc_server", platform) +def workspace_dir(device): + return conftest.make_workspace_dir("arduino_rpc_server", device) def _make_session(model, arduino_board, arduino_cli_cmd, workspace_dir, mod, build_config): @@ -83,10 +84,10 @@ def _make_add_sess(model, arduino_board, arduino_cli_cmd, workspace_dir, build_c # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_compile_runtime(platform, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_compile_runtime(device, arduino_cli_cmd, tvm_debug, workspace_dir): """Test compiling the on-device runtime.""" - model, arduino_board = conftest.PLATFORMS[platform] + model, arduino_board = MICRO_DEVICES[device] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -108,10 +109,10 @@ def test_basic_add(sess): @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_platform_timer(platform, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_platform_timer(device, arduino_cli_cmd, tvm_debug, workspace_dir): """Test compiling the on-device runtime.""" - model, arduino_board = conftest.PLATFORMS[platform] + model, arduino_board = MICRO_DEVICES[device] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -138,9 +139,9 @@ def test_basic_add(sess): @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_relay(platform, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_relay(device, arduino_cli_cmd, tvm_debug, workspace_dir): """Testing a simple relay graph""" - model, arduino_board = conftest.PLATFORMS[platform] + model, arduino_board = MICRO_DEVICES[device] build_config = {"debug": tvm_debug} shape = (10,) @@ -172,9 +173,9 @@ def test_relay(platform, arduino_cli_cmd, tvm_debug, workspace_dir): @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_onnx(platform, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_onnx(device, arduino_cli_cmd, tvm_debug, workspace_dir): """Testing a simple ONNX model.""" - model, arduino_board = conftest.PLATFORMS[platform] + model, arduino_board = MICRO_DEVICES[device] build_config = {"debug": tvm_debug} # Load test images. @@ -260,9 +261,9 @@ def check_result( @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_byoc_microtvm(platform, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_byoc_microtvm(device, arduino_cli_cmd, tvm_debug, workspace_dir): """This is a simple test case to check BYOC capabilities of microTVM""" - model, arduino_board = conftest.PLATFORMS[platform] + model, arduino_board = MICRO_DEVICES[device] build_config = {"debug": tvm_debug} x = relay.var("x", shape=(10, 10)) @@ -344,9 +345,9 @@ def _make_add_sess_with_shape( ) @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_rpc_large_array(platform, arduino_cli_cmd, tvm_debug, workspace_dir, shape): +def test_rpc_large_array(device, arduino_cli_cmd, tvm_debug, workspace_dir, shape): """Test large RPC array transfer.""" - model, arduino_board = conftest.PLATFORMS[platform] + model, arduino_board = MICRO_DEVICES[device] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. diff --git a/tests/micro/arduino/test_arduino_workflow.py b/tests/micro/arduino/test_arduino_workflow.py index cd97aa37b5a4..2663704afc7b 100644 --- a/tests/micro/arduino/test_arduino_workflow.py +++ b/tests/micro/arduino/test_arduino_workflow.py @@ -29,7 +29,7 @@ This unit test simulates a simple user workflow, where we: 1. Generate a base sketch using a simple audio model 2. Modify the .ino file, much like a user would -3. Compile the sketch for the target platform +3. Compile the sketch for the target device -- If physical hardware is present -- 4. Upload the sketch to a connected board 5. Open a serial connection to the board @@ -40,8 +40,8 @@ # Since these tests are sequential, we'll use the same project/workspace # directory for all tests in this file @pytest.fixture(scope="module") -def workspace_dir(request, platform): - return conftest.make_workspace_dir("arduino_workflow", platform) +def workspace_dir(request, device): + return conftest.make_workspace_dir("arduino_workflow", device) @pytest.fixture(scope="module") @@ -51,8 +51,8 @@ def project_dir(workspace_dir): # We MUST pass workspace_dir, not project_dir, or the workspace will be dereferenced too soon @pytest.fixture(scope="module") -def project(platform, arduino_cli_cmd, tvm_debug, workspace_dir): - return conftest.make_kws_project(platform, arduino_cli_cmd, tvm_debug, workspace_dir) +def project(device, arduino_cli_cmd, tvm_debug, workspace_dir): + return conftest.make_kws_project(device, arduino_cli_cmd, tvm_debug, workspace_dir) def _get_directory_elements(directory): diff --git a/tests/scripts/task_python_microtvm.sh b/tests/scripts/task_python_microtvm.sh index 49ef3b461581..48921201b6d2 100755 --- a/tests/scripts/task_python_microtvm.sh +++ b/tests/scripts/task_python_microtvm.sh @@ -29,5 +29,5 @@ run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-device=qe # run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-device=mps2_an521 run_pytest ctypes python-microtvm-arduino apps/microtvm/arduino/template_project/tests -run_pytest ctypes python-microtvm-arduino-nano33ble tests/micro/arduino --test-build-only --microtvm-platforms=nano33ble -run_pytest ctypes python-microtvm-arduino-due tests/micro/arduino --test-build-only --microtvm-platforms=due +run_pytest ctypes python-microtvm-arduino-nano33ble tests/micro/arduino --test-build-only --microtvm-device=nano33ble +run_pytest ctypes python-microtvm-arduino-due tests/micro/arduino --test-build-only --microtvm-device=due From 240eba6d3a7d64081dce4c34280ac5789fa62f66 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Mon, 6 Sep 2021 11:28:54 -0700 Subject: [PATCH 03/12] fix RVM build scripts --- apps/microtvm/reference-vm/README.md | 8 +-- .../arduino/base-box/base_box_test.sh | 14 ++-- apps/microtvm/reference-vm/base-box-tool.py | 69 +++++++++---------- .../zephyr/base-box/base_box_test.sh | 14 ++-- 4 files changed, 51 insertions(+), 54 deletions(-) diff --git a/apps/microtvm/reference-vm/README.md b/apps/microtvm/reference-vm/README.md index 9303c0a64ece..37d3cc94d484 100644 --- a/apps/microtvm/reference-vm/README.md +++ b/apps/microtvm/reference-vm/README.md @@ -72,20 +72,20 @@ For example: $ ./base-box-tool.py --provider virtualbox build zephyr ``` -2. **Run** release tests for each platform: +2. **Run** release tests for each microTVM device: A. Connect any needed hardware to the VM host machine; B. Run tests: ```bash - $ ./base-box-tool.py [--provider=PROVIDER] test --microtvm-platform=MICROTVM_PLATFORM [--test-device-serial=SERIAL] PLATFORM + $ ./base-box-tool.py [--provider=PROVIDER] test --microtvm-device=MICROTVM_DEVICE [--test-device-serial=SERIAL] PLATFORM ``` - where MICROTVM_PLATFORM is one of the options listed in the + where MICROTVM_DEVICE is one of the options listed in the PLATFORM/base-box/test-config.json file. For example: ```base - $ ./base-box-tool.py --provider virtualbox test --microtvm-platform=stm32f746xx_disco zephyr + $ ./base-box-tool.py --provider virtualbox test --microtvm-device=stm32f746xx_disco zephyr ``` This command does the following for the specified provider: diff --git a/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh b/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh index 3d8597f19b64..aed3bab60d2d 100755 --- a/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh +++ b/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. # -# Usage: base_box_test.sh +# Usage: base_box_test.sh # Execute microTVM Arduino tests. # @@ -24,17 +24,17 @@ set -e set -x if [ "$#" -lt 1 ]; then - echo "Usage: base_box_test.sh " + echo "Usage: base_box_test.sh " exit -1 fi -microtvm_platform=$1 +microtvm_device=$1 -pytest tests/micro/arduino/test_arduino_workflow.py --microtvm-platforms=${microtvm_platform} +pytest tests/micro/arduino/test_arduino_workflow.py --microtvm-device=${microtvm_device} -if [ $microtvm_platform == "nano33ble" ]; then +if [ $microtvm_device == "nano33ble" ]; then # https://github.com/apache/tvm/issues/8730 - echo "NOTE: skipped test_arduino_rpc_server.py on $microtvm_platform -- known failure" + echo "NOTE: skipped test_arduino_rpc_server.py on $microtvm_device -- known failure" else - pytest tests/micro/arduino/test_arduino_rpc_server.py --microtvm-platforms=${microtvm_platform} + pytest tests/micro/arduino/test_arduino_rpc_server.py --microtvm-device=${microtvm_device} fi diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py index f32885433c2b..3b9c59216183 100755 --- a/apps/microtvm/reference-vm/base-box-tool.py +++ b/apps/microtvm/reference-vm/base-box-tool.py @@ -27,7 +27,7 @@ import shutil import subprocess import sys - +import pathlib _LOG = logging.getLogger(__name__) @@ -48,29 +48,6 @@ "zephyr", ) -# List of identifying strings for microTVM platforms for testing. -# Must match PLATFORMS as defined in tvm/tests/micro/[platform]/conftest.py -# TODO add a way to declare supported platforms to ProjectAPI -ALL_MICROTVM_PLATFORMS = { - "arduino": ( - "due", - "feathers2", - "metrom4", - "nano33ble", - "pybadge", - "spresense", - "teensy40", - "teensy41", - "wioterminal", - ), - "zephyr": ( - "stm32f746xx_nucleo", - "stm32f746xx_disco", - "nrf5340dk", - "mps2_an521", - ), -} - # Extra scripts required to execute on provisioning # in [platform]/base-box/base_box_provision.sh EXTRA_SCRIPTS = { @@ -80,6 +57,26 @@ PACKER_FILE_NAME = "packer.json" +def get_micro_devices(platform: str) -> dict: + template_project = (pathlib.Path(__file__).parent + / platform + / "template_project" + ).resolve() + + sys.path.insert(0, str(arduino_template_project)) + try: + import microtvm_api_server + finally: + sys.path.pop(0) + + return microtvm_api_server.MICRO_DEVICES + +# List of identifying strings for microTVM devices for testing. +# TODO add a way to declare supported platforms to ProjectAPI +ALL_MICROTVM_DEVICES = { + "arduino": get_micro_devices("arduino").keys(), + "zephyr": get_micro_devices("zephyr").keys(), +} def parse_virtualbox_devices(): output = subprocess.check_output(["VBoxManage", "list", "usbhost"], encoding="utf-8") @@ -362,7 +359,7 @@ def _quote_cmd(cmd): + _quote_cmd( [ f"apps/microtvm/reference-vm/{platform}/base-box/base_box_test.sh", - test_config["microtvm_platform"], + test_config["microtvm_device"], ] ) ) @@ -376,17 +373,17 @@ def test_command(args): with open(test_config_file) as f: test_config = json.load(f) - # select microTVM test platform - microtvm_test_platform = test_config[args.microtvm_platform] + # select microTVM test device + microtvm_test_device = test_config[args.microtvm_device] for key, expected_type in REQUIRED_TEST_CONFIG_KEYS.items(): - assert key in microtvm_test_platform and isinstance( - microtvm_test_platform[key], expected_type + assert key in microtvm_test_device and isinstance( + microtvm_test_device[key], expected_type ), f"Expected key {key} of type {expected_type} in {test_config_file}: {test_config!r}" - microtvm_test_platform["vid_hex"] = microtvm_test_platform["vid_hex"].lower() - microtvm_test_platform["pid_hex"] = microtvm_test_platform["pid_hex"].lower() - microtvm_test_platform["microtvm_platform"] = args.microtvm_platform + microtvm_test_device["vid_hex"] = microtvm_test_device["vid_hex"].lower() + microtvm_test_device["pid_hex"] = microtvm_test_device["pid_hex"].lower() + microtvm_test_device["microtvm_device"] = args.microtvm_device providers = args.provider provider_passed = {p: False for p in providers} @@ -406,7 +403,7 @@ def test_command(args): release_test_dir, args.platform, provider_name, - microtvm_test_platform, + microtvm_test_device, args.test_device_serial, ) provider_passed[provider_name] = True @@ -511,10 +508,10 @@ def parse_args(): platform_specific_parser = parser_test_platform_subparsers.add_parser(platform) platform_specific_parser.set_defaults(platform=platform) platform_specific_parser.add_argument( - "--microtvm-platform", - choices=ALL_MICROTVM_PLATFORMS[platform], + "--microtvm-device", + choices=ALL_MICROTVM_DEVICES[platform], required=True, - help="MicroTVM platfrom used for testing.", + help="MicroTVM device used for testing.", ) # Options for release subcommand diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh index 8eba63e9e331..c9587ea3a86d 100755 --- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh +++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. # -# Usage: base_box_test.sh +# Usage: base_box_test.sh # Execute microTVM Zephyr tests. # @@ -24,16 +24,16 @@ set -e set -x if [ "$#" -lt 1 ]; then - echo "Usage: base_box_test.sh " + echo "Usage: base_box_test.sh " exit -1 fi -microtvm_platform=$1 +microtvm_device=$1 -pytest tests/micro/zephyr/test_zephyr.py --microtvm-platforms=${microtvm_platform} +pytest tests/micro/zephyr/test_zephyr.py --microtvm-device=${microtvm_device} -if [ $microtvm_platform == "stm32f746xx" ]; then - echo "NOTE: skipped test_zephyr_aot.py on $microtvm_platform -- known failure" +if [ $microtvm_device == "stm32f746xx" ]; then + echo "NOTE: skipped test_zephyr_aot.py on $microtvm_device -- known failure" else - pytest tests/micro/zephyr/test_zephyr_aot.py --microtvm-platforms=${microtvm_platform} + pytest tests/micro/zephyr/test_zephyr_aot.py --microtvm-device=${microtvm_device} fi From fecdae674c16eb360ec1190dfa1eba69dbea12f3 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Mon, 6 Sep 2021 11:29:26 -0700 Subject: [PATCH 04/12] test and tutorial --- tests/micro/arduino/README.md | 4 ++-- tests/micro/zephyr/README.md | 6 +++--- tutorials/micro/micro_reference_vm.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/micro/arduino/README.md b/tests/micro/arduino/README.md index 78e63cabb7e2..bcad615593c1 100644 --- a/tests/micro/arduino/README.md +++ b/tests/micro/arduino/README.md @@ -22,14 +22,14 @@ all of the appropriate TVM dependencies installed. You can run the test with: ``` $ cd tvm/tests/micro/arduino -$ pytest --microtvm-platforms spresense +$ pytest --microtvm-device=spresense ``` Most of these tests require a supported Arduino board to be connected. If you don't want to run these tests, you can pass the flag `--test-build-only` to only test project generation and compilation. -To see the list of supported values for `----microtvm-platforms`, run: +To see the list of supported values for `----microtvm-device`, run: ``` $ pytest --help ``` diff --git a/tests/micro/zephyr/README.md b/tests/micro/zephyr/README.md index 9769cae2b53b..0293d2276c5c 100644 --- a/tests/micro/zephyr/README.md +++ b/tests/micro/zephyr/README.md @@ -32,11 +32,11 @@ device) using: ``` $ cd tvm/tests/micro/zephyr -$ pytest test_zephyr.py --microtvm-platforms=host # For QEMU emulation -$ pytest test_zephyr.py --microtvm-platforms=nrf5340dk # For nRF5340DK +$ pytest test_zephyr.py --microtvm-device=qemu_x86 # For QEMU emulation +$ pytest test_zephyr.py --microtvm-device=nrf5340dk # For nRF5340DK ``` -To see the list of supported values for `--microtvm-platforms`, run: +To see the list of supported values for `--microtvm-device`, run: ``` $ pytest test_zephyr.py --help ``` diff --git a/tutorials/micro/micro_reference_vm.py b/tutorials/micro/micro_reference_vm.py index bb262893eb6b..feb746c57dcb 100644 --- a/tutorials/micro/micro_reference_vm.py +++ b/tutorials/micro/micro_reference_vm.py @@ -143,7 +143,7 @@ .. code-block:: bash $ cd apps/microtvm/reference-vm/zephyr - $ poetry run python3 ../../../../tests/micro/qemu/test_zephyr.py --microtvm-platforms=stm32f746xx + $ poetry run python3 ../../../../tests/micro/qemu/test_zephyr.py --microtvm-device=stm32f746xx If you do not have physical hardware attached, but wish to run the tests using the local QEMU emulator running within the VM, run the following commands instead: @@ -152,7 +152,7 @@ $ cd /Users/yourusername/path/to/tvm $ cd apps/microtvm/reference-vm/zephyr/ - $ poetry run pytest ../../../../tests/micro/qemu/test_zephyr.py --microtvm-platforms=host + $ poetry run pytest ../../../../tests/micro/qemu/test_zephyr.py --microtvm-device=qemu_x86 From 71fd5d37227c0b81b455926a1d28b332ec28f495 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Mon, 6 Sep 2021 11:49:40 -0700 Subject: [PATCH 05/12] fix release directory --- apps/microtvm/reference-vm/base-box-tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py index 3b9c59216183..e59f7060d524 100755 --- a/apps/microtvm/reference-vm/base-box-tool.py +++ b/apps/microtvm/reference-vm/base-box-tool.py @@ -388,7 +388,7 @@ def test_command(args): providers = args.provider provider_passed = {p: False for p in providers} - release_test_dir = os.path.join(THIS_DIR, "release-test") + release_test_dir = os.path.join(THIS_DIR, f"release-test-{args.platform}") if args.skip_build: assert len(providers) == 1, "--skip-build was given, but >1 provider specified" From 0e72c7c7340494e985ff426c7698e1fab84a3b45 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Mon, 6 Sep 2021 12:12:49 -0700 Subject: [PATCH 06/12] fix test --- tests/micro/zephyr/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/micro/zephyr/conftest.py b/tests/micro/zephyr/conftest.py index 8ee576f1b363..97bbdebf8f1c 100644 --- a/tests/micro/zephyr/conftest.py +++ b/tests/micro/zephyr/conftest.py @@ -66,7 +66,7 @@ def pytest_addoption(parser): def pytest_generate_tests(metafunc): if "device" in metafunc.fixturenames: - metafunc.parametrize("device", metafunc.config.getoption("microtvm_device")) + metafunc.parametrize("device", [metafunc.config.getoption("microtvm_device")]) @pytest.fixture From 8a01d8ba260709aa2d617a4e2d07ee526ba688fc Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Mon, 6 Sep 2021 12:18:15 -0700 Subject: [PATCH 07/12] lint --- .../arduino/template_project/microtvm_api_server.py | 1 + apps/microtvm/reference-vm/base-box-tool.py | 8 ++++---- .../zephyr/template_project/microtvm_api_server.py | 1 + tests/micro/arduino/conftest.py | 2 ++ tests/micro/zephyr/conftest.py | 2 ++ 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/microtvm/arduino/template_project/microtvm_api_server.py b/apps/microtvm/arduino/template_project/microtvm_api_server.py index e88f81f4dd80..960ebe8ff2e3 100644 --- a/apps/microtvm/arduino/template_project/microtvm_api_server.py +++ b/apps/microtvm/arduino/template_project/microtvm_api_server.py @@ -56,6 +56,7 @@ "wioterminal": ("atsamd51", "wioterminal"), } + class BoardAutodetectFailed(Exception): """Raised when no attached hardware is found matching the requested board""" diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py index e59f7060d524..20abe80e3e4e 100755 --- a/apps/microtvm/reference-vm/base-box-tool.py +++ b/apps/microtvm/reference-vm/base-box-tool.py @@ -57,11 +57,9 @@ PACKER_FILE_NAME = "packer.json" + def get_micro_devices(platform: str) -> dict: - template_project = (pathlib.Path(__file__).parent - / platform - / "template_project" - ).resolve() + template_project = (pathlib.Path(__file__).parent / platform / "template_project").resolve() sys.path.insert(0, str(arduino_template_project)) try: @@ -71,6 +69,7 @@ def get_micro_devices(platform: str) -> dict: return microtvm_api_server.MICRO_DEVICES + # List of identifying strings for microTVM devices for testing. # TODO add a way to declare supported platforms to ProjectAPI ALL_MICROTVM_DEVICES = { @@ -78,6 +77,7 @@ def get_micro_devices(platform: str) -> dict: "zephyr": get_micro_devices("zephyr").keys(), } + def parse_virtualbox_devices(): output = subprocess.check_output(["VBoxManage", "list", "usbhost"], encoding="utf-8") devices = [] diff --git a/apps/microtvm/zephyr/template_project/microtvm_api_server.py b/apps/microtvm/zephyr/template_project/microtvm_api_server.py index 0ddd66f05b56..b3d9a0a6b87c 100644 --- a/apps/microtvm/zephyr/template_project/microtvm_api_server.py +++ b/apps/microtvm/zephyr/template_project/microtvm_api_server.py @@ -70,6 +70,7 @@ "zynq_mp_r5": ("zynq_mp_r5", "qemu_cortex_r5"), } + def check_call(cmd_args, *args, **kwargs): cwd_str = "" if "cwd" not in kwargs else f" (in cwd: {kwargs['cwd']})" _LOG.info("run%s: %s", cwd_str, " ".join(shlex.quote(a) for a in cmd_args)) diff --git a/tests/micro/arduino/conftest.py b/tests/micro/arduino/conftest.py index fabe3fdd157c..cf937812b426 100644 --- a/tests/micro/arduino/conftest.py +++ b/tests/micro/arduino/conftest.py @@ -17,6 +17,7 @@ import datetime import pathlib +import sys import pytest import tvm.target.target @@ -33,6 +34,7 @@ / "template_project" ).resolve() + def arduino_micro_devices() -> dict: sys.path.insert(0, str(TEMPLATE_PROJECT_DIR)) try: diff --git a/tests/micro/zephyr/conftest.py b/tests/micro/zephyr/conftest.py index 97bbdebf8f1c..3326b1a32335 100644 --- a/tests/micro/zephyr/conftest.py +++ b/tests/micro/zephyr/conftest.py @@ -35,6 +35,7 @@ / "template_project" ).resolve() + def zephyr_micro_devices() -> dict: sys.path.insert(0, str(TEMPLATE_PROJECT_DIR)) try: @@ -44,6 +45,7 @@ def zephyr_micro_devices() -> dict: return microtvm_api_server.MICRO_DEVICES + def pytest_addoption(parser): parser.addoption( "--microtvm-device", From c1ed48177721caf5f2e1c312dcd5f580627f2b7b Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Mon, 6 Sep 2021 12:28:16 -0700 Subject: [PATCH 08/12] revert api server import --- apps/microtvm/reference-vm/base-box-tool.py | 34 +++++++++++---------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py index 20abe80e3e4e..1951f40ccace 100755 --- a/apps/microtvm/reference-vm/base-box-tool.py +++ b/apps/microtvm/reference-vm/base-box-tool.py @@ -27,7 +27,6 @@ import shutil import subprocess import sys -import pathlib _LOG = logging.getLogger(__name__) @@ -58,23 +57,26 @@ PACKER_FILE_NAME = "packer.json" -def get_micro_devices(platform: str) -> dict: - template_project = (pathlib.Path(__file__).parent / platform / "template_project").resolve() - - sys.path.insert(0, str(arduino_template_project)) - try: - import microtvm_api_server - finally: - sys.path.pop(0) - - return microtvm_api_server.MICRO_DEVICES - - # List of identifying strings for microTVM devices for testing. -# TODO add a way to declare supported platforms to ProjectAPI +# TODO add a way to declare supported devices to ProjectAPI ALL_MICROTVM_DEVICES = { - "arduino": get_micro_devices("arduino").keys(), - "zephyr": get_micro_devices("zephyr").keys(), + "arduino": ( + "due", + "feathers2", + "metrom4", + "nano33ble", + "pybadge", + "spresense", + "teensy40", + "teensy41", + "wioterminal", + ), + "zephyr": ( + "stm32f746xx_nucleo", + "stm32f746xx_disco", + "nrf5340dk", + "mps2_an521", + ), } From 2968f131953310434a7cc9fcb6d26a8f0752b5de Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Wed, 8 Sep 2021 11:00:28 -0700 Subject: [PATCH 09/12] fix arduino --- .../template_project/microtvm_api_server.py | 28 ++++++------- tests/micro/arduino/conftest.py | 39 +++++++++-------- .../arduino/test_arduino_error_detection.py | 8 ++-- .../micro/arduino/test_arduino_rpc_server.py | 42 +++++++++---------- tests/micro/arduino/test_arduino_workflow.py | 8 ++-- 5 files changed, 64 insertions(+), 61 deletions(-) diff --git a/apps/microtvm/arduino/template_project/microtvm_api_server.py b/apps/microtvm/arduino/template_project/microtvm_api_server.py index 960ebe8ff2e3..be134ade0b7c 100644 --- a/apps/microtvm/arduino/template_project/microtvm_api_server.py +++ b/apps/microtvm/arduino/template_project/microtvm_api_server.py @@ -43,20 +43,6 @@ IS_TEMPLATE = not (API_SERVER_DIR / MODEL_LIBRARY_FORMAT_RELPATH).exists() -# Maps a short, identifying microtvm device string to (target, arduino_board). -MICRO_DEVICES = { - "due": ("sam3x8e", "due"), - "feathers2": ("esp32", "feathers2"), - "metrom4": ("atsamd51", "metrom4"), - "nano33ble": ("nrf52840", "nano33ble"), - "pybadge": ("atsamd51", "pybadge"), - "spresense": ("cxd5602gg", "spresense"), - "teensy40": ("imxrt1060", "teensy40"), - "teensy41": ("imxrt1060", "teensy41"), - "wioterminal": ("atsamd51", "wioterminal"), -} - - class BoardAutodetectFailed(Exception): """Raised when no attached hardware is found matching the requested board""" @@ -70,6 +56,7 @@ class BoardAutodetectFailed(Exception): "package": "arduino", "architecture": "sam", "board": "arduino_due_x_dbg", + "target": "sam3x8e", }, # Due to the way the Feather S2 bootloader works, compilation # behaves fine but uploads cannot be done automatically @@ -77,27 +64,32 @@ class BoardAutodetectFailed(Exception): "package": "esp32", "architecture": "esp32", "board": "feathers2", + "target": "esp32", }, "metrom4": { "package": "adafruit", "architecture": "samd", "board": "adafruit_metro_m4", + "target": "atsamd51" }, # Spresense only works as of its v2.3.0 sdk "spresense": { "package": "SPRESENSE", "architecture": "spresense", "board": "spresense", + "target": "cxd5602gg", }, "nano33ble": { "package": "arduino", "architecture": "mbed_nano", "board": "nano33ble", + "target": "nrf52840", }, "pybadge": { "package": "adafruit", "architecture": "samd", "board": "adafruit_pybadge_m4", + "target": "atsamd51", }, # The Teensy boards are listed here for completeness, but they # won't work until https://github.com/arduino/arduino-cli/issues/700 @@ -106,16 +98,19 @@ class BoardAutodetectFailed(Exception): "package": "teensy", "architecture": "avr", "board": "teensy40", + "target": "imxrt1060", }, "teensy41": { "package": "teensy", "architecture": "avr", "board": "teensy41", + "target": "imxrt1060", }, "wioterminal": { "package": "Seeeduino", "architecture": "samd", "board": "seeed_wio_terminal", + "target": "atsamd51", }, } @@ -127,6 +122,11 @@ class BoardAutodetectFailed(Exception): choices=list(BOARD_PROPERTIES), help="Name of the Arduino board to build for", ), + server.ProjectOption( + "arduino_target", + choices=[board["target"] for _,board in BOARD_PROPERTIES.items()], + help="Name of the target for each Arduino board.", + ), server.ProjectOption("arduino_cli_cmd", help="Path to the arduino-cli tool."), server.ProjectOption("port", help="Port to use for connecting to hardware"), server.ProjectOption( diff --git a/tests/micro/arduino/conftest.py b/tests/micro/arduino/conftest.py index cf937812b426..e1d8bc0a4f98 100644 --- a/tests/micro/arduino/conftest.py +++ b/tests/micro/arduino/conftest.py @@ -21,6 +21,7 @@ import pytest import tvm.target.target +from tvm.micro import project from tvm import micro, relay TEMPLATE_PROJECT_DIR = ( @@ -35,22 +36,27 @@ ).resolve() -def arduino_micro_devices() -> dict: - sys.path.insert(0, str(TEMPLATE_PROJECT_DIR)) - try: - import microtvm_api_server - finally: - sys.path.pop(0) +def arduino_boards() -> dict: + """Returns a dict mapping board to target""" + template = project.TemplateProject.from_directory(TEMPLATE_PROJECT_DIR) + project_options = template.info()["project_options"] + for option in project_options: + if option["name"] == "arduino_board": + boards = option["choices"] + if option["name"] == "arduino_target": + targets = option["choices"] - return microtvm_api_server.MICRO_DEVICES + arduino_boards = {boards[i]: targets[i] for i in range(len(boards))} + return arduino_boards +ARDUINO_BOARDS = arduino_boards() def pytest_addoption(parser): parser.addoption( - "--microtvm-device", + "--arduino-board", nargs="+", required=True, - choices=arduino_micro_devices().keys(), + choices=ARDUINO_BOARDS.keys(), help="MicroTVM device for tests.", ) parser.addoption( @@ -89,8 +95,8 @@ def pytest_collection_modifyitems(config, items): # (to take advantage of multiple cores / external memory / etc.), so all tests # are parameterized by board def pytest_generate_tests(metafunc): - device = metafunc.config.getoption("microtvm_device") - metafunc.parametrize("device", device, scope="session") + board = metafunc.config.getoption("arduino_board") + metafunc.parametrize("board", board, scope="session") @pytest.fixture(scope="session") @@ -103,12 +109,11 @@ def tvm_debug(request): return request.config.getoption("--tvm-debug") -def make_workspace_dir(test_name, device): - _, arduino_board = arduino_micro_devices()[device] +def make_workspace_dir(test_name, board): filepath = pathlib.Path(__file__) board_workspace = ( filepath.parent - / f"workspace_{test_name}_{arduino_board}" + / f"workspace_{test_name}_{board}" / datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S") ) @@ -122,9 +127,9 @@ def make_workspace_dir(test_name, device): return t -def make_kws_project(device, arduino_cli_cmd, tvm_debug, workspace_dir): +def make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir): this_dir = pathlib.Path(__file__).parent - model, arduino_board = arduino_micro_devices()[device] + model = ARDUINO_BOARDS[board] build_config = {"debug": tvm_debug} with open(this_dir.parent / "testdata" / "kws" / "yes_no.tflite", "rb") as f: @@ -153,7 +158,7 @@ def make_kws_project(device, arduino_cli_cmd, tvm_debug, workspace_dir): mod, workspace_dir / "project", { - "arduino_board": arduino_board, + "arduino_board": board, "arduino_cli_cmd": arduino_cli_cmd, "project_type": "example_project", "verbose": bool(build_config.get("debug")), diff --git a/tests/micro/arduino/test_arduino_error_detection.py b/tests/micro/arduino/test_arduino_error_detection.py index cdccf91a92e1..64e2c14d1c18 100644 --- a/tests/micro/arduino/test_arduino_error_detection.py +++ b/tests/micro/arduino/test_arduino_error_detection.py @@ -27,13 +27,13 @@ # A new project and workspace dir is created for EVERY test @pytest.fixture -def workspace_dir(request, device): - return conftest.make_workspace_dir("arduino_error_detection", device) +def workspace_dir(request, board): + return conftest.make_workspace_dir("arduino_error_detection", board) @pytest.fixture -def project(device, arduino_cli_cmd, tvm_debug, workspace_dir): - return conftest.make_kws_project(device, arduino_cli_cmd, tvm_debug, workspace_dir) +def project(board, arduino_cli_cmd, tvm_debug, workspace_dir): + return conftest.make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir) def test_blank_project_compiles(workspace_dir, project): diff --git a/tests/micro/arduino/test_arduino_rpc_server.py b/tests/micro/arduino/test_arduino_rpc_server.py index 154bda62e24e..78ddd9a613c8 100644 --- a/tests/micro/arduino/test_arduino_rpc_server.py +++ b/tests/micro/arduino/test_arduino_rpc_server.py @@ -36,12 +36,10 @@ import conftest -MICRO_DEVICES = conftest.arduino_micro_devices() - # # A new project and workspace dir is created for EVERY test @pytest.fixture -def workspace_dir(device): - return conftest.make_workspace_dir("arduino_rpc_server", device) +def workspace_dir(board): + return conftest.make_workspace_dir("arduino_rpc_server", board) def _make_session(model, arduino_board, arduino_cli_cmd, workspace_dir, mod, build_config): @@ -84,10 +82,10 @@ def _make_add_sess(model, arduino_board, arduino_cli_cmd, workspace_dir, build_c # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_compile_runtime(device, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_compile_runtime(board, arduino_cli_cmd, tvm_debug, workspace_dir): """Test compiling the on-device runtime.""" - model, arduino_board = MICRO_DEVICES[device] + model = conftest.ARDUINO_BOARDS[board] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -103,16 +101,16 @@ def test_basic_add(sess): system_lib.get_function("add")(A_data, B_data, C_data) assert (C_data.numpy() == np.array([6, 7])).all() - with _make_add_sess(model, arduino_board, arduino_cli_cmd, workspace_dir, build_config) as sess: + with _make_add_sess(model, board, arduino_cli_cmd, workspace_dir, build_config) as sess: test_basic_add(sess) @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_platform_timer(device, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_platform_timer(board, arduino_cli_cmd, tvm_debug, workspace_dir): """Test compiling the on-device runtime.""" - model, arduino_board = MICRO_DEVICES[device] + model = conftest.ARDUINO_BOARDS[board] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -133,15 +131,15 @@ def test_basic_add(sess): assert result.mean > 0 assert len(result.results) == 3 - with _make_add_sess(model, arduino_board, arduino_cli_cmd, workspace_dir, build_config) as sess: + with _make_add_sess(model, board, arduino_cli_cmd, workspace_dir, build_config) as sess: test_basic_add(sess) @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_relay(device, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_relay(board, arduino_cli_cmd, tvm_debug, workspace_dir): """Testing a simple relay graph""" - model, arduino_board = MICRO_DEVICES[device] + model = conftest.ARDUINO_BOARDS[board] build_config = {"debug": tvm_debug} shape = (10,) @@ -158,7 +156,7 @@ def test_relay(device, arduino_cli_cmd, tvm_debug, workspace_dir): mod = tvm.relay.build(func, target=target) with _make_session( - model, arduino_board, arduino_cli_cmd, workspace_dir, mod, build_config + model, board, arduino_cli_cmd, workspace_dir, mod, build_config ) as session: graph_mod = tvm.micro.create_local_graph_executor( mod.get_graph_json(), session.get_system_lib(), session.device @@ -173,9 +171,9 @@ def test_relay(device, arduino_cli_cmd, tvm_debug, workspace_dir): @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_onnx(device, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_onnx(board, arduino_cli_cmd, tvm_debug, workspace_dir): """Testing a simple ONNX model.""" - model, arduino_board = MICRO_DEVICES[device] + model = conftest.ARDUINO_BOARDS[board] build_config = {"debug": tvm_debug} # Load test images. @@ -201,7 +199,7 @@ def test_onnx(device, arduino_cli_cmd, tvm_debug, workspace_dir): graph = lowered.get_graph_json() with _make_session( - model, arduino_board, arduino_cli_cmd, workspace_dir, lowered, build_config + model, board, arduino_cli_cmd, workspace_dir, lowered, build_config ) as session: graph_mod = tvm.micro.create_local_graph_executor( graph, session.get_system_lib(), session.device @@ -261,9 +259,9 @@ def check_result( @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_byoc_microtvm(device, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_byoc_microtvm(board, arduino_cli_cmd, tvm_debug, workspace_dir): """This is a simple test case to check BYOC capabilities of microTVM""" - model, arduino_board = MICRO_DEVICES[device] + model = conftest.ARDUINO_BOARDS[board] build_config = {"debug": tvm_debug} x = relay.var("x", shape=(10, 10)) @@ -318,7 +316,7 @@ def test_byoc_microtvm(device, arduino_cli_cmd, tvm_debug, workspace_dir): ), model=model, build_config=build_config, - arduino_board=arduino_board, + arduino_board=board, arduino_cli_cmd=arduino_cli_cmd, workspace_dir=workspace_dir, ) @@ -345,9 +343,9 @@ def _make_add_sess_with_shape( ) @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_rpc_large_array(device, arduino_cli_cmd, tvm_debug, workspace_dir, shape): +def test_rpc_large_array(board, arduino_cli_cmd, tvm_debug, workspace_dir, shape): """Test large RPC array transfer.""" - model, arduino_board = MICRO_DEVICES[device] + model = conftest.ARDUINO_BOARDS[board] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -360,7 +358,7 @@ def test_tensors(sess): assert (C_data.numpy() == np.zeros(shape)).all() with _make_add_sess_with_shape( - model, arduino_board, arduino_cli_cmd, workspace_dir, shape, build_config + model, board, arduino_cli_cmd, workspace_dir, shape, build_config ) as sess: test_tensors(sess) diff --git a/tests/micro/arduino/test_arduino_workflow.py b/tests/micro/arduino/test_arduino_workflow.py index 2663704afc7b..b080ca2bc9e9 100644 --- a/tests/micro/arduino/test_arduino_workflow.py +++ b/tests/micro/arduino/test_arduino_workflow.py @@ -40,8 +40,8 @@ # Since these tests are sequential, we'll use the same project/workspace # directory for all tests in this file @pytest.fixture(scope="module") -def workspace_dir(request, device): - return conftest.make_workspace_dir("arduino_workflow", device) +def workspace_dir(request, board): + return conftest.make_workspace_dir("arduino_workflow", board) @pytest.fixture(scope="module") @@ -51,8 +51,8 @@ def project_dir(workspace_dir): # We MUST pass workspace_dir, not project_dir, or the workspace will be dereferenced too soon @pytest.fixture(scope="module") -def project(device, arduino_cli_cmd, tvm_debug, workspace_dir): - return conftest.make_kws_project(device, arduino_cli_cmd, tvm_debug, workspace_dir) +def project(board, arduino_cli_cmd, tvm_debug, workspace_dir): + return conftest.make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir) def _get_directory_elements(directory): From 569c523b8f3343918345a1f2c099523570f82575 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Wed, 8 Sep 2021 11:16:29 -0700 Subject: [PATCH 10/12] fix zephyr --- .../template_project/microtvm_api_server.py | 57 +++++++++++++++---- tests/micro/arduino/conftest.py | 2 +- tests/micro/zephyr/conftest.py | 34 ++++++----- tests/micro/zephyr/test_zephyr.py | 48 ++++++++-------- tests/micro/zephyr/test_zephyr_aot.py | 18 +++--- tests/scripts/task_python_microtvm.sh | 8 +-- 6 files changed, 100 insertions(+), 67 deletions(-) diff --git a/apps/microtvm/zephyr/template_project/microtvm_api_server.py b/apps/microtvm/zephyr/template_project/microtvm_api_server.py index 680a51118429..a4e74b65040d 100644 --- a/apps/microtvm/zephyr/template_project/microtvm_api_server.py +++ b/apps/microtvm/zephyr/template_project/microtvm_api_server.py @@ -57,17 +57,45 @@ IS_TEMPLATE = not (API_SERVER_DIR / MODEL_LIBRARY_FORMAT_RELPATH).exists() -# Maps a short, identifying microtvm device string to (target, zephyr_board). -MICRO_DEVICES = { - "qemu_x86": ("host", "qemu_x86"), - "qemu_riscv32": ("host", "qemu_riscv32"), - "qemu_riscv64": ("host", "qemu_riscv64"), - "mps2_an521": ("mps2_an521", "mps2_an521"), - "nrf5340dk": ("nrf5340dk", "nrf5340dk_nrf5340_cpuapp"), - "stm32f746xx_disco": ("stm32f746xx", "stm32f746g_disco"), - "stm32f746xx_nucleo": ("stm32f746xx", "nucleo_f746zg"), - "stm32l4r5zi_nucleo": ("stm32l4r5zi", "nucleo_l4r5zi"), - "zynq_mp_r5": ("zynq_mp_r5", "qemu_cortex_r5"), +# Data structure to hold the information microtvm_api_server.py needs +# to communicate with each of these boards. +BOARD_PROPERTIES = { + "qemu_x86": { + "board": "qemu_x86", + "target": "host", + }, + "qemu_riscv32": { + "board": "qemu_riscv32", + "target": "host", + }, + "qemu_riscv64": { + "board": "qemu_riscv64", + "target": "host", + }, + "mps2_an521": { + "board": "mps2_an521", + "target": "mps2_an521", + }, + "nrf5340dk_nrf5340_cpuapp": { + "board": "nrf5340dk_nrf5340_cpuapp", + "target": "nrf5340dk", + }, + "stm32f746xx_disco": { + "board": "stm32f746xx_disco", + "target": "stm32f746xx", + }, + "nucleo_f746zg": { + "board": "nucleo_f746zg", + "target": "stm32f746xx", + }, + "nucleo_l4r5zi": { + "board": "nucleo_l4r5zi", + "target": "stm32l4r5zi", + }, + "qemu_cortex_r5": { + "board": "qemu_cortex_r5", + "target": "zynq_mp_r5", + }, } @@ -256,7 +284,12 @@ def _get_nrf_device_args(options): ), ), server.ProjectOption("zephyr_base", help="Path to the zephyr base directory."), - server.ProjectOption("zephyr_board", help="Name of the Zephyr board to build for."), + server.ProjectOption("zephyr_board", choices=list(BOARD_PROPERTIES), help="Name of the Zephyr board to build for."), + server.ProjectOption( + "zephyr_target", + choices=[board["target"] for _,board in BOARD_PROPERTIES.items()], + help="Name of the target for each Zephyr board.", + ), ] diff --git a/tests/micro/arduino/conftest.py b/tests/micro/arduino/conftest.py index e1d8bc0a4f98..108f270b06ca 100644 --- a/tests/micro/arduino/conftest.py +++ b/tests/micro/arduino/conftest.py @@ -37,7 +37,7 @@ def arduino_boards() -> dict: - """Returns a dict mapping board to target""" + """Returns a dict mapping board to target model""" template = project.TemplateProject.from_directory(TEMPLATE_PROJECT_DIR) project_options = template.info()["project_options"] for option in project_options: diff --git a/tests/micro/zephyr/conftest.py b/tests/micro/zephyr/conftest.py index 3326b1a32335..e2ade72eef21 100644 --- a/tests/micro/zephyr/conftest.py +++ b/tests/micro/zephyr/conftest.py @@ -21,6 +21,7 @@ import pytest +from tvm.micro import project import tvm.contrib.utils import tvm.target.target @@ -36,21 +37,25 @@ ).resolve() -def zephyr_micro_devices() -> dict: - sys.path.insert(0, str(TEMPLATE_PROJECT_DIR)) - try: - import microtvm_api_server - finally: - sys.path.pop(0) +def zephyr_boards() -> dict: + """Returns a dict mapping board to target model""" + template = project.TemplateProject.from_directory(TEMPLATE_PROJECT_DIR) + project_options = template.info()["project_options"] + for option in project_options: + if option["name"] == "zephyr_board": + boards = option["choices"] + if option["name"] == "zephyr_target": + targets = option["choices"] - return microtvm_api_server.MICRO_DEVICES + arduino_boards = {boards[i]: targets[i] for i in range(len(boards))} + return arduino_boards +ZEPHYR_BOARDS = zephyr_boards() def pytest_addoption(parser): parser.addoption( - "--microtvm-device", - default="qemu_x86", - choices=zephyr_micro_devices().keys(), + "--zephyr-board", + choices=ZEPHYR_BOARDS.keys(), help=( "Specify a microtvm device (i.e. as passed to tvm.target.micro()) for microTVM tests." ), @@ -67,8 +72,8 @@ def pytest_addoption(parser): def pytest_generate_tests(metafunc): - if "device" in metafunc.fixturenames: - metafunc.parametrize("device", [metafunc.config.getoption("microtvm_device")]) + if "board" in metafunc.fixturenames: + metafunc.parametrize("board", [metafunc.config.getoption("zephyr_board")]) @pytest.fixture @@ -82,13 +87,12 @@ def tvm_debug(request): @pytest.fixture -def temp_dir(device): - _, zephyr_board = zephyr_micro_devices()[device] +def temp_dir(board): parent_dir = pathlib.Path(os.path.dirname(__file__)) filename = os.path.splitext(os.path.basename(__file__))[0] board_workspace = ( parent_dir - / f"workspace_{filename}_{zephyr_board}" + / f"workspace_{filename}_{board}" / datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S") ) board_workspace_base = str(board_workspace) diff --git a/tests/micro/zephyr/test_zephyr.py b/tests/micro/zephyr/test_zephyr.py index b4309f7fdf14..8c843f03b98c 100644 --- a/tests/micro/zephyr/test_zephyr.py +++ b/tests/micro/zephyr/test_zephyr.py @@ -46,8 +46,6 @@ _LOG = logging.getLogger(__name__) -MICRO_DEVICES = conftest.zephyr_micro_devices() - def _make_sess_from_op( temp_dir, model, zephyr_board, west_cmd, op_name, sched, arg_bufs, build_config @@ -89,10 +87,10 @@ def _make_add_sess(temp_dir, model, zephyr_board, west_cmd, build_config, dtype= # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro -def test_add_uint(temp_dir, device, west_cmd, tvm_debug): +def test_add_uint(temp_dir, board, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" - model, zephyr_board = MICRO_DEVICES[device] + model = conftest.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -108,7 +106,7 @@ def test_basic_add(sess): system_lib.get_function("add")(A_data, B_data, C_data) assert (C_data.numpy() == np.array([6, 7])).all() - with _make_add_sess(temp_dir, model, zephyr_board, west_cmd, build_config) as sess: + with _make_add_sess(temp_dir, model, board, west_cmd, build_config) as sess: test_basic_add(sess) @@ -124,11 +122,11 @@ def has_fpu(zephyr_board): # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro -def test_add_float(temp_dir, device, west_cmd, tvm_debug): +def test_add_float(temp_dir, board, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" - model, zephyr_board = MICRO_DEVICES[device] - if not has_fpu(zephyr_board): - pytest.skip(f"FPU not enabled for {device}") + model = conftest.ZEPHYR_BOARDS[board] + if not has_fpu(board): + pytest.skip(f"FPU not enabled for {board}") build_config = {"debug": tvm_debug} @@ -146,16 +144,16 @@ def test_basic_add(sess): assert (C_data.numpy() == np.array([7, 8])).all() with _make_add_sess( - temp_dir, model, zephyr_board, west_cmd, build_config, dtype="float32" + temp_dir, model, board, west_cmd, build_config, dtype="float32" ) as sess: test_basic_add(sess) @tvm.testing.requires_micro -def test_platform_timer(temp_dir, device, west_cmd, tvm_debug): +def test_platform_timer(temp_dir, board, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" - model, zephyr_board = MICRO_DEVICES[device] + model = conftest.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -176,14 +174,14 @@ def test_basic_add(sess): assert result.mean > 0 assert len(result.results) == 3 - with _make_add_sess(temp_dir, model, zephyr_board, west_cmd, build_config) as sess: + with _make_add_sess(temp_dir, model, board, west_cmd, build_config) as sess: test_basic_add(sess) @tvm.testing.requires_micro -def test_relay(temp_dir, device, west_cmd, tvm_debug): +def test_relay(temp_dir, board, west_cmd, tvm_debug): """Testing a simple relay graph""" - model, zephyr_board = MICRO_DEVICES[device] + model = conftest.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} shape = (10,) dtype = "int8" @@ -199,7 +197,7 @@ def test_relay(temp_dir, device, west_cmd, tvm_debug): with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.relay.build(ir_mod, target=target) - with _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config) as session: + with _make_session(temp_dir, board, west_cmd, mod, build_config) as session: graph_mod = tvm.micro.create_local_graph_executor( mod.get_graph_json(), session.get_system_lib(), session.device ) @@ -212,9 +210,9 @@ def test_relay(temp_dir, device, west_cmd, tvm_debug): @tvm.testing.requires_micro -def test_onnx(temp_dir, device, west_cmd, tvm_debug): +def test_onnx(temp_dir, board, west_cmd, tvm_debug): """Testing a simple ONNX model.""" - model, zephyr_board = MICRO_DEVICES[device] + model = conftest.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} this_dir = pathlib.Path(os.path.dirname(__file__)) @@ -242,7 +240,7 @@ def test_onnx(temp_dir, device, west_cmd, tvm_debug): lowered = relay.build(relay_mod, target, params=params) graph = lowered.get_graph_json() - with _make_session(temp_dir, zephyr_board, west_cmd, lowered, build_config) as session: + with _make_session(temp_dir, board, west_cmd, lowered, build_config) as session: graph_mod = tvm.micro.create_local_graph_executor( graph, session.get_system_lib(), session.device ) @@ -289,9 +287,9 @@ def check_result( @tvm.testing.requires_micro -def test_byoc_microtvm(temp_dir, device, west_cmd, tvm_debug): +def test_byoc_microtvm(temp_dir, board, west_cmd, tvm_debug): """This is a simple test case to check BYOC capabilities of microTVM""" - model, zephyr_board = MICRO_DEVICES[device] + model = conftest.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} x = relay.var("x", shape=(10, 10)) w0 = relay.var("w0", shape=(10, 10)) @@ -345,7 +343,7 @@ def test_byoc_microtvm(temp_dir, device, west_cmd, tvm_debug): axis=0, ), model=model, - zephyr_board=zephyr_board, + zephyr_board=board, west_cmd=west_cmd, build_config=build_config, ) @@ -369,9 +367,9 @@ def _make_add_sess_with_shape(temp_dir, model, zephyr_board, west_cmd, shape, bu ], ) @tvm.testing.requires_micro -def test_rpc_large_array(temp_dir, device, west_cmd, tvm_debug, shape): +def test_rpc_large_array(temp_dir, board, west_cmd, tvm_debug, shape): """Test large RPC array transfer.""" - model, zephyr_board = MICRO_DEVICES[device] + model = conftest.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -384,7 +382,7 @@ def test_tensors(sess): assert (C_data.numpy() == np.zeros(shape)).all() with _make_add_sess_with_shape( - temp_dir, model, zephyr_board, west_cmd, shape, build_config + temp_dir, model, board, west_cmd, shape, build_config ) as sess: test_tensors(sess) diff --git a/tests/micro/zephyr/test_zephyr_aot.py b/tests/micro/zephyr/test_zephyr_aot.py index 719b5a9a0ae1..c0fb04668bac 100644 --- a/tests/micro/zephyr/test_zephyr_aot.py +++ b/tests/micro/zephyr/test_zephyr_aot.py @@ -42,8 +42,6 @@ _LOG = logging.getLogger(__name__) -MICRO_DEVICES = conftest.zephyr_micro_devices() - def _build_project(temp_dir, zephyr_board, west_cmd, mod, build_config, extra_files_tar=None): template_project_dir = ( @@ -135,13 +133,13 @@ def _get_message(fd, expr: str, timeout_sec: int): @tvm.testing.requires_micro -def test_tflite(temp_dir, device, west_cmd, tvm_debug): +def test_tflite(temp_dir, board, west_cmd, tvm_debug): """Testing a TFLite model.""" - if device not in ["qemu_x86", "mps2_an521", "nrf5340dk", "stm32l4r5zi_nucleo", "zynq_mp_r5"]: + if board not in ["qemu_x86", "mps2_an521", "nrf5340dk", "stm32l4r5zi_nucleo", "zynq_mp_r5"]: pytest.skip(msg="Model does not fit.") - model, zephyr_board = MICRO_DEVICES[device] + model = conftest.ZEPHYR_BOARDS[board] input_shape = (1, 32, 32, 3) output_shape = (1, 10) build_config = {"debug": tvm_debug} @@ -195,7 +193,7 @@ def test_tflite(temp_dir, device, west_cmd, tvm_debug): project, _ = _build_project( temp_dir, - zephyr_board, + board, west_cmd, lowered, build_config, @@ -218,12 +216,12 @@ def test_tflite(temp_dir, device, west_cmd, tvm_debug): @tvm.testing.requires_micro -def test_qemu_make_fail(temp_dir, device, west_cmd, tvm_debug): +def test_qemu_make_fail(temp_dir, board, west_cmd, tvm_debug): """Testing QEMU make fail.""" - if device not in ["qemu_x86", "mps2_an521"]: + if board not in ["qemu_x86", "mps2_an521"]: pytest.skip(msg="Only for QEMU targets.") - model, zephyr_board = MICRO_DEVICES[device] + model = conftest.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} shape = (10,) dtype = "float32" @@ -254,7 +252,7 @@ def test_qemu_make_fail(temp_dir, device, west_cmd, tvm_debug): project, project_dir = _build_project( temp_dir, - zephyr_board, + board, west_cmd, lowered, build_config, diff --git a/tests/scripts/task_python_microtvm.sh b/tests/scripts/task_python_microtvm.sh index 48921201b6d2..4bee8d566f11 100755 --- a/tests/scripts/task_python_microtvm.sh +++ b/tests/scripts/task_python_microtvm.sh @@ -23,11 +23,11 @@ set -x # NOTE(areusch): Adding to diagnose flaky timeouts source tests/scripts/setup-pytest-env.sh make cython3 -run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-device=qemu_x86 +run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --zephyr-board=qemu_x86 # Temporarily removing mps2_an512 from CI due to issue 8728: # https://github.com/apache/tvm/issues/8728 -# run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-device=mps2_an521 +# run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --zephyr-board=mps2_an521 run_pytest ctypes python-microtvm-arduino apps/microtvm/arduino/template_project/tests -run_pytest ctypes python-microtvm-arduino-nano33ble tests/micro/arduino --test-build-only --microtvm-device=nano33ble -run_pytest ctypes python-microtvm-arduino-due tests/micro/arduino --test-build-only --microtvm-device=due +run_pytest ctypes python-microtvm-arduino-nano33ble tests/micro/arduino --test-build-only --arduino-board=nano33ble +run_pytest ctypes python-microtvm-arduino-due tests/micro/arduino --test-build-only --arduino-board=due From 4d1230b6c50c0314489d39829d3f31581c089d8d Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Wed, 8 Sep 2021 12:13:58 -0700 Subject: [PATCH 11/12] address comments --- .../template_project/microtvm_api_server.py | 25 +++++++------- apps/microtvm/reference-vm/README.md | 6 ++-- .../arduino/base-box/base_box_test.sh | 14 ++++---- apps/microtvm/reference-vm/base-box-tool.py | 34 +++++++++---------- .../zephyr/base-box/base_box_test.sh | 14 ++++---- .../template_project/microtvm_api_server.py | 30 +++++++++------- tests/micro/arduino/README.md | 4 +-- tests/micro/arduino/conftest.py | 9 ++--- .../micro/arduino/test_arduino_rpc_server.py | 4 +-- tests/micro/zephyr/README.md | 6 ++-- tests/micro/zephyr/conftest.py | 9 ++--- tests/micro/zephyr/test_zephyr.py | 8 ++--- tests/micro/zephyr/test_zephyr_aot.py | 8 ++++- tutorials/micro/micro_reference_vm.py | 4 +-- 14 files changed, 91 insertions(+), 84 deletions(-) diff --git a/apps/microtvm/arduino/template_project/microtvm_api_server.py b/apps/microtvm/arduino/template_project/microtvm_api_server.py index be134ade0b7c..3d25d0bcad8f 100644 --- a/apps/microtvm/arduino/template_project/microtvm_api_server.py +++ b/apps/microtvm/arduino/template_project/microtvm_api_server.py @@ -43,6 +43,7 @@ IS_TEMPLATE = not (API_SERVER_DIR / MODEL_LIBRARY_FORMAT_RELPATH).exists() + class BoardAutodetectFailed(Exception): """Raised when no attached hardware is found matching the requested board""" @@ -56,7 +57,7 @@ class BoardAutodetectFailed(Exception): "package": "arduino", "architecture": "sam", "board": "arduino_due_x_dbg", - "target": "sam3x8e", + "model": "sam3x8e", }, # Due to the way the Feather S2 bootloader works, compilation # behaves fine but uploads cannot be done automatically @@ -64,32 +65,32 @@ class BoardAutodetectFailed(Exception): "package": "esp32", "architecture": "esp32", "board": "feathers2", - "target": "esp32", + "model": "esp32", }, "metrom4": { "package": "adafruit", "architecture": "samd", "board": "adafruit_metro_m4", - "target": "atsamd51" + "model": "atsamd51", }, # Spresense only works as of its v2.3.0 sdk "spresense": { "package": "SPRESENSE", "architecture": "spresense", "board": "spresense", - "target": "cxd5602gg", + "model": "cxd5602gg", }, "nano33ble": { "package": "arduino", "architecture": "mbed_nano", "board": "nano33ble", - "target": "nrf52840", + "model": "nrf52840", }, "pybadge": { "package": "adafruit", "architecture": "samd", "board": "adafruit_pybadge_m4", - "target": "atsamd51", + "model": "atsamd51", }, # The Teensy boards are listed here for completeness, but they # won't work until https://github.com/arduino/arduino-cli/issues/700 @@ -98,19 +99,19 @@ class BoardAutodetectFailed(Exception): "package": "teensy", "architecture": "avr", "board": "teensy40", - "target": "imxrt1060", + "model": "imxrt1060", }, "teensy41": { "package": "teensy", "architecture": "avr", "board": "teensy41", - "target": "imxrt1060", + "model": "imxrt1060", }, "wioterminal": { "package": "Seeeduino", "architecture": "samd", "board": "seeed_wio_terminal", - "target": "atsamd51", + "model": "atsamd51", }, } @@ -123,9 +124,9 @@ class BoardAutodetectFailed(Exception): help="Name of the Arduino board to build for", ), server.ProjectOption( - "arduino_target", - choices=[board["target"] for _,board in BOARD_PROPERTIES.items()], - help="Name of the target for each Arduino board.", + "arduino_model", + choices=[board["model"] for _, board in BOARD_PROPERTIES.items()], + help="Name of the model for each Arduino board.", ), server.ProjectOption("arduino_cli_cmd", help="Path to the arduino-cli tool."), server.ProjectOption("port", help="Port to use for connecting to hardware"), diff --git a/apps/microtvm/reference-vm/README.md b/apps/microtvm/reference-vm/README.md index 37d3cc94d484..d84a36bb450a 100644 --- a/apps/microtvm/reference-vm/README.md +++ b/apps/microtvm/reference-vm/README.md @@ -72,20 +72,20 @@ For example: $ ./base-box-tool.py --provider virtualbox build zephyr ``` -2. **Run** release tests for each microTVM device: +2. **Run** release tests for each platform: A. Connect any needed hardware to the VM host machine; B. Run tests: ```bash - $ ./base-box-tool.py [--provider=PROVIDER] test --microtvm-device=MICROTVM_DEVICE [--test-device-serial=SERIAL] PLATFORM + $ ./base-box-tool.py [--provider=PROVIDER] test --microtvm-board=MICROTVM_DEVICE [--test-device-serial=SERIAL] PLATFORM ``` where MICROTVM_DEVICE is one of the options listed in the PLATFORM/base-box/test-config.json file. For example: ```base - $ ./base-box-tool.py --provider virtualbox test --microtvm-device=stm32f746xx_disco zephyr + $ ./base-box-tool.py --provider virtualbox test --microtvm-board=stm32f746xx_disco zephyr ``` This command does the following for the specified provider: diff --git a/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh b/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh index aed3bab60d2d..5c3d96dfc7df 100755 --- a/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh +++ b/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. # -# Usage: base_box_test.sh +# Usage: base_box_test.sh # Execute microTVM Arduino tests. # @@ -24,17 +24,17 @@ set -e set -x if [ "$#" -lt 1 ]; then - echo "Usage: base_box_test.sh " + echo "Usage: base_box_test.sh " exit -1 fi -microtvm_device=$1 +board=$1 -pytest tests/micro/arduino/test_arduino_workflow.py --microtvm-device=${microtvm_device} +pytest tests/micro/arduino/test_arduino_workflow.py --arduino-board=${board} -if [ $microtvm_device == "nano33ble" ]; then +if [ $board == "nano33ble" ]; then # https://github.com/apache/tvm/issues/8730 - echo "NOTE: skipped test_arduino_rpc_server.py on $microtvm_device -- known failure" + echo "NOTE: skipped test_arduino_rpc_server.py on $board -- known failure" else - pytest tests/micro/arduino/test_arduino_rpc_server.py --microtvm-device=${microtvm_device} + pytest tests/micro/arduino/test_arduino_rpc_server.py --arduino-board=${board} fi diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py index 1951f40ccace..1682f5e1e032 100755 --- a/apps/microtvm/reference-vm/base-box-tool.py +++ b/apps/microtvm/reference-vm/base-box-tool.py @@ -57,9 +57,9 @@ PACKER_FILE_NAME = "packer.json" -# List of identifying strings for microTVM devices for testing. -# TODO add a way to declare supported devices to ProjectAPI -ALL_MICROTVM_DEVICES = { +# List of identifying strings for microTVM boards for testing. +# TODO add a way to declare supported boards to ProjectAPI +ALL_MICROTVM_BOARDS = { "arduino": ( "due", "feathers2", @@ -72,9 +72,9 @@ "wioterminal", ), "zephyr": ( - "stm32f746xx_nucleo", + "nucleo_f746zg", "stm32f746xx_disco", - "nrf5340dk", + "nrf5340dk_nrf5340_cpuapp", "mps2_an521", ), } @@ -361,7 +361,7 @@ def _quote_cmd(cmd): + _quote_cmd( [ f"apps/microtvm/reference-vm/{platform}/base-box/base_box_test.sh", - test_config["microtvm_device"], + test_config["microtvm_board"], ] ) ) @@ -375,17 +375,17 @@ def test_command(args): with open(test_config_file) as f: test_config = json.load(f) - # select microTVM test device - microtvm_test_device = test_config[args.microtvm_device] + # select microTVM test config + microtvm_test_config = test_config[args.microtvm_board] for key, expected_type in REQUIRED_TEST_CONFIG_KEYS.items(): - assert key in microtvm_test_device and isinstance( - microtvm_test_device[key], expected_type + assert key in microtvm_test_config and isinstance( + microtvm_test_config[key], expected_type ), f"Expected key {key} of type {expected_type} in {test_config_file}: {test_config!r}" - microtvm_test_device["vid_hex"] = microtvm_test_device["vid_hex"].lower() - microtvm_test_device["pid_hex"] = microtvm_test_device["pid_hex"].lower() - microtvm_test_device["microtvm_device"] = args.microtvm_device + microtvm_test_config["vid_hex"] = microtvm_test_config["vid_hex"].lower() + microtvm_test_config["pid_hex"] = microtvm_test_config["pid_hex"].lower() + microtvm_test_config["microtvm_board"] = args.microtvm_board providers = args.provider provider_passed = {p: False for p in providers} @@ -405,7 +405,7 @@ def test_command(args): release_test_dir, args.platform, provider_name, - microtvm_test_device, + microtvm_test_config, args.test_device_serial, ) provider_passed[provider_name] = True @@ -510,10 +510,10 @@ def parse_args(): platform_specific_parser = parser_test_platform_subparsers.add_parser(platform) platform_specific_parser.set_defaults(platform=platform) platform_specific_parser.add_argument( - "--microtvm-device", - choices=ALL_MICROTVM_DEVICES[platform], + "--microtvm-board", + choices=ALL_MICROTVM_BOARDS[platform], required=True, - help="MicroTVM device used for testing.", + help="MicroTVM board used for testing.", ) # Options for release subcommand diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh index c9587ea3a86d..2a023b520b01 100755 --- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh +++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. # -# Usage: base_box_test.sh +# Usage: base_box_test.sh # Execute microTVM Zephyr tests. # @@ -24,16 +24,16 @@ set -e set -x if [ "$#" -lt 1 ]; then - echo "Usage: base_box_test.sh " + echo "Usage: base_box_test.sh " exit -1 fi -microtvm_device=$1 +board=$1 -pytest tests/micro/zephyr/test_zephyr.py --microtvm-device=${microtvm_device} +pytest tests/micro/zephyr/test_zephyr.py --zephyr-board=${board} -if [ $microtvm_device == "stm32f746xx" ]; then - echo "NOTE: skipped test_zephyr_aot.py on $microtvm_device -- known failure" +if [ $board == "stm32f746xx" ]; then + echo "NOTE: skipped test_zephyr_aot.py on $board -- known failure" else - pytest tests/micro/zephyr/test_zephyr_aot.py --microtvm-device=${microtvm_device} + pytest tests/micro/zephyr/test_zephyr_aot.py --zephyr-board=${board} fi diff --git a/apps/microtvm/zephyr/template_project/microtvm_api_server.py b/apps/microtvm/zephyr/template_project/microtvm_api_server.py index a4e74b65040d..08ba1a16e5ec 100644 --- a/apps/microtvm/zephyr/template_project/microtvm_api_server.py +++ b/apps/microtvm/zephyr/template_project/microtvm_api_server.py @@ -62,39 +62,39 @@ BOARD_PROPERTIES = { "qemu_x86": { "board": "qemu_x86", - "target": "host", + "model": "host", }, "qemu_riscv32": { "board": "qemu_riscv32", - "target": "host", + "model": "host", }, "qemu_riscv64": { "board": "qemu_riscv64", - "target": "host", + "model": "host", }, "mps2_an521": { "board": "mps2_an521", - "target": "mps2_an521", + "model": "mps2_an521", }, "nrf5340dk_nrf5340_cpuapp": { "board": "nrf5340dk_nrf5340_cpuapp", - "target": "nrf5340dk", + "model": "nrf5340dk", }, "stm32f746xx_disco": { "board": "stm32f746xx_disco", - "target": "stm32f746xx", + "model": "stm32f746xx", }, "nucleo_f746zg": { "board": "nucleo_f746zg", - "target": "stm32f746xx", + "model": "stm32f746xx", }, "nucleo_l4r5zi": { "board": "nucleo_l4r5zi", - "target": "stm32l4r5zi", + "model": "stm32l4r5zi", }, "qemu_cortex_r5": { "board": "qemu_cortex_r5", - "target": "zynq_mp_r5", + "model": "zynq_mp_r5", }, } @@ -284,11 +284,15 @@ def _get_nrf_device_args(options): ), ), server.ProjectOption("zephyr_base", help="Path to the zephyr base directory."), - server.ProjectOption("zephyr_board", choices=list(BOARD_PROPERTIES), help="Name of the Zephyr board to build for."), server.ProjectOption( - "zephyr_target", - choices=[board["target"] for _,board in BOARD_PROPERTIES.items()], - help="Name of the target for each Zephyr board.", + "zephyr_board", + choices=list(BOARD_PROPERTIES), + help="Name of the Zephyr board to build for.", + ), + server.ProjectOption( + "zephyr_model", + choices=[board["model"] for _, board in BOARD_PROPERTIES.items()], + help="Name of the model for each Zephyr board.", ), ] diff --git a/tests/micro/arduino/README.md b/tests/micro/arduino/README.md index bcad615593c1..0b039ba6de7c 100644 --- a/tests/micro/arduino/README.md +++ b/tests/micro/arduino/README.md @@ -22,14 +22,14 @@ all of the appropriate TVM dependencies installed. You can run the test with: ``` $ cd tvm/tests/micro/arduino -$ pytest --microtvm-device=spresense +$ pytest --arduino-board=spresense ``` Most of these tests require a supported Arduino board to be connected. If you don't want to run these tests, you can pass the flag `--test-build-only` to only test project generation and compilation. -To see the list of supported values for `----microtvm-device`, run: +To see the list of supported values for `--arduino-board`, run: ``` $ pytest --help ``` diff --git a/tests/micro/arduino/conftest.py b/tests/micro/arduino/conftest.py index 108f270b06ca..0ff4a4d98f6d 100644 --- a/tests/micro/arduino/conftest.py +++ b/tests/micro/arduino/conftest.py @@ -17,7 +17,6 @@ import datetime import pathlib -import sys import pytest import tvm.target.target @@ -43,14 +42,16 @@ def arduino_boards() -> dict: for option in project_options: if option["name"] == "arduino_board": boards = option["choices"] - if option["name"] == "arduino_target": - targets = option["choices"] + if option["name"] == "arduino_model": + models = option["choices"] - arduino_boards = {boards[i]: targets[i] for i in range(len(boards))} + arduino_boards = {boards[i]: models[i] for i in range(len(boards))} return arduino_boards + ARDUINO_BOARDS = arduino_boards() + def pytest_addoption(parser): parser.addoption( "--arduino-board", diff --git a/tests/micro/arduino/test_arduino_rpc_server.py b/tests/micro/arduino/test_arduino_rpc_server.py index 78ddd9a613c8..f157214241c9 100644 --- a/tests/micro/arduino/test_arduino_rpc_server.py +++ b/tests/micro/arduino/test_arduino_rpc_server.py @@ -155,9 +155,7 @@ def test_relay(board, arduino_cli_cmd, tvm_debug, workspace_dir): with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.relay.build(func, target=target) - with _make_session( - model, board, arduino_cli_cmd, workspace_dir, mod, build_config - ) as session: + with _make_session(model, board, arduino_cli_cmd, workspace_dir, mod, build_config) as session: graph_mod = tvm.micro.create_local_graph_executor( mod.get_graph_json(), session.get_system_lib(), session.device ) diff --git a/tests/micro/zephyr/README.md b/tests/micro/zephyr/README.md index 0293d2276c5c..09376e42f8bb 100644 --- a/tests/micro/zephyr/README.md +++ b/tests/micro/zephyr/README.md @@ -32,11 +32,11 @@ device) using: ``` $ cd tvm/tests/micro/zephyr -$ pytest test_zephyr.py --microtvm-device=qemu_x86 # For QEMU emulation -$ pytest test_zephyr.py --microtvm-device=nrf5340dk # For nRF5340DK +$ pytest test_zephyr.py --zephyr-board=qemu_x86 # For QEMU emulation +$ pytest test_zephyr.py --zephyr-board=nrf5340dk_nrf5340_cpuapp # For nRF5340DK ``` -To see the list of supported values for `--microtvm-device`, run: +To see the list of supported values for `--zephyr-board`, run: ``` $ pytest test_zephyr.py --help ``` diff --git a/tests/micro/zephyr/conftest.py b/tests/micro/zephyr/conftest.py index e2ade72eef21..5d1a70911ccd 100644 --- a/tests/micro/zephyr/conftest.py +++ b/tests/micro/zephyr/conftest.py @@ -17,7 +17,6 @@ import datetime import os import pathlib -import sys import pytest @@ -44,14 +43,16 @@ def zephyr_boards() -> dict: for option in project_options: if option["name"] == "zephyr_board": boards = option["choices"] - if option["name"] == "zephyr_target": - targets = option["choices"] + if option["name"] == "zephyr_model": + models = option["choices"] - arduino_boards = {boards[i]: targets[i] for i in range(len(boards))} + arduino_boards = {boards[i]: models[i] for i in range(len(boards))} return arduino_boards + ZEPHYR_BOARDS = zephyr_boards() + def pytest_addoption(parser): parser.addoption( "--zephyr-board", diff --git a/tests/micro/zephyr/test_zephyr.py b/tests/micro/zephyr/test_zephyr.py index 8c843f03b98c..9f5334fb5bd6 100644 --- a/tests/micro/zephyr/test_zephyr.py +++ b/tests/micro/zephyr/test_zephyr.py @@ -143,9 +143,7 @@ def test_basic_add(sess): system_lib.get_function("add")(A_data, B_data, C_data) assert (C_data.numpy() == np.array([7, 8])).all() - with _make_add_sess( - temp_dir, model, board, west_cmd, build_config, dtype="float32" - ) as sess: + with _make_add_sess(temp_dir, model, board, west_cmd, build_config, dtype="float32") as sess: test_basic_add(sess) @@ -381,9 +379,7 @@ def test_tensors(sess): C_data = tvm.nd.array(np.zeros(shape, dtype="int8"), device=sess.device) assert (C_data.numpy() == np.zeros(shape)).all() - with _make_add_sess_with_shape( - temp_dir, model, board, west_cmd, shape, build_config - ) as sess: + with _make_add_sess_with_shape(temp_dir, model, board, west_cmd, shape, build_config) as sess: test_tensors(sess) diff --git a/tests/micro/zephyr/test_zephyr_aot.py b/tests/micro/zephyr/test_zephyr_aot.py index c0fb04668bac..1499d1ef27eb 100644 --- a/tests/micro/zephyr/test_zephyr_aot.py +++ b/tests/micro/zephyr/test_zephyr_aot.py @@ -136,7 +136,13 @@ def _get_message(fd, expr: str, timeout_sec: int): def test_tflite(temp_dir, board, west_cmd, tvm_debug): """Testing a TFLite model.""" - if board not in ["qemu_x86", "mps2_an521", "nrf5340dk", "stm32l4r5zi_nucleo", "zynq_mp_r5"]: + if board not in [ + "qemu_x86", + "mps2_an521", + "nrf5340dk_nrf5340_cpuapp", + "stm32l4r5zi_nucleo", + "zynq_mp_r5", + ]: pytest.skip(msg="Model does not fit.") model = conftest.ZEPHYR_BOARDS[board] diff --git a/tutorials/micro/micro_reference_vm.py b/tutorials/micro/micro_reference_vm.py index a21e4c7a204e..848d13acc391 100644 --- a/tutorials/micro/micro_reference_vm.py +++ b/tutorials/micro/micro_reference_vm.py @@ -143,7 +143,7 @@ .. code-block:: bash $ cd apps/microtvm/reference-vm/zephyr - $ poetry run python3 ../../../../tests/micro/qemu/test_zephyr.py --microtvm-device=stm32f746xx + $ poetry run python3 ../../../../tests/micro/qemu/test_zephyr.py --zephyr-board=stm32f746xx If you do not have physical hardware attached, but wish to run the tests using the local QEMU emulator running within the VM, run the following commands instead: @@ -152,7 +152,7 @@ $ cd /Users/yourusername/path/to/tvm $ cd apps/microtvm/reference-vm/zephyr/ - $ poetry run pytest ../../../../tests/micro/qemu/test_zephyr.py --microtvm-device=qemu_x86 + $ poetry run pytest ../../../../tests/micro/qemu/test_zephyr.py --zephyr-board=qemu_x86 From 3af0e932e1deb99a0b18ad0c67969467a0031a80 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Wed, 8 Sep 2021 12:25:50 -0700 Subject: [PATCH 12/12] fix missed changes based on comments --- apps/microtvm/reference-vm/README.md | 4 ++-- tests/micro/arduino/conftest.py | 2 +- tests/micro/arduino/test_arduino_workflow.py | 2 +- tests/micro/zephyr/conftest.py | 4 +--- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/apps/microtvm/reference-vm/README.md b/apps/microtvm/reference-vm/README.md index d84a36bb450a..a2302ef24c1f 100644 --- a/apps/microtvm/reference-vm/README.md +++ b/apps/microtvm/reference-vm/README.md @@ -78,9 +78,9 @@ $ ./base-box-tool.py --provider virtualbox build zephyr B. Run tests: ```bash - $ ./base-box-tool.py [--provider=PROVIDER] test --microtvm-board=MICROTVM_DEVICE [--test-device-serial=SERIAL] PLATFORM + $ ./base-box-tool.py [--provider=PROVIDER] test --microtvm-board=MICROTVM_BOARD [--test-device-serial=SERIAL] PLATFORM ``` - where MICROTVM_DEVICE is one of the options listed in the + where MICROTVM_BOARD is one of the options listed in the PLATFORM/base-box/test-config.json file. For example: diff --git a/tests/micro/arduino/conftest.py b/tests/micro/arduino/conftest.py index 0ff4a4d98f6d..bb9c69bf4a0e 100644 --- a/tests/micro/arduino/conftest.py +++ b/tests/micro/arduino/conftest.py @@ -58,7 +58,7 @@ def pytest_addoption(parser): nargs="+", required=True, choices=ARDUINO_BOARDS.keys(), - help="MicroTVM device for tests.", + help="Arduino board for tests.", ) parser.addoption( "--arduino-cli-cmd", diff --git a/tests/micro/arduino/test_arduino_workflow.py b/tests/micro/arduino/test_arduino_workflow.py index b080ca2bc9e9..fe6ea8fe3b2e 100644 --- a/tests/micro/arduino/test_arduino_workflow.py +++ b/tests/micro/arduino/test_arduino_workflow.py @@ -29,7 +29,7 @@ This unit test simulates a simple user workflow, where we: 1. Generate a base sketch using a simple audio model 2. Modify the .ino file, much like a user would -3. Compile the sketch for the target device +3. Compile the sketch for the target board -- If physical hardware is present -- 4. Upload the sketch to a connected board 5. Open a serial connection to the board diff --git a/tests/micro/zephyr/conftest.py b/tests/micro/zephyr/conftest.py index 5d1a70911ccd..f7b3b3e31efb 100644 --- a/tests/micro/zephyr/conftest.py +++ b/tests/micro/zephyr/conftest.py @@ -57,9 +57,7 @@ def pytest_addoption(parser): parser.addoption( "--zephyr-board", choices=ZEPHYR_BOARDS.keys(), - help=( - "Specify a microtvm device (i.e. as passed to tvm.target.micro()) for microTVM tests." - ), + help=("Zephyr board for test."), ) parser.addoption( "--west-cmd", default="west", help="Path to `west` command for flashing device."