diff --git a/apps/microtvm/arduino/template_project/microtvm_api_server.py b/apps/microtvm/arduino/template_project/microtvm_api_server.py index 38fc73c5e53a..3d25d0bcad8f 100644 --- a/apps/microtvm/arduino/template_project/microtvm_api_server.py +++ b/apps/microtvm/arduino/template_project/microtvm_api_server.py @@ -57,6 +57,7 @@ class BoardAutodetectFailed(Exception): "package": "arduino", "architecture": "sam", "board": "arduino_due_x_dbg", + "model": "sam3x8e", }, # Due to the way the Feather S2 bootloader works, compilation # behaves fine but uploads cannot be done automatically @@ -64,27 +65,32 @@ class BoardAutodetectFailed(Exception): "package": "esp32", "architecture": "esp32", "board": "feathers2", + "model": "esp32", }, "metrom4": { "package": "adafruit", "architecture": "samd", "board": "adafruit_metro_m4", + "model": "atsamd51", }, # Spresense only works as of its v2.3.0 sdk "spresense": { "package": "SPRESENSE", "architecture": "spresense", "board": "spresense", + "model": "cxd5602gg", }, "nano33ble": { "package": "arduino", "architecture": "mbed_nano", "board": "nano33ble", + "model": "nrf52840", }, "pybadge": { "package": "adafruit", "architecture": "samd", "board": "adafruit_pybadge_m4", + "model": "atsamd51", }, # The Teensy boards are listed here for completeness, but they # won't work until https://github.com/arduino/arduino-cli/issues/700 @@ -93,16 +99,19 @@ class BoardAutodetectFailed(Exception): "package": "teensy", "architecture": "avr", "board": "teensy40", + "model": "imxrt1060", }, "teensy41": { "package": "teensy", "architecture": "avr", "board": "teensy41", + "model": "imxrt1060", }, "wioterminal": { "package": "Seeeduino", "architecture": "samd", "board": "seeed_wio_terminal", + "model": "atsamd51", }, } @@ -114,6 +123,11 @@ class BoardAutodetectFailed(Exception): choices=list(BOARD_PROPERTIES), help="Name of the Arduino board to build for", ), + server.ProjectOption( + "arduino_model", + choices=[board["model"] for _, board in BOARD_PROPERTIES.items()], + help="Name of the model for each Arduino board.", + ), server.ProjectOption("arduino_cli_cmd", help="Path to the arduino-cli tool."), server.ProjectOption("port", help="Port to use for connecting to hardware"), server.ProjectOption( diff --git a/apps/microtvm/reference-vm/README.md b/apps/microtvm/reference-vm/README.md index 9303c0a64ece..a2302ef24c1f 100644 --- a/apps/microtvm/reference-vm/README.md +++ b/apps/microtvm/reference-vm/README.md @@ -78,14 +78,14 @@ $ ./base-box-tool.py --provider virtualbox build zephyr B. Run tests: ```bash - $ ./base-box-tool.py [--provider=PROVIDER] test --microtvm-platform=MICROTVM_PLATFORM [--test-device-serial=SERIAL] PLATFORM + $ ./base-box-tool.py [--provider=PROVIDER] test --microtvm-board=MICROTVM_BOARD [--test-device-serial=SERIAL] PLATFORM ``` - where MICROTVM_PLATFORM is one of the options listed in the + where MICROTVM_BOARD is one of the options listed in the PLATFORM/base-box/test-config.json file. For example: ```base - $ ./base-box-tool.py --provider virtualbox test --microtvm-platform=stm32f746xx_disco zephyr + $ ./base-box-tool.py --provider virtualbox test --microtvm-board=stm32f746xx_disco zephyr ``` This command does the following for the specified provider: diff --git a/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh b/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh index 3d8597f19b64..5c3d96dfc7df 100755 --- a/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh +++ b/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. # -# Usage: base_box_test.sh +# Usage: base_box_test.sh # Execute microTVM Arduino tests. # @@ -24,17 +24,17 @@ set -e set -x if [ "$#" -lt 1 ]; then - echo "Usage: base_box_test.sh " + echo "Usage: base_box_test.sh " exit -1 fi -microtvm_platform=$1 +board=$1 -pytest tests/micro/arduino/test_arduino_workflow.py --microtvm-platforms=${microtvm_platform} +pytest tests/micro/arduino/test_arduino_workflow.py --arduino-board=${board} -if [ $microtvm_platform == "nano33ble" ]; then +if [ $board == "nano33ble" ]; then # https://github.com/apache/tvm/issues/8730 - echo "NOTE: skipped test_arduino_rpc_server.py on $microtvm_platform -- known failure" + echo "NOTE: skipped test_arduino_rpc_server.py on $board -- known failure" else - pytest tests/micro/arduino/test_arduino_rpc_server.py --microtvm-platforms=${microtvm_platform} + pytest tests/micro/arduino/test_arduino_rpc_server.py --arduino-board=${board} fi diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py index f32885433c2b..1682f5e1e032 100755 --- a/apps/microtvm/reference-vm/base-box-tool.py +++ b/apps/microtvm/reference-vm/base-box-tool.py @@ -28,7 +28,6 @@ import subprocess import sys - _LOG = logging.getLogger(__name__) @@ -48,10 +47,19 @@ "zephyr", ) -# List of identifying strings for microTVM platforms for testing. -# Must match PLATFORMS as defined in tvm/tests/micro/[platform]/conftest.py -# TODO add a way to declare supported platforms to ProjectAPI -ALL_MICROTVM_PLATFORMS = { +# Extra scripts required to execute on provisioning +# in [platform]/base-box/base_box_provision.sh +EXTRA_SCRIPTS = { + "arduino": (), + "zephyr": ("docker/install/ubuntu_init_zephyr_project.sh",), +} + +PACKER_FILE_NAME = "packer.json" + + +# List of identifying strings for microTVM boards for testing. +# TODO add a way to declare supported boards to ProjectAPI +ALL_MICROTVM_BOARDS = { "arduino": ( "due", "feathers2", @@ -64,22 +72,13 @@ "wioterminal", ), "zephyr": ( - "stm32f746xx_nucleo", + "nucleo_f746zg", "stm32f746xx_disco", - "nrf5340dk", + "nrf5340dk_nrf5340_cpuapp", "mps2_an521", ), } -# Extra scripts required to execute on provisioning -# in [platform]/base-box/base_box_provision.sh -EXTRA_SCRIPTS = { - "arduino": (), - "zephyr": ("docker/install/ubuntu_init_zephyr_project.sh",), -} - -PACKER_FILE_NAME = "packer.json" - def parse_virtualbox_devices(): output = subprocess.check_output(["VBoxManage", "list", "usbhost"], encoding="utf-8") @@ -362,7 +361,7 @@ def _quote_cmd(cmd): + _quote_cmd( [ f"apps/microtvm/reference-vm/{platform}/base-box/base_box_test.sh", - test_config["microtvm_platform"], + test_config["microtvm_board"], ] ) ) @@ -376,22 +375,22 @@ def test_command(args): with open(test_config_file) as f: test_config = json.load(f) - # select microTVM test platform - microtvm_test_platform = test_config[args.microtvm_platform] + # select microTVM test config + microtvm_test_config = test_config[args.microtvm_board] for key, expected_type in REQUIRED_TEST_CONFIG_KEYS.items(): - assert key in microtvm_test_platform and isinstance( - microtvm_test_platform[key], expected_type + assert key in microtvm_test_config and isinstance( + microtvm_test_config[key], expected_type ), f"Expected key {key} of type {expected_type} in {test_config_file}: {test_config!r}" - microtvm_test_platform["vid_hex"] = microtvm_test_platform["vid_hex"].lower() - microtvm_test_platform["pid_hex"] = microtvm_test_platform["pid_hex"].lower() - microtvm_test_platform["microtvm_platform"] = args.microtvm_platform + microtvm_test_config["vid_hex"] = microtvm_test_config["vid_hex"].lower() + microtvm_test_config["pid_hex"] = microtvm_test_config["pid_hex"].lower() + microtvm_test_config["microtvm_board"] = args.microtvm_board providers = args.provider provider_passed = {p: False for p in providers} - release_test_dir = os.path.join(THIS_DIR, "release-test") + release_test_dir = os.path.join(THIS_DIR, f"release-test-{args.platform}") if args.skip_build: assert len(providers) == 1, "--skip-build was given, but >1 provider specified" @@ -406,7 +405,7 @@ def test_command(args): release_test_dir, args.platform, provider_name, - microtvm_test_platform, + microtvm_test_config, args.test_device_serial, ) provider_passed[provider_name] = True @@ -511,10 +510,10 @@ def parse_args(): platform_specific_parser = parser_test_platform_subparsers.add_parser(platform) platform_specific_parser.set_defaults(platform=platform) platform_specific_parser.add_argument( - "--microtvm-platform", - choices=ALL_MICROTVM_PLATFORMS[platform], + "--microtvm-board", + choices=ALL_MICROTVM_BOARDS[platform], required=True, - help="MicroTVM platfrom used for testing.", + help="MicroTVM board used for testing.", ) # Options for release subcommand diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh index 8eba63e9e331..2a023b520b01 100755 --- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh +++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. # -# Usage: base_box_test.sh +# Usage: base_box_test.sh # Execute microTVM Zephyr tests. # @@ -24,16 +24,16 @@ set -e set -x if [ "$#" -lt 1 ]; then - echo "Usage: base_box_test.sh " + echo "Usage: base_box_test.sh " exit -1 fi -microtvm_platform=$1 +board=$1 -pytest tests/micro/zephyr/test_zephyr.py --microtvm-platforms=${microtvm_platform} +pytest tests/micro/zephyr/test_zephyr.py --zephyr-board=${board} -if [ $microtvm_platform == "stm32f746xx" ]; then - echo "NOTE: skipped test_zephyr_aot.py on $microtvm_platform -- known failure" +if [ $board == "stm32f746xx" ]; then + echo "NOTE: skipped test_zephyr_aot.py on $board -- known failure" else - pytest tests/micro/zephyr/test_zephyr_aot.py --microtvm-platforms=${microtvm_platform} + pytest tests/micro/zephyr/test_zephyr_aot.py --zephyr-board=${board} fi diff --git a/apps/microtvm/zephyr/template_project/microtvm_api_server.py b/apps/microtvm/zephyr/template_project/microtvm_api_server.py index 3fdf962cbc16..08ba1a16e5ec 100644 --- a/apps/microtvm/zephyr/template_project/microtvm_api_server.py +++ b/apps/microtvm/zephyr/template_project/microtvm_api_server.py @@ -57,6 +57,47 @@ IS_TEMPLATE = not (API_SERVER_DIR / MODEL_LIBRARY_FORMAT_RELPATH).exists() +# Data structure to hold the information microtvm_api_server.py needs +# to communicate with each of these boards. +BOARD_PROPERTIES = { + "qemu_x86": { + "board": "qemu_x86", + "model": "host", + }, + "qemu_riscv32": { + "board": "qemu_riscv32", + "model": "host", + }, + "qemu_riscv64": { + "board": "qemu_riscv64", + "model": "host", + }, + "mps2_an521": { + "board": "mps2_an521", + "model": "mps2_an521", + }, + "nrf5340dk_nrf5340_cpuapp": { + "board": "nrf5340dk_nrf5340_cpuapp", + "model": "nrf5340dk", + }, + "stm32f746xx_disco": { + "board": "stm32f746xx_disco", + "model": "stm32f746xx", + }, + "nucleo_f746zg": { + "board": "nucleo_f746zg", + "model": "stm32f746xx", + }, + "nucleo_l4r5zi": { + "board": "nucleo_l4r5zi", + "model": "stm32l4r5zi", + }, + "qemu_cortex_r5": { + "board": "qemu_cortex_r5", + "model": "zynq_mp_r5", + }, +} + def check_call(cmd_args, *args, **kwargs): cwd_str = "" if "cwd" not in kwargs else f" (in cwd: {kwargs['cwd']})" @@ -243,7 +284,16 @@ def _get_nrf_device_args(options): ), ), server.ProjectOption("zephyr_base", help="Path to the zephyr base directory."), - server.ProjectOption("zephyr_board", help="Name of the Zephyr board to build for."), + server.ProjectOption( + "zephyr_board", + choices=list(BOARD_PROPERTIES), + help="Name of the Zephyr board to build for.", + ), + server.ProjectOption( + "zephyr_model", + choices=[board["model"] for _, board in BOARD_PROPERTIES.items()], + help="Name of the model for each Zephyr board.", + ), ] diff --git a/tests/micro/arduino/README.md b/tests/micro/arduino/README.md index 78e63cabb7e2..0b039ba6de7c 100644 --- a/tests/micro/arduino/README.md +++ b/tests/micro/arduino/README.md @@ -22,14 +22,14 @@ all of the appropriate TVM dependencies installed. You can run the test with: ``` $ cd tvm/tests/micro/arduino -$ pytest --microtvm-platforms spresense +$ pytest --arduino-board=spresense ``` Most of these tests require a supported Arduino board to be connected. If you don't want to run these tests, you can pass the flag `--test-build-only` to only test project generation and compilation. -To see the list of supported values for `----microtvm-platforms`, run: +To see the list of supported values for `--arduino-board`, run: ``` $ pytest --help ``` diff --git a/tests/micro/arduino/conftest.py b/tests/micro/arduino/conftest.py index 38870b6b4dfe..bb9c69bf4a0e 100644 --- a/tests/micro/arduino/conftest.py +++ b/tests/micro/arduino/conftest.py @@ -20,22 +20,9 @@ import pytest import tvm.target.target +from tvm.micro import project from tvm import micro, relay -# The models that should pass this configuration. Maps a short, identifying platform string to -# (model, zephyr_board). -PLATFORMS = { - "due": ("sam3x8e", "due"), - "feathers2": ("esp32", "feathers2"), - "metrom4": ("atsamd51", "metrom4"), - "nano33ble": ("nrf52840", "nano33ble"), - "pybadge": ("atsamd51", "pybadge"), - "spresense": ("cxd5602gg", "spresense"), - "teensy40": ("imxrt1060", "teensy40"), - "teensy41": ("imxrt1060", "teensy41"), - "wioterminal": ("atsamd51", "wioterminal"), -} - TEMPLATE_PROJECT_DIR = ( pathlib.Path(__file__).parent / ".." @@ -48,13 +35,30 @@ ).resolve() +def arduino_boards() -> dict: + """Returns a dict mapping board to target model""" + template = project.TemplateProject.from_directory(TEMPLATE_PROJECT_DIR) + project_options = template.info()["project_options"] + for option in project_options: + if option["name"] == "arduino_board": + boards = option["choices"] + if option["name"] == "arduino_model": + models = option["choices"] + + arduino_boards = {boards[i]: models[i] for i in range(len(boards))} + return arduino_boards + + +ARDUINO_BOARDS = arduino_boards() + + def pytest_addoption(parser): parser.addoption( - "--microtvm-platforms", + "--arduino-board", nargs="+", required=True, - choices=PLATFORMS.keys(), - help="Target platforms for microTVM tests.", + choices=ARDUINO_BOARDS.keys(), + help="Arduino board for tests.", ) parser.addoption( "--arduino-cli-cmd", @@ -92,8 +96,8 @@ def pytest_collection_modifyitems(config, items): # (to take advantage of multiple cores / external memory / etc.), so all tests # are parameterized by board def pytest_generate_tests(metafunc): - platforms = metafunc.config.getoption("microtvm_platforms") - metafunc.parametrize("platform", platforms, scope="session") + board = metafunc.config.getoption("arduino_board") + metafunc.parametrize("board", board, scope="session") @pytest.fixture(scope="session") @@ -106,12 +110,11 @@ def tvm_debug(request): return request.config.getoption("--tvm-debug") -def make_workspace_dir(test_name, platform): - _, arduino_board = PLATFORMS[platform] +def make_workspace_dir(test_name, board): filepath = pathlib.Path(__file__) board_workspace = ( filepath.parent - / f"workspace_{test_name}_{arduino_board}" + / f"workspace_{test_name}_{board}" / datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S") ) @@ -125,9 +128,9 @@ def make_workspace_dir(test_name, platform): return t -def make_kws_project(platform, arduino_cli_cmd, tvm_debug, workspace_dir): +def make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir): this_dir = pathlib.Path(__file__).parent - model, arduino_board = PLATFORMS[platform] + model = ARDUINO_BOARDS[board] build_config = {"debug": tvm_debug} with open(this_dir.parent / "testdata" / "kws" / "yes_no.tflite", "rb") as f: @@ -156,7 +159,7 @@ def make_kws_project(platform, arduino_cli_cmd, tvm_debug, workspace_dir): mod, workspace_dir / "project", { - "arduino_board": arduino_board, + "arduino_board": board, "arduino_cli_cmd": arduino_cli_cmd, "project_type": "example_project", "verbose": bool(build_config.get("debug")), diff --git a/tests/micro/arduino/test_arduino_error_detection.py b/tests/micro/arduino/test_arduino_error_detection.py index 1789fff2e7bb..64e2c14d1c18 100644 --- a/tests/micro/arduino/test_arduino_error_detection.py +++ b/tests/micro/arduino/test_arduino_error_detection.py @@ -27,13 +27,13 @@ # A new project and workspace dir is created for EVERY test @pytest.fixture -def workspace_dir(request, platform): - return conftest.make_workspace_dir("arduino_error_detection", platform) +def workspace_dir(request, board): + return conftest.make_workspace_dir("arduino_error_detection", board) @pytest.fixture -def project(platform, arduino_cli_cmd, tvm_debug, workspace_dir): - return conftest.make_kws_project(platform, arduino_cli_cmd, tvm_debug, workspace_dir) +def project(board, arduino_cli_cmd, tvm_debug, workspace_dir): + return conftest.make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir) def test_blank_project_compiles(workspace_dir, project): diff --git a/tests/micro/arduino/test_arduino_rpc_server.py b/tests/micro/arduino/test_arduino_rpc_server.py index 57ebbc605197..f157214241c9 100644 --- a/tests/micro/arduino/test_arduino_rpc_server.py +++ b/tests/micro/arduino/test_arduino_rpc_server.py @@ -36,11 +36,10 @@ import conftest - # # A new project and workspace dir is created for EVERY test @pytest.fixture -def workspace_dir(platform): - return conftest.make_workspace_dir("arduino_rpc_server", platform) +def workspace_dir(board): + return conftest.make_workspace_dir("arduino_rpc_server", board) def _make_session(model, arduino_board, arduino_cli_cmd, workspace_dir, mod, build_config): @@ -83,10 +82,10 @@ def _make_add_sess(model, arduino_board, arduino_cli_cmd, workspace_dir, build_c # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_compile_runtime(platform, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_compile_runtime(board, arduino_cli_cmd, tvm_debug, workspace_dir): """Test compiling the on-device runtime.""" - model, arduino_board = conftest.PLATFORMS[platform] + model = conftest.ARDUINO_BOARDS[board] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -102,16 +101,16 @@ def test_basic_add(sess): system_lib.get_function("add")(A_data, B_data, C_data) assert (C_data.numpy() == np.array([6, 7])).all() - with _make_add_sess(model, arduino_board, arduino_cli_cmd, workspace_dir, build_config) as sess: + with _make_add_sess(model, board, arduino_cli_cmd, workspace_dir, build_config) as sess: test_basic_add(sess) @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_platform_timer(platform, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_platform_timer(board, arduino_cli_cmd, tvm_debug, workspace_dir): """Test compiling the on-device runtime.""" - model, arduino_board = conftest.PLATFORMS[platform] + model = conftest.ARDUINO_BOARDS[board] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -132,15 +131,15 @@ def test_basic_add(sess): assert result.mean > 0 assert len(result.results) == 3 - with _make_add_sess(model, arduino_board, arduino_cli_cmd, workspace_dir, build_config) as sess: + with _make_add_sess(model, board, arduino_cli_cmd, workspace_dir, build_config) as sess: test_basic_add(sess) @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_relay(platform, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_relay(board, arduino_cli_cmd, tvm_debug, workspace_dir): """Testing a simple relay graph""" - model, arduino_board = conftest.PLATFORMS[platform] + model = conftest.ARDUINO_BOARDS[board] build_config = {"debug": tvm_debug} shape = (10,) @@ -156,9 +155,7 @@ def test_relay(platform, arduino_cli_cmd, tvm_debug, workspace_dir): with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.relay.build(func, target=target) - with _make_session( - model, arduino_board, arduino_cli_cmd, workspace_dir, mod, build_config - ) as session: + with _make_session(model, board, arduino_cli_cmd, workspace_dir, mod, build_config) as session: graph_mod = tvm.micro.create_local_graph_executor( mod.get_graph_json(), session.get_system_lib(), session.device ) @@ -172,9 +169,9 @@ def test_relay(platform, arduino_cli_cmd, tvm_debug, workspace_dir): @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_onnx(platform, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_onnx(board, arduino_cli_cmd, tvm_debug, workspace_dir): """Testing a simple ONNX model.""" - model, arduino_board = conftest.PLATFORMS[platform] + model = conftest.ARDUINO_BOARDS[board] build_config = {"debug": tvm_debug} # Load test images. @@ -200,7 +197,7 @@ def test_onnx(platform, arduino_cli_cmd, tvm_debug, workspace_dir): graph = lowered.get_graph_json() with _make_session( - model, arduino_board, arduino_cli_cmd, workspace_dir, lowered, build_config + model, board, arduino_cli_cmd, workspace_dir, lowered, build_config ) as session: graph_mod = tvm.micro.create_local_graph_executor( graph, session.get_system_lib(), session.device @@ -260,9 +257,9 @@ def check_result( @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_byoc_microtvm(platform, arduino_cli_cmd, tvm_debug, workspace_dir): +def test_byoc_microtvm(board, arduino_cli_cmd, tvm_debug, workspace_dir): """This is a simple test case to check BYOC capabilities of microTVM""" - model, arduino_board = conftest.PLATFORMS[platform] + model = conftest.ARDUINO_BOARDS[board] build_config = {"debug": tvm_debug} x = relay.var("x", shape=(10, 10)) @@ -317,7 +314,7 @@ def test_byoc_microtvm(platform, arduino_cli_cmd, tvm_debug, workspace_dir): ), model=model, build_config=build_config, - arduino_board=arduino_board, + arduino_board=board, arduino_cli_cmd=arduino_cli_cmd, workspace_dir=workspace_dir, ) @@ -344,9 +341,9 @@ def _make_add_sess_with_shape( ) @tvm.testing.requires_micro @pytest.mark.requires_hardware -def test_rpc_large_array(platform, arduino_cli_cmd, tvm_debug, workspace_dir, shape): +def test_rpc_large_array(board, arduino_cli_cmd, tvm_debug, workspace_dir, shape): """Test large RPC array transfer.""" - model, arduino_board = conftest.PLATFORMS[platform] + model = conftest.ARDUINO_BOARDS[board] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -359,7 +356,7 @@ def test_tensors(sess): assert (C_data.numpy() == np.zeros(shape)).all() with _make_add_sess_with_shape( - model, arduino_board, arduino_cli_cmd, workspace_dir, shape, build_config + model, board, arduino_cli_cmd, workspace_dir, shape, build_config ) as sess: test_tensors(sess) diff --git a/tests/micro/arduino/test_arduino_workflow.py b/tests/micro/arduino/test_arduino_workflow.py index cd97aa37b5a4..fe6ea8fe3b2e 100644 --- a/tests/micro/arduino/test_arduino_workflow.py +++ b/tests/micro/arduino/test_arduino_workflow.py @@ -29,7 +29,7 @@ This unit test simulates a simple user workflow, where we: 1. Generate a base sketch using a simple audio model 2. Modify the .ino file, much like a user would -3. Compile the sketch for the target platform +3. Compile the sketch for the target board -- If physical hardware is present -- 4. Upload the sketch to a connected board 5. Open a serial connection to the board @@ -40,8 +40,8 @@ # Since these tests are sequential, we'll use the same project/workspace # directory for all tests in this file @pytest.fixture(scope="module") -def workspace_dir(request, platform): - return conftest.make_workspace_dir("arduino_workflow", platform) +def workspace_dir(request, board): + return conftest.make_workspace_dir("arduino_workflow", board) @pytest.fixture(scope="module") @@ -51,8 +51,8 @@ def project_dir(workspace_dir): # We MUST pass workspace_dir, not project_dir, or the workspace will be dereferenced too soon @pytest.fixture(scope="module") -def project(platform, arduino_cli_cmd, tvm_debug, workspace_dir): - return conftest.make_kws_project(platform, arduino_cli_cmd, tvm_debug, workspace_dir) +def project(board, arduino_cli_cmd, tvm_debug, workspace_dir): + return conftest.make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir) def _get_directory_elements(directory): diff --git a/tests/micro/zephyr/README.md b/tests/micro/zephyr/README.md index 9769cae2b53b..09376e42f8bb 100644 --- a/tests/micro/zephyr/README.md +++ b/tests/micro/zephyr/README.md @@ -32,11 +32,11 @@ device) using: ``` $ cd tvm/tests/micro/zephyr -$ pytest test_zephyr.py --microtvm-platforms=host # For QEMU emulation -$ pytest test_zephyr.py --microtvm-platforms=nrf5340dk # For nRF5340DK +$ pytest test_zephyr.py --zephyr-board=qemu_x86 # For QEMU emulation +$ pytest test_zephyr.py --zephyr-board=nrf5340dk_nrf5340_cpuapp # For nRF5340DK ``` -To see the list of supported values for `--microtvm-platforms`, run: +To see the list of supported values for `--zephyr-board`, run: ``` $ pytest test_zephyr.py --help ``` diff --git a/tests/micro/zephyr/conftest.py b/tests/micro/zephyr/conftest.py index cfdb208c92b8..f7b3b3e31efb 100644 --- a/tests/micro/zephyr/conftest.py +++ b/tests/micro/zephyr/conftest.py @@ -20,33 +20,44 @@ import pytest +from tvm.micro import project import tvm.contrib.utils import tvm.target.target -# The models that should pass this configuration. Maps a short, identifying platform string to -# (model, zephyr_board). -PLATFORMS = { - "qemu_x86": ("host", "qemu_x86"), - "qemu_riscv32": ("host", "qemu_riscv32"), - "qemu_riscv64": ("host", "qemu_riscv64"), - "mps2_an521": ("mps2_an521", "mps2_an521"), - "nrf5340dk": ("nrf5340dk", "nrf5340dk_nrf5340_cpuapp"), - "stm32f746xx_disco": ("stm32f746xx", "stm32f746g_disco"), - "stm32f746xx_nucleo": ("stm32f746xx", "nucleo_f746zg"), - "stm32l4r5zi_nucleo": ("stm32l4r5zi", "nucleo_l4r5zi"), - "zynq_mp_r5": ("zynq_mp_r5", "qemu_cortex_r5"), -} +TEMPLATE_PROJECT_DIR = ( + pathlib.Path(__file__).parent + / ".." + / ".." + / ".." + / "apps" + / "microtvm" + / "zephyr" + / "template_project" +).resolve() + + +def zephyr_boards() -> dict: + """Returns a dict mapping board to target model""" + template = project.TemplateProject.from_directory(TEMPLATE_PROJECT_DIR) + project_options = template.info()["project_options"] + for option in project_options: + if option["name"] == "zephyr_board": + boards = option["choices"] + if option["name"] == "zephyr_model": + models = option["choices"] + + arduino_boards = {boards[i]: models[i] for i in range(len(boards))} + return arduino_boards + + +ZEPHYR_BOARDS = zephyr_boards() def pytest_addoption(parser): parser.addoption( - "--microtvm-platforms", - default="qemu_x86", - choices=PLATFORMS.keys(), - help=( - "Specify a comma-separated list of test models (i.e. as passed to tvm.target.micro()) " - "for microTVM tests." - ), + "--zephyr-board", + choices=ZEPHYR_BOARDS.keys(), + help=("Zephyr board for test."), ) parser.addoption( "--west-cmd", default="west", help="Path to `west` command for flashing device." @@ -60,8 +71,8 @@ def pytest_addoption(parser): def pytest_generate_tests(metafunc): - if "platform" in metafunc.fixturenames: - metafunc.parametrize("platform", metafunc.config.getoption("microtvm_platforms").split(",")) + if "board" in metafunc.fixturenames: + metafunc.parametrize("board", [metafunc.config.getoption("zephyr_board")]) @pytest.fixture @@ -75,13 +86,12 @@ def tvm_debug(request): @pytest.fixture -def temp_dir(platform): - _, zephyr_board = PLATFORMS[platform] +def temp_dir(board): parent_dir = pathlib.Path(os.path.dirname(__file__)) filename = os.path.splitext(os.path.basename(__file__))[0] board_workspace = ( parent_dir - / f"workspace_{filename}_{zephyr_board}" + / f"workspace_{filename}_{board}" / datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S") ) board_workspace_base = str(board_workspace) diff --git a/tests/micro/zephyr/test_zephyr.py b/tests/micro/zephyr/test_zephyr.py index 5a7e69e3c7f9..9f5334fb5bd6 100644 --- a/tests/micro/zephyr/test_zephyr.py +++ b/tests/micro/zephyr/test_zephyr.py @@ -46,8 +46,6 @@ _LOG = logging.getLogger(__name__) -PLATFORMS = conftest.PLATFORMS - def _make_sess_from_op( temp_dir, model, zephyr_board, west_cmd, op_name, sched, arg_bufs, build_config @@ -60,21 +58,9 @@ def _make_sess_from_op( return _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config) -TEMPLATE_PROJECT_DIR = ( - pathlib.Path(__file__).parent - / ".." - / ".." - / ".." - / "apps" - / "microtvm" - / "zephyr" - / "template_project" -).resolve() - - def _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config): project = tvm.micro.generate_project( - str(TEMPLATE_PROJECT_DIR), + str(conftest.TEMPLATE_PROJECT_DIR), mod, temp_dir / "project", { @@ -101,10 +87,10 @@ def _make_add_sess(temp_dir, model, zephyr_board, west_cmd, build_config, dtype= # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro -def test_add_uint(temp_dir, platform, west_cmd, tvm_debug): +def test_add_uint(temp_dir, board, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" - model, zephyr_board = PLATFORMS[platform] + model = conftest.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -120,12 +106,12 @@ def test_basic_add(sess): system_lib.get_function("add")(A_data, B_data, C_data) assert (C_data.numpy() == np.array([6, 7])).all() - with _make_add_sess(temp_dir, model, zephyr_board, west_cmd, build_config) as sess: + with _make_add_sess(temp_dir, model, board, west_cmd, build_config) as sess: test_basic_add(sess) def has_fpu(zephyr_board): - sys.path.insert(0, str(TEMPLATE_PROJECT_DIR)) + sys.path.insert(0, str(conftest.TEMPLATE_PROJECT_DIR)) try: import microtvm_api_server finally: @@ -136,11 +122,11 @@ def has_fpu(zephyr_board): # The same test code can be executed on both the QEMU simulation and on real hardware. @tvm.testing.requires_micro -def test_add_float(temp_dir, platform, west_cmd, tvm_debug): +def test_add_float(temp_dir, board, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" - model, zephyr_board = PLATFORMS[platform] - if not has_fpu(zephyr_board): - pytest.skip(f"FPU not enabled for {platform}") + model = conftest.ZEPHYR_BOARDS[board] + if not has_fpu(board): + pytest.skip(f"FPU not enabled for {board}") build_config = {"debug": tvm_debug} @@ -157,17 +143,15 @@ def test_basic_add(sess): system_lib.get_function("add")(A_data, B_data, C_data) assert (C_data.numpy() == np.array([7, 8])).all() - with _make_add_sess( - temp_dir, model, zephyr_board, west_cmd, build_config, dtype="float32" - ) as sess: + with _make_add_sess(temp_dir, model, board, west_cmd, build_config, dtype="float32") as sess: test_basic_add(sess) @tvm.testing.requires_micro -def test_platform_timer(temp_dir, platform, west_cmd, tvm_debug): +def test_platform_timer(temp_dir, board, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" - model, zephyr_board = PLATFORMS[platform] + model = conftest.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -188,14 +172,14 @@ def test_basic_add(sess): assert result.mean > 0 assert len(result.results) == 3 - with _make_add_sess(temp_dir, model, zephyr_board, west_cmd, build_config) as sess: + with _make_add_sess(temp_dir, model, board, west_cmd, build_config) as sess: test_basic_add(sess) @tvm.testing.requires_micro -def test_relay(temp_dir, platform, west_cmd, tvm_debug): +def test_relay(temp_dir, board, west_cmd, tvm_debug): """Testing a simple relay graph""" - model, zephyr_board = PLATFORMS[platform] + model = conftest.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} shape = (10,) dtype = "int8" @@ -211,7 +195,7 @@ def test_relay(temp_dir, platform, west_cmd, tvm_debug): with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): mod = tvm.relay.build(ir_mod, target=target) - with _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config) as session: + with _make_session(temp_dir, board, west_cmd, mod, build_config) as session: graph_mod = tvm.micro.create_local_graph_executor( mod.get_graph_json(), session.get_system_lib(), session.device ) @@ -224,9 +208,9 @@ def test_relay(temp_dir, platform, west_cmd, tvm_debug): @tvm.testing.requires_micro -def test_onnx(temp_dir, platform, west_cmd, tvm_debug): +def test_onnx(temp_dir, board, west_cmd, tvm_debug): """Testing a simple ONNX model.""" - model, zephyr_board = PLATFORMS[platform] + model = conftest.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} this_dir = pathlib.Path(os.path.dirname(__file__)) @@ -254,7 +238,7 @@ def test_onnx(temp_dir, platform, west_cmd, tvm_debug): lowered = relay.build(relay_mod, target, params=params) graph = lowered.get_graph_json() - with _make_session(temp_dir, zephyr_board, west_cmd, lowered, build_config) as session: + with _make_session(temp_dir, board, west_cmd, lowered, build_config) as session: graph_mod = tvm.micro.create_local_graph_executor( graph, session.get_system_lib(), session.device ) @@ -301,9 +285,9 @@ def check_result( @tvm.testing.requires_micro -def test_byoc_microtvm(temp_dir, platform, west_cmd, tvm_debug): +def test_byoc_microtvm(temp_dir, board, west_cmd, tvm_debug): """This is a simple test case to check BYOC capabilities of microTVM""" - model, zephyr_board = PLATFORMS[platform] + model = conftest.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} x = relay.var("x", shape=(10, 10)) w0 = relay.var("w0", shape=(10, 10)) @@ -357,7 +341,7 @@ def test_byoc_microtvm(temp_dir, platform, west_cmd, tvm_debug): axis=0, ), model=model, - zephyr_board=zephyr_board, + zephyr_board=board, west_cmd=west_cmd, build_config=build_config, ) @@ -381,9 +365,9 @@ def _make_add_sess_with_shape(temp_dir, model, zephyr_board, west_cmd, shape, bu ], ) @tvm.testing.requires_micro -def test_rpc_large_array(temp_dir, platform, west_cmd, tvm_debug, shape): +def test_rpc_large_array(temp_dir, board, west_cmd, tvm_debug, shape): """Test large RPC array transfer.""" - model, zephyr_board = PLATFORMS[platform] + model = conftest.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -395,9 +379,7 @@ def test_tensors(sess): C_data = tvm.nd.array(np.zeros(shape, dtype="int8"), device=sess.device) assert (C_data.numpy() == np.zeros(shape)).all() - with _make_add_sess_with_shape( - temp_dir, model, zephyr_board, west_cmd, shape, build_config - ) as sess: + with _make_add_sess_with_shape(temp_dir, model, board, west_cmd, shape, build_config) as sess: test_tensors(sess) diff --git a/tests/micro/zephyr/test_zephyr_aot.py b/tests/micro/zephyr/test_zephyr_aot.py index 37aa0f76a852..1499d1ef27eb 100644 --- a/tests/micro/zephyr/test_zephyr_aot.py +++ b/tests/micro/zephyr/test_zephyr_aot.py @@ -42,8 +42,6 @@ _LOG = logging.getLogger(__name__) -PLATFORMS = conftest.PLATFORMS - def _build_project(temp_dir, zephyr_board, west_cmd, mod, build_config, extra_files_tar=None): template_project_dir = ( @@ -135,13 +133,19 @@ def _get_message(fd, expr: str, timeout_sec: int): @tvm.testing.requires_micro -def test_tflite(temp_dir, platform, west_cmd, tvm_debug): +def test_tflite(temp_dir, board, west_cmd, tvm_debug): """Testing a TFLite model.""" - if platform not in ["qemu_x86", "mps2_an521", "nrf5340dk", "stm32l4r5zi_nucleo", "zynq_mp_r5"]: + if board not in [ + "qemu_x86", + "mps2_an521", + "nrf5340dk_nrf5340_cpuapp", + "stm32l4r5zi_nucleo", + "zynq_mp_r5", + ]: pytest.skip(msg="Model does not fit.") - model, zephyr_board = PLATFORMS[platform] + model = conftest.ZEPHYR_BOARDS[board] input_shape = (1, 32, 32, 3) output_shape = (1, 10) build_config = {"debug": tvm_debug} @@ -195,7 +199,7 @@ def test_tflite(temp_dir, platform, west_cmd, tvm_debug): project, _ = _build_project( temp_dir, - zephyr_board, + board, west_cmd, lowered, build_config, @@ -218,12 +222,12 @@ def test_tflite(temp_dir, platform, west_cmd, tvm_debug): @tvm.testing.requires_micro -def test_qemu_make_fail(temp_dir, platform, west_cmd, tvm_debug): +def test_qemu_make_fail(temp_dir, board, west_cmd, tvm_debug): """Testing QEMU make fail.""" - if platform not in ["qemu_x86", "mps2_an521"]: + if board not in ["qemu_x86", "mps2_an521"]: pytest.skip(msg="Only for QEMU targets.") - model, zephyr_board = PLATFORMS[platform] + model = conftest.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} shape = (10,) dtype = "float32" @@ -254,7 +258,7 @@ def test_qemu_make_fail(temp_dir, platform, west_cmd, tvm_debug): project, project_dir = _build_project( temp_dir, - zephyr_board, + board, west_cmd, lowered, build_config, diff --git a/tests/scripts/task_python_microtvm.sh b/tests/scripts/task_python_microtvm.sh index ff1b50e61d41..4bee8d566f11 100755 --- a/tests/scripts/task_python_microtvm.sh +++ b/tests/scripts/task_python_microtvm.sh @@ -23,11 +23,11 @@ set -x # NOTE(areusch): Adding to diagnose flaky timeouts source tests/scripts/setup-pytest-env.sh make cython3 -run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-platforms=qemu_x86 +run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --zephyr-board=qemu_x86 # Temporarily removing mps2_an512 from CI due to issue 8728: # https://github.com/apache/tvm/issues/8728 -# run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --microtvm-platforms=mps2_an521 +# run_pytest ctypes python-microtvm-zephyr tests/micro/zephyr --zephyr-board=mps2_an521 run_pytest ctypes python-microtvm-arduino apps/microtvm/arduino/template_project/tests -run_pytest ctypes python-microtvm-arduino-nano33ble tests/micro/arduino --test-build-only --microtvm-platforms=nano33ble -run_pytest ctypes python-microtvm-arduino-due tests/micro/arduino --test-build-only --microtvm-platforms=due +run_pytest ctypes python-microtvm-arduino-nano33ble tests/micro/arduino --test-build-only --arduino-board=nano33ble +run_pytest ctypes python-microtvm-arduino-due tests/micro/arduino --test-build-only --arduino-board=due diff --git a/tutorials/micro/micro_reference_vm.py b/tutorials/micro/micro_reference_vm.py index e2b75ea42a1f..848d13acc391 100644 --- a/tutorials/micro/micro_reference_vm.py +++ b/tutorials/micro/micro_reference_vm.py @@ -143,7 +143,7 @@ .. code-block:: bash $ cd apps/microtvm/reference-vm/zephyr - $ poetry run python3 ../../../../tests/micro/qemu/test_zephyr.py --microtvm-platforms=stm32f746xx + $ poetry run python3 ../../../../tests/micro/qemu/test_zephyr.py --zephyr-board=stm32f746xx If you do not have physical hardware attached, but wish to run the tests using the local QEMU emulator running within the VM, run the following commands instead: @@ -152,7 +152,7 @@ $ cd /Users/yourusername/path/to/tvm $ cd apps/microtvm/reference-vm/zephyr/ - $ poetry run pytest ../../../../tests/micro/qemu/test_zephyr.py --microtvm-platforms=host + $ poetry run pytest ../../../../tests/micro/qemu/test_zephyr.py --zephyr-board=qemu_x86