From 78f92f87e2eed0fc21781d1329d32b058462db51 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Thu, 27 May 2021 15:59:54 -0700
Subject: [PATCH 01/40] num of cores

---
 docker/install/ubuntu_install_qemu.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/docker/install/ubuntu_install_qemu.sh b/docker/install/ubuntu_install_qemu.sh
index 0adc81b207d7..93cbabb17ce1 100755
--- a/docker/install/ubuntu_install_qemu.sh
+++ b/docker/install/ubuntu_install_qemu.sh
@@ -16,6 +16,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
+if [[ "${CI_NUM_CORE}" ]]; then
+  NUM_CORE=${CI_NUM_CORE}
+else
+  NUM_CORE=2
+fi
+
 set -e
 set -u
 set -o pipefail
@@ -45,7 +51,7 @@ gpg --verify qemu-5.1.0.tar.xz.sig
 tar -xf qemu-5.1.0.tar.xz
 cd qemu-5.1.0
 ./configure --target-list=aarch64-softmmu,arm-softmmu,i386-softmmu,riscv32-softmmu,riscv64-softmmu,x86_64-softmmu
-make -j2
+make -j${NUM_CORE}
 sudo make install
 
 # For debugging with qemu

From c2c8d748ff9232fec0ec34ace47d7f8e280d2148 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Fri, 28 May 2021 09:50:11 -0700
Subject: [PATCH 02/40] add target list

---
 docker/install/ubuntu_install_qemu.sh | 58 +++++++++++++++++++--------
 1 file changed, 42 insertions(+), 16 deletions(-)

diff --git a/docker/install/ubuntu_install_qemu.sh b/docker/install/ubuntu_install_qemu.sh
index 93cbabb17ce1..11e30d492376 100755
--- a/docker/install/ubuntu_install_qemu.sh
+++ b/docker/install/ubuntu_install_qemu.sh
@@ -16,22 +16,46 @@
 # specific language governing permissions and limitations
 # under the License.
 
+#
+# Install QEMU on Ubuntu.
+#
+# Usage: ubuntu_install_qemu.sh [--target-list target0,target1,...]
+#   --target-list is list of target for QEMU comma seperated. e.g. aarch64-softmmu,arm-softmmu,...
+#
+
+set -e
+set -o pipefail
+
+QEMU_DIR=qemu-5.1.0
+
+# Get number of cores for build
 if [[ "${CI_NUM_CORE}" ]]; then
-  NUM_CORE=${CI_NUM_CORE}
+  num_cores=${CI_NUM_CORE}
 else
-  NUM_CORE=2
+  num_cores=2
 fi
 
-set -e
-set -u
-set -o pipefail
+# Set target list for QEMU
+if [ "$1" == "--target-list" ]; then
+    shift
+    target_list=$1
+else
+    target_list="aarch64-softmmu,arm-softmmu,i386-softmmu,riscv32-softmmu,riscv64-softmmu,x86_64-softmmu"
+fi
 
-sudo sed -i '/deb-src/s/^# //' /etc/apt/sources.list
-apt update
-apt-get -y build-dep qemu
+# Check if QEMU already built
+rebuild=0
+if [ -d ${QEMU_DIR} ]; then
+    rebuild=1
+fi
+
+if [ $rebuild -eq 0 ]; then
+    sudo sed -i '/deb-src/s/^# //' /etc/apt/sources.list
+    apt update
+    apt-get -y build-dep qemu
 
-gpg --keyserver keys.gnupg.net --recv-keys 0x3353C9CEF108B584
-cat <<EOF | gpg --dearmor >qemu-5.1.0.tar.xz.sig
+    gpg --keyserver keys.gnupg.net --recv-keys 0x3353C9CEF108B584
+    cat <<EOF | gpg --dearmor >qemu-5.1.0.tar.xz.sig
 -----BEGIN PGP ARMORED FILE-----
 Comment: Use "gpg --dearmor" for unpacking
 
@@ -45,14 +69,16 @@ p5ez/+2k4VAIwIQoP5DoO06waLBffvLIAdPPKYsx71K67OoGG2svc7duC/+5qf1x
 =hCS7
 -----END PGP ARMORED FILE-----
 EOF
-curl -OLs https://download.qemu.org/qemu-5.1.0.tar.xz
-gpg --verify qemu-5.1.0.tar.xz.sig
+    curl -OLs https://download.qemu.org/qemu-5.1.0.tar.xz
+    gpg --verify qemu-5.1.0.tar.xz.sig
+
+    tar -xf qemu-5.1.0.tar.xz
+fi
 
-tar -xf qemu-5.1.0.tar.xz
 cd qemu-5.1.0
-./configure --target-list=aarch64-softmmu,arm-softmmu,i386-softmmu,riscv32-softmmu,riscv64-softmmu,x86_64-softmmu
-make -j${NUM_CORE}
+./configure --target-list=${target_list}
+make -j${num_cores}
 sudo make install
 
-# For debugging with qemu
+# # For debugging with qemu
 apt-get -y install libpython3.8

From 30a2d794384e027383809baafb617e53a753790c Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Fri, 28 May 2021 11:15:04 -0700
Subject: [PATCH 03/40] extension

---
 apps/microtvm/reference-vm/base-box-tool.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py
index fb7a9c0b5ce6..46a14dabf01e 100755
--- a/apps/microtvm/reference-vm/base-box-tool.py
+++ b/apps/microtvm/reference-vm/base-box-tool.py
@@ -202,7 +202,7 @@ def generate_packer_config(file_path, providers):
 
 def build_command(args):
     generate_packer_config(
-        os.path.join(THIS_DIR, args.platform, "base-box", "packer.json"),
+        os.path.join(THIS_DIR, args.platform, "base-box", "packer.pkr.json"),
         args.provider or ALL_PROVIDERS,
     )
     env = copy.copy(os.environ)
@@ -212,7 +212,7 @@ def build_command(args):
     if args.debug_packer:
         packer_args += ["-debug"]
 
-    packer_args += ["packer.json"]
+    packer_args += ["packer.pkr.json"]
     subprocess.check_call(
         packer_args, cwd=os.path.join(THIS_DIR, args.platform, "base-box"), env=env
     )
@@ -407,7 +407,6 @@ def parse_args():
         "--provider",
         choices=ALL_PROVIDERS,
         action="append",
-        default=list(ALL_PROVIDERS),
         help="Name of the provider or providers to act on; if not specified, act on all.",
     )
 

From 6750395dceaa3610bddf53638972f4516e4afe2d Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Fri, 28 May 2021 11:15:46 -0700
Subject: [PATCH 04/40] qemu

---
 apps/microtvm/reference-vm/zephyr/setup.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/apps/microtvm/reference-vm/zephyr/setup.sh b/apps/microtvm/reference-vm/zephyr/setup.sh
index e0f382cfc23e..470b2e52170b 100644
--- a/apps/microtvm/reference-vm/zephyr/setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/setup.sh
@@ -24,8 +24,11 @@ cd "${TVM_HOME}"
 
 apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
 
-cd apps/microtvm/reference-vm/zephyr
+# Install QEMU
+sudo docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu
 
+# Build poetry
+cd apps/microtvm/reference-vm/zephyr
 poetry env use 3.6
 # NOTE: due to https://github.com/python-poetry/poetry/issues/2247, download torch here.
 poetry run pip3 install torch==1.4.0 torchvision==0.5.0

From b2ad212835cba9655f3992f0e1bac1f0d8364056 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Fri, 28 May 2021 11:26:11 -0700
Subject: [PATCH 05/40] fix

---
 apps/microtvm/reference-vm/base-box-tool.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py
index 46a14dabf01e..8221b9f97526 100755
--- a/apps/microtvm/reference-vm/base-box-tool.py
+++ b/apps/microtvm/reference-vm/base-box-tool.py
@@ -201,8 +201,9 @@ def generate_packer_config(file_path, providers):
 
 
 def build_command(args):
+    packer_file_name = "packer.json"
     generate_packer_config(
-        os.path.join(THIS_DIR, args.platform, "base-box", "packer.pkr.json"),
+        os.path.join(THIS_DIR, args.platform, "base-box", packer_file_name),
         args.provider or ALL_PROVIDERS,
     )
     env = copy.copy(os.environ)
@@ -212,7 +213,7 @@ def build_command(args):
     if args.debug_packer:
         packer_args += ["-debug"]
 
-    packer_args += ["packer.pkr.json"]
+    packer_args += [packer_file_name]
     subprocess.check_call(
         packer_args, cwd=os.path.join(THIS_DIR, args.platform, "base-box"), env=env
     )

From 560cce0c296bb3aff5f2e18d174da6eab56dbd85 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Fri, 28 May 2021 14:50:17 -0700
Subject: [PATCH 06/40] comments

---
 docker/install/ubuntu_install_qemu.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docker/install/ubuntu_install_qemu.sh b/docker/install/ubuntu_install_qemu.sh
index 11e30d492376..be53e5aa388f 100755
--- a/docker/install/ubuntu_install_qemu.sh
+++ b/docker/install/ubuntu_install_qemu.sh
@@ -29,8 +29,8 @@ set -o pipefail
 QEMU_DIR=qemu-5.1.0
 
 # Get number of cores for build
-if [[ "${CI_NUM_CORE}" ]]; then
-  num_cores=${CI_NUM_CORE}
+if [[ "${TVM_CI_NUM_CORES}" ]]; then
+  num_cores=${TVM_CI_NUM_CORES}
 else
   num_cores=2
 fi
@@ -40,6 +40,7 @@ if [ "$1" == "--target-list" ]; then
     shift
     target_list=$1
 else
+    # Build these by defualt for microtvm reference virtual machine and ci_qemu.
     target_list="aarch64-softmmu,arm-softmmu,i386-softmmu,riscv32-softmmu,riscv64-softmmu,x86_64-softmmu"
 fi
 
@@ -80,5 +81,5 @@ cd qemu-5.1.0
 make -j${num_cores}
 sudo make install
 
-# # For debugging with qemu
+# For debugging with qemu
 apt-get -y install libpython3.8

From 4b3b23276d70f57e8e32470c27b6de0412c9ab02 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 1 Jun 2021 09:00:58 -0700
Subject: [PATCH 07/40] add qemu to setup build

---
 apps/microtvm/reference-vm/zephyr/Vagrantfile | 2 +-
 apps/microtvm/reference-vm/zephyr/setup.sh    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/microtvm/reference-vm/zephyr/Vagrantfile b/apps/microtvm/reference-vm/zephyr/Vagrantfile
index f335565341db..b24e6a1bcf94 100644
--- a/apps/microtvm/reference-vm/zephyr/Vagrantfile
+++ b/apps/microtvm/reference-vm/zephyr/Vagrantfile
@@ -34,7 +34,7 @@ Vagrant.configure("2") do |config|
     end
   end
 
-  config.vm.provision "shell", path: "setup.sh", env: {"TVM_HOME": dirs_to_mount[0]}, privileged: false
+  config.vm.provision "shell", path: "setup.sh", env: {"TVM_HOME": dirs_to_mount[0]}, privileged: true
 
   # Enable USB Controller on VirtualBox
   vm_name = "microtvm-#{Time.now.tv_sec}"
diff --git a/apps/microtvm/reference-vm/zephyr/setup.sh b/apps/microtvm/reference-vm/zephyr/setup.sh
index 470b2e52170b..f9c324f80c1a 100644
--- a/apps/microtvm/reference-vm/zephyr/setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/setup.sh
@@ -25,7 +25,7 @@ cd "${TVM_HOME}"
 apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
 
 # Install QEMU
-sudo docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu
+sudo ./docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu
 
 # Build poetry
 cd apps/microtvm/reference-vm/zephyr

From 7e690a295e096a807f69c22395b44c6af30699ee Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 1 Jun 2021 09:48:22 -0700
Subject: [PATCH 08/40] fix

---
 apps/microtvm/reference-vm/zephyr/setup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/microtvm/reference-vm/zephyr/setup.sh b/apps/microtvm/reference-vm/zephyr/setup.sh
index f9c324f80c1a..470b2e52170b 100644
--- a/apps/microtvm/reference-vm/zephyr/setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/setup.sh
@@ -25,7 +25,7 @@ cd "${TVM_HOME}"
 apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
 
 # Install QEMU
-sudo ./docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu
+sudo docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu
 
 # Build poetry
 cd apps/microtvm/reference-vm/zephyr

From 7f62f78bccb2cbdfc613053b3f5295f6223783c9 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 1 Jun 2021 11:50:59 -0700
Subject: [PATCH 09/40] add mps2 test

---
 apps/microtvm/reference-vm/base-box-tool.py      | 16 ++++++++++------
 .../zephyr/base-box/test-config.json             |  5 +++++
 apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh |  9 ++++++++-
 3 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py
index 8221b9f97526..5b9e8ad86356 100755
--- a/apps/microtvm/reference-vm/base-box-tool.py
+++ b/apps/microtvm/reference-vm/base-box-tool.py
@@ -46,6 +46,7 @@
 ALL_MICROTVM_PLATFORMS = (
     "stm32f746xx",
     "nrf5340dk",
+    "mps2_an521",
 )
 
 
@@ -294,12 +295,15 @@ def do_run_release_test(release_test_dir, provider_name, test_config, test_devic
         os.path.join(release_test_dir, ".vagrant", "machines", "default", provider_name, "id")
     ) as f:
         machine_uuid = f.read()
-    ATTACH_USB_DEVICE[provider_name](
-        machine_uuid,
-        vid_hex=test_config["vid_hex"],
-        pid_hex=test_config["pid_hex"],
-        serial=test_device_serial,
-    )
+
+    # Check if target is not QEMU
+    if test_config["vid_hex"] >= 0 and test_config["pid_hex"] >= 0:
+        ATTACH_USB_DEVICE[provider_name](
+            machine_uuid,
+            vid_hex=test_config["vid_hex"],
+            pid_hex=test_config["pid_hex"],
+            serial=test_device_serial,
+        )
     tvm_home = os.path.realpath(os.path.join(THIS_DIR, "..", "..", ".."))
 
     def _quote_cmd(cmd):
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/test-config.json b/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
index 1a39d34c7e64..6b5214b0ef03 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
+++ b/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
@@ -8,5 +8,10 @@
         "vid_hex": "1366",
         "pid_hex": "1055",
         "test_cmd": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=nrf5340dk"]
+        },
+    "mps2_an521": {
+        "vid_hex": "-1",
+        "pid_hex": "-1",
+        "test_cmd": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=mps2_an521"]
         }
 }
diff --git a/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh b/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
index 2eb55e385520..bb24302f96a6 100755
--- a/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
+++ b/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
@@ -18,6 +18,13 @@
 
 set -e
 
+# Get number of cores for build
+if [[ "${CI_NUM_CORE}" ]]; then
+  num_cores=${CI_NUM_CORE}
+else
+  num_cores=2
+fi
+
 cd "$(dirname $0)"
 cd "$(git rev-parse --show-toplevel)"
 BUILD_DIR=build-microtvm
@@ -32,4 +39,4 @@ sed -i 's/USE_GRAPH_EXECUTOR_DEBUG OFF/USE_GRAPH_EXECUTOR_DEBUG ON/' config.cmak
 sed -i 's/USE_LLVM OFF/USE_LLVM ON/' config.cmake
 cmake ..
 rm -rf standalone_crt host_standalone_crt  # remove stale generated files
-make -j4
+make -j${num_cores}

From e14c83baa1fa25190c862e028d342ee17b50d58f Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 1 Jun 2021 11:57:54 -0700
Subject: [PATCH 10/40] merge fix

---
 apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh b/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
index bb24302f96a6..379b002de654 100755
--- a/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
+++ b/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
@@ -19,8 +19,8 @@
 set -e
 
 # Get number of cores for build
-if [[ "${CI_NUM_CORE}" ]]; then
-  num_cores=${CI_NUM_CORE}
+if [[ "${TVM_CI_NUM_CORES}" ]]; then
+  num_cores=${TVM_CI_NUM_CORES}
 else
   num_cores=2
 fi

From 9420eec070e8ce2dff36db20e815fe36474a9f01 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 1 Jun 2021 12:16:25 -0700
Subject: [PATCH 11/40] add commit option

---
 .../reference-vm/zephyr/base-box/setup.sh     |  2 +-
 docker/install/ubuntu_init_zephyr_project.sh  | 26 +++++++++++++++++--
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/apps/microtvm/reference-vm/zephyr/base-box/setup.sh b/apps/microtvm/reference-vm/zephyr/base-box/setup.sh
index 8f7ed41af337..34a489da68ca 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/base-box/setup.sh
@@ -82,7 +82,7 @@ echo 'export PATH=$HOME/.local/bin:"$PATH"' >> ~/.profile
 source ~/.profile
 echo PATH=$PATH
 REPO_ROOT=$(git rev-parse --show-toplevel)
-${REPO_ROOT}/docker/install/ubuntu_init_zephyr_project.sh ~/zephyr v2.5.0
+${REPO_ROOT}/docker/install/ubuntu_init_zephyr_project.sh ~/zephyr v2.5-branch dabf23758417fd041fec2a2a821d8f526afac29d
 
 cd ~
 echo "Downloading zephyr SDK..."
diff --git a/docker/install/ubuntu_init_zephyr_project.sh b/docker/install/ubuntu_init_zephyr_project.sh
index 2116a4d981f5..c7be46b5ecb9 100755
--- a/docker/install/ubuntu_init_zephyr_project.sh
+++ b/docker/install/ubuntu_init_zephyr_project.sh
@@ -16,10 +16,32 @@
 # specific language governing permissions and limitations
 # under the License.
 
+#
+# Initialize Zephyr Project.
+#
+# Usage: ubuntu_init_zephyr_project.sh path branch [--commit hash]
+#   path is the installation path for the repository.
+#   branch is the zephyr branch.
+#   --commit is the commit hash number of zephyrproject repository. If not specified, it uses the latest commit.
+#
+
 DOWNLOAD_DIR=$1
-ZEPHYR_BRANCH=$2
+shift
+ZEPHYR_BRANCH=$1
+shift
+
+commit="0"
+if [ "$1" == "--commit" ]; then
+    commit=$1
+fi
 
 west init --mr ${ZEPHYR_BRANCH} ${DOWNLOAD_DIR}
-cd ${DOWNLOAD_DIR}
+
+if [ "$commit" != "0"]; then
+    cd ${DOWNLOAD_DIR}/zephyr
+    git checkout ${commit}
+fi
+
+cd ${DOWNLOAD_DIR}/zephyr
 west update
 west zephyr-export

From bfa74238e43b3d003b6a0ecbdb5b326bdb9a0a0b Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 1 Jun 2021 14:39:39 -0700
Subject: [PATCH 12/40] add log

---
 apps/microtvm/reference-vm/zephyr/Vagrantfile                 | 2 +-
 .../reference-vm/zephyr/base-box/Vagrantfile.packer-template  | 4 ++--
 .../zephyr/base-box/{setup.sh => base_box_setup.sh}           | 1 +
 apps/microtvm/reference-vm/zephyr/{setup.sh => init_setup.sh} | 4 ++--
 docker/install/ubuntu_init_zephyr_project.sh                  | 3 +++
 5 files changed, 9 insertions(+), 5 deletions(-)
 rename apps/microtvm/reference-vm/zephyr/base-box/{setup.sh => base_box_setup.sh} (99%)
 rename apps/microtvm/reference-vm/zephyr/{setup.sh => init_setup.sh} (94%)

diff --git a/apps/microtvm/reference-vm/zephyr/Vagrantfile b/apps/microtvm/reference-vm/zephyr/Vagrantfile
index 7dafc0bee945..1204f1b67a82 100644
--- a/apps/microtvm/reference-vm/zephyr/Vagrantfile
+++ b/apps/microtvm/reference-vm/zephyr/Vagrantfile
@@ -46,7 +46,7 @@ Vagrant.configure("2") do |config|
     end
   end
 
-  config.vm.provision "shell", path: "setup.sh", env: {"TVM_HOME": dirs_to_mount[0]}, privileged: true
+  config.vm.provision "shell", path: "init_setup.sh", env: {"TVM_HOME": dirs_to_mount[0]}, privileged: true
 
   # Enable USB Controller on VirtualBox
   vm_name = "microtvm-#{Time.now.tv_sec}"
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/Vagrantfile.packer-template b/apps/microtvm/reference-vm/zephyr/base-box/Vagrantfile.packer-template
index 38f9a20b56cf..b43596bb83c1 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/Vagrantfile.packer-template
+++ b/apps/microtvm/reference-vm/zephyr/base-box/Vagrantfile.packer-template
@@ -41,7 +41,7 @@ Vagrant.configure("2") do |config|
   config.vm.provision "shell", inline: "touch ~/skip_zeroing_disk", privileged: false
   {{- end}}
 
-  # NOTE: setup.sh resides in the parent directory (../) because this template is expanded into a
+  # NOTE: base_box_setup.sh resides in the parent directory (../) because this template is expanded into a
   # sub-directory of base-box (output-packer-*).
-  config.vm.provision "shell", path: "../setup.sh", privileged: false
+  config.vm.provision "shell", path: "../base_box_setup.sh", privileged: false
 end
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/setup.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh
similarity index 99%
rename from apps/microtvm/reference-vm/zephyr/base-box/setup.sh
rename to apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh
index 34a489da68ca..f2a9e2382ae5 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh
@@ -17,6 +17,7 @@
 # under the License.
 
 set -e
+set -x
 
 skip_zeroing_disk=0
 if [ -e "$HOME/skip_zeroing_disk" ]; then
diff --git a/apps/microtvm/reference-vm/zephyr/setup.sh b/apps/microtvm/reference-vm/zephyr/init_setup.sh
similarity index 94%
rename from apps/microtvm/reference-vm/zephyr/setup.sh
rename to apps/microtvm/reference-vm/zephyr/init_setup.sh
index 470b2e52170b..399a270adacb 100644
--- a/apps/microtvm/reference-vm/zephyr/setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/init_setup.sh
@@ -22,10 +22,10 @@ set -ex
 # NOTE: TVM is presumed to be mounted already by Vagrantfile.
 cd "${TVM_HOME}"
 
-apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
+# apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
 
 # Install QEMU
-sudo docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu
+# sudo docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu
 
 # Build poetry
 cd apps/microtvm/reference-vm/zephyr
diff --git a/docker/install/ubuntu_init_zephyr_project.sh b/docker/install/ubuntu_init_zephyr_project.sh
index c7be46b5ecb9..553ab8d50a7f 100755
--- a/docker/install/ubuntu_init_zephyr_project.sh
+++ b/docker/install/ubuntu_init_zephyr_project.sh
@@ -25,6 +25,8 @@
 #   --commit is the commit hash number of zephyrproject repository. If not specified, it uses the latest commit.
 #
 
+set -x
+
 DOWNLOAD_DIR=$1
 shift
 ZEPHYR_BRANCH=$1
@@ -32,6 +34,7 @@ shift
 
 commit="0"
 if [ "$1" == "--commit" ]; then
+    shift
     commit=$1
 fi
 

From f6d3498fc5961d52523b227bd2cfcc9c6ef2a765 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 1 Jun 2021 17:44:08 -0700
Subject: [PATCH 13/40] fix

---
 apps/microtvm/reference-vm/base-box-tool.py          | 12 ++++++++++++
 .../reference-vm/zephyr/base-box/base_box_setup.sh   |  4 +---
 apps/microtvm/reference-vm/zephyr/init_setup.sh      |  4 ++--
 3 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py
index 5b9e8ad86356..fe576eade3f0 100755
--- a/apps/microtvm/reference-vm/base-box-tool.py
+++ b/apps/microtvm/reference-vm/base-box-tool.py
@@ -34,6 +34,8 @@
 
 THIS_DIR = os.path.realpath(os.path.dirname(__file__) or ".")
 
+ZEPHYR_BRANCH = "v2.5-branch"
+ZEPHYR_COMMIT = "dabf23758417fd041fec2a2a821d8f526afac29d"
 
 # List of vagrant providers supported by this tool
 ALL_PROVIDERS = (
@@ -177,6 +179,7 @@ def attach_vmware(uuid, vid_hex=None, pid_hex=None, serial=None):
 
 def generate_packer_config(file_path, providers):
     builders = []
+    provisioners = []
     for provider_name in providers:
         builders.append(
             {
@@ -190,10 +193,19 @@ def generate_packer_config(file_path, providers):
             }
         )
 
+    provisioners.append(
+        {
+            "type": "shell-local",
+            "script": "../../../../../docker/install/ubuntu_init_zephyr_project.sh",
+            "execute_command": f"sh '{{.Path}}' ~/zephyrproject {ZEPHYR_BRANCH} {ZEPHYR_COMMIT}"
+        }
+    )
+
     with open(file_path, "w") as f:
         json.dump(
             {
                 "builders": builders,
+                "provisioners": provisioners,
             },
             f,
             sort_keys=True,
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh
index f2a9e2382ae5..31efdb851d19 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh
@@ -82,8 +82,6 @@ pip3 install --user -U west
 echo 'export PATH=$HOME/.local/bin:"$PATH"' >> ~/.profile
 source ~/.profile
 echo PATH=$PATH
-REPO_ROOT=$(git rev-parse --show-toplevel)
-${REPO_ROOT}/docker/install/ubuntu_init_zephyr_project.sh ~/zephyr v2.5-branch dabf23758417fd041fec2a2a821d8f526afac29d
 
 cd ~
 echo "Downloading zephyr SDK..."
@@ -105,7 +103,7 @@ sudo udevadm control --reload
 # Poetry
 curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python3
 sed -i "/^# If not running interactively,/ i source \$HOME/.poetry/env" ~/.bashrc
-sed -i "/^# If not running interactively,/ i export ZEPHYR_BASE=$HOME/zephyr/zephyr" ~/.bashrc
+sed -i "/^# If not running interactively,/ i export ZEPHYR_BASE=$HOME/zephyrproject/zephyr" ~/.bashrc
 sed -i "/^# If not running interactively,/ i\\ " ~/.bashrc
 
 # Clean box for packaging as a base box
diff --git a/apps/microtvm/reference-vm/zephyr/init_setup.sh b/apps/microtvm/reference-vm/zephyr/init_setup.sh
index 399a270adacb..470b2e52170b 100644
--- a/apps/microtvm/reference-vm/zephyr/init_setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/init_setup.sh
@@ -22,10 +22,10 @@ set -ex
 # NOTE: TVM is presumed to be mounted already by Vagrantfile.
 cd "${TVM_HOME}"
 
-# apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
+apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
 
 # Install QEMU
-# sudo docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu
+sudo docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu
 
 # Build poetry
 cd apps/microtvm/reference-vm/zephyr

From 233238375a2a0c7f2d7c772d40177735ed763636 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 1 Jun 2021 17:53:57 -0700
Subject: [PATCH 14/40] fix zephyr init

---
 apps/microtvm/reference-vm/base-box-tool.py  | 2 +-
 docker/install/ubuntu_init_zephyr_project.sh | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py
index fe576eade3f0..c8df2c6ca016 100755
--- a/apps/microtvm/reference-vm/base-box-tool.py
+++ b/apps/microtvm/reference-vm/base-box-tool.py
@@ -197,7 +197,7 @@ def generate_packer_config(file_path, providers):
         {
             "type": "shell-local",
             "script": "../../../../../docker/install/ubuntu_init_zephyr_project.sh",
-            "execute_command": f"sh '{{.Path}}' ~/zephyrproject {ZEPHYR_BRANCH} {ZEPHYR_COMMIT}"
+            "execute_command": f"sh '{{.Path}}' ~/zephyrproject {ZEPHYR_BRANCH} --commit {ZEPHYR_COMMIT}"
         }
     )
 
diff --git a/docker/install/ubuntu_init_zephyr_project.sh b/docker/install/ubuntu_init_zephyr_project.sh
index 553ab8d50a7f..2ec60a0efeb0 100755
--- a/docker/install/ubuntu_init_zephyr_project.sh
+++ b/docker/install/ubuntu_init_zephyr_project.sh
@@ -32,17 +32,17 @@ shift
 ZEPHYR_BRANCH=$1
 shift
 
-commit="0"
+commit_hash="0"
 if [ "$1" == "--commit" ]; then
     shift
-    commit=$1
+    commit_hash=$1
 fi
 
 west init --mr ${ZEPHYR_BRANCH} ${DOWNLOAD_DIR}
 
-if [ "$commit" != "0"]; then
+if [ "$commit_hash" != "0" ]; then
     cd ${DOWNLOAD_DIR}/zephyr
-    git checkout ${commit}
+    git checkout ${commit_hash}
 fi
 
 cd ${DOWNLOAD_DIR}/zephyr

From 139dc8b562319114c44a3e23413caacf057e7f6f Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 1 Jun 2021 17:55:42 -0700
Subject: [PATCH 15/40] rename

---
 apps/microtvm/reference-vm/zephyr/Vagrantfile                 | 2 +-
 .../reference-vm/zephyr/{init_setup.sh => provision_setup.sh} | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)
 rename apps/microtvm/reference-vm/zephyr/{init_setup.sh => provision_setup.sh} (94%)

diff --git a/apps/microtvm/reference-vm/zephyr/Vagrantfile b/apps/microtvm/reference-vm/zephyr/Vagrantfile
index 1204f1b67a82..4e6d01de1772 100644
--- a/apps/microtvm/reference-vm/zephyr/Vagrantfile
+++ b/apps/microtvm/reference-vm/zephyr/Vagrantfile
@@ -46,7 +46,7 @@ Vagrant.configure("2") do |config|
     end
   end
 
-  config.vm.provision "shell", path: "init_setup.sh", env: {"TVM_HOME": dirs_to_mount[0]}, privileged: true
+  config.vm.provision "shell", path: "provision_setup.sh", env: {"TVM_HOME": dirs_to_mount[0]}, privileged: true
 
   # Enable USB Controller on VirtualBox
   vm_name = "microtvm-#{Time.now.tv_sec}"
diff --git a/apps/microtvm/reference-vm/zephyr/init_setup.sh b/apps/microtvm/reference-vm/zephyr/provision_setup.sh
similarity index 94%
rename from apps/microtvm/reference-vm/zephyr/init_setup.sh
rename to apps/microtvm/reference-vm/zephyr/provision_setup.sh
index 470b2e52170b..399a270adacb 100644
--- a/apps/microtvm/reference-vm/zephyr/init_setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/provision_setup.sh
@@ -22,10 +22,10 @@ set -ex
 # NOTE: TVM is presumed to be mounted already by Vagrantfile.
 cd "${TVM_HOME}"
 
-apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
+# apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
 
 # Install QEMU
-sudo docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu
+# sudo docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu
 
 # Build poetry
 cd apps/microtvm/reference-vm/zephyr

From 5999db98a619d15811a82b146557096d00375c0f Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 1 Jun 2021 19:04:34 -0700
Subject: [PATCH 16/40] fix zephyr init

---
 apps/microtvm/reference-vm/base-box-tool.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py
index c8df2c6ca016..fb0d909f38f2 100755
--- a/apps/microtvm/reference-vm/base-box-tool.py
+++ b/apps/microtvm/reference-vm/base-box-tool.py
@@ -195,9 +195,9 @@ def generate_packer_config(file_path, providers):
 
     provisioners.append(
         {
-            "type": "shell-local",
+            "type": "shell",
             "script": "../../../../../docker/install/ubuntu_init_zephyr_project.sh",
-            "execute_command": f"sh '{{.Path}}' ~/zephyrproject {ZEPHYR_BRANCH} --commit {ZEPHYR_COMMIT}"
+            "execute_command": f"chmod +x {{{{ .Path }}}}; source ~/.profile; {{{{ .Vars }}}} {{{{ .Path }}}} ~/zephyrproject {ZEPHYR_BRANCH} --commit {ZEPHYR_COMMIT}",
         }
     )
 

From 96cde66eedd40209329bcbc5f44a0904db5b22ca Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 1 Jun 2021 19:07:52 -0700
Subject: [PATCH 17/40] uncomment

---
 apps/microtvm/reference-vm/zephyr/provision_setup.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/microtvm/reference-vm/zephyr/provision_setup.sh b/apps/microtvm/reference-vm/zephyr/provision_setup.sh
index 399a270adacb..470b2e52170b 100644
--- a/apps/microtvm/reference-vm/zephyr/provision_setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/provision_setup.sh
@@ -22,10 +22,10 @@ set -ex
 # NOTE: TVM is presumed to be mounted already by Vagrantfile.
 cd "${TVM_HOME}"
 
-# apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
+apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
 
 # Install QEMU
-# sudo docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu
+sudo docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu
 
 # Build poetry
 cd apps/microtvm/reference-vm/zephyr

From 0ab5af207ac5d510e47e9fec7a03907b98ccc210 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Wed, 2 Jun 2021 09:46:43 -0700
Subject: [PATCH 18/40] fixed qemu isntall

---
 apps/microtvm/reference-vm/base-box-tool.py   | 11 ++++++-
 apps/microtvm/reference-vm/zephyr/Vagrantfile |  2 +-
 .../zephyr/base-box/test-config.json          |  4 +--
 .../reference-vm/zephyr/provision_setup.sh    |  7 ++--
 docker/install/ubuntu_install_qemu.sh         | 33 +++++++++----------
 5 files changed, 30 insertions(+), 27 deletions(-)

diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py
index fb0d909f38f2..626e039b0c69 100755
--- a/apps/microtvm/reference-vm/base-box-tool.py
+++ b/apps/microtvm/reference-vm/base-box-tool.py
@@ -299,6 +299,15 @@ def do_build_release_test_vm(release_test_dir, user_box_dir, base_box_dir, provi
     assert return_code in (0, 1), f'{" ".join(remove_args)} returned exit code {return_code}'
     subprocess.check_call(["vagrant", "up", f"--provider={provider_name}"], cwd=release_test_dir)
 
+    # Build QEMU
+    tvm_home = subprocess.check_output(
+        ["git", "rev-parse", "--show-toplevel"], encoding="utf-8"
+    ).strip()
+    qemu_cmd = (
+        f"cd {tvm_home} && sudo docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu"
+    )
+    subprocess.check_call(["vagrant", "ssh", "-c", qemu_cmd], cwd=release_test_dir)
+
     return True
 
 
@@ -309,7 +318,7 @@ def do_run_release_test(release_test_dir, provider_name, test_config, test_devic
         machine_uuid = f.read()
 
     # Check if target is not QEMU
-    if test_config["vid_hex"] >= 0 and test_config["pid_hex"] >= 0:
+    if test_config["vid_hex"] and test_config["pid_hex"]:
         ATTACH_USB_DEVICE[provider_name](
             machine_uuid,
             vid_hex=test_config["vid_hex"],
diff --git a/apps/microtvm/reference-vm/zephyr/Vagrantfile b/apps/microtvm/reference-vm/zephyr/Vagrantfile
index 4e6d01de1772..be41c0b733e5 100644
--- a/apps/microtvm/reference-vm/zephyr/Vagrantfile
+++ b/apps/microtvm/reference-vm/zephyr/Vagrantfile
@@ -46,7 +46,7 @@ Vagrant.configure("2") do |config|
     end
   end
 
-  config.vm.provision "shell", path: "provision_setup.sh", env: {"TVM_HOME": dirs_to_mount[0]}, privileged: true
+  config.vm.provision "shell", path: "provision_setup.sh", env: {"TVM_HOME": dirs_to_mount[0]}, privileged: false
 
   # Enable USB Controller on VirtualBox
   vm_name = "microtvm-#{Time.now.tv_sec}"
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/test-config.json b/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
index 6b5214b0ef03..e196d0a8b18a 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
+++ b/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
@@ -10,8 +10,8 @@
         "test_cmd": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=nrf5340dk"]
         },
     "mps2_an521": {
-        "vid_hex": "-1",
-        "pid_hex": "-1",
+        "vid_hex": "",
+        "pid_hex": "",
         "test_cmd": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=mps2_an521"]
         }
 }
diff --git a/apps/microtvm/reference-vm/zephyr/provision_setup.sh b/apps/microtvm/reference-vm/zephyr/provision_setup.sh
index 470b2e52170b..dd04139678a7 100644
--- a/apps/microtvm/reference-vm/zephyr/provision_setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/provision_setup.sh
@@ -22,10 +22,7 @@ set -ex
 # NOTE: TVM is presumed to be mounted already by Vagrantfile.
 cd "${TVM_HOME}"
 
-apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
-
-# Install QEMU
-sudo docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu
+# apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
 
 # Build poetry
 cd apps/microtvm/reference-vm/zephyr
@@ -44,7 +41,7 @@ echo "downloaded and cached for future use."
 echo "------------------------------[ TVM Message ]------------------------------"
 poetry lock -vvv
 poetry install
-poetry run pip3 install -r ~/zephyr/zephyr/scripts/requirements.txt
+poetry run pip3 install -r ~/zephyrproject/zephyr/scripts/requirements.txt
 
 echo "export TVM_LIBRARY_PATH=\"$TVM_HOME\"/build-microtvm" >>~/.profile
 echo "VENV_PATH=\$((cd \"$TVM_HOME\"/apps/microtvm/reference-vm/zephyr && poetry env list --full-path) | sed -E 's/^(.*)[[:space:]]\(Activated\)\$/\1/g')" >>~/.profile
diff --git a/docker/install/ubuntu_install_qemu.sh b/docker/install/ubuntu_install_qemu.sh
index be53e5aa388f..b31d0c748de9 100755
--- a/docker/install/ubuntu_install_qemu.sh
+++ b/docker/install/ubuntu_install_qemu.sh
@@ -26,7 +26,12 @@
 set -e
 set -o pipefail
 
-QEMU_DIR=qemu-5.1.0
+QEMU_NAME=qemu-5.1.0
+QEMU_SIG_FILE=${QEMU_NAME}.tar.xz.sig
+QEMU_TAR_FILE=${QEMU_NAME}.tar.xz
+
+# Clean previous build
+rm -rf ${QEMU_NAME} ${QEMU_SIG_FILE} ${QEMU_TAR_FILE}
 
 # Get number of cores for build
 if [[ "${TVM_CI_NUM_CORES}" ]]; then
@@ -44,19 +49,12 @@ else
     target_list="aarch64-softmmu,arm-softmmu,i386-softmmu,riscv32-softmmu,riscv64-softmmu,x86_64-softmmu"
 fi
 
-# Check if QEMU already built
-rebuild=0
-if [ -d ${QEMU_DIR} ]; then
-    rebuild=1
-fi
-
-if [ $rebuild -eq 0 ]; then
-    sudo sed -i '/deb-src/s/^# //' /etc/apt/sources.list
-    apt update
-    apt-get -y build-dep qemu
+sudo sed -i '/deb-src/s/^# //' /etc/apt/sources.list
+apt update
+apt-get -y build-dep qemu
 
-    gpg --keyserver keys.gnupg.net --recv-keys 0x3353C9CEF108B584
-    cat <<EOF | gpg --dearmor >qemu-5.1.0.tar.xz.sig
+gpg --keyserver keys.gnupg.net --recv-keys 0x3353C9CEF108B584
+cat <<EOF | gpg --dearmor >qemu-5.1.0.tar.xz.sig
 -----BEGIN PGP ARMORED FILE-----
 Comment: Use "gpg --dearmor" for unpacking
 
@@ -70,13 +68,12 @@ p5ez/+2k4VAIwIQoP5DoO06waLBffvLIAdPPKYsx71K67OoGG2svc7duC/+5qf1x
 =hCS7
 -----END PGP ARMORED FILE-----
 EOF
-    curl -OLs https://download.qemu.org/qemu-5.1.0.tar.xz
-    gpg --verify qemu-5.1.0.tar.xz.sig
+curl -OLs https://download.qemu.org/qemu-5.1.0.tar.xz
+gpg --verify ${QEMU_SIG_FILE}
 
-    tar -xf qemu-5.1.0.tar.xz
-fi
+tar -xf ${QEMU_TAR_FILE}
 
-cd qemu-5.1.0
+cd ${QEMU_NAME}
 ./configure --target-list=${target_list}
 make -j${num_cores}
 sudo make install

From 05c034781068556938517eff2f45c094f4042f76 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Wed, 2 Jun 2021 10:44:29 -0700
Subject: [PATCH 19/40] cleanup

---
 apps/microtvm/reference-vm/zephyr/provision_setup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/microtvm/reference-vm/zephyr/provision_setup.sh b/apps/microtvm/reference-vm/zephyr/provision_setup.sh
index dd04139678a7..6ed9d576dfbe 100644
--- a/apps/microtvm/reference-vm/zephyr/provision_setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/provision_setup.sh
@@ -22,7 +22,7 @@ set -ex
 # NOTE: TVM is presumed to be mounted already by Vagrantfile.
 cd "${TVM_HOME}"
 
-# apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
+apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
 
 # Build poetry
 cd apps/microtvm/reference-vm/zephyr

From 91654d688cc71628d3f409abb6b22c87eb80ea03 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Thu, 3 Jun 2021 09:47:40 -0700
Subject: [PATCH 20/40] version

---
 apps/microtvm/reference-vm/zephyr/Vagrantfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/microtvm/reference-vm/zephyr/Vagrantfile b/apps/microtvm/reference-vm/zephyr/Vagrantfile
index be41c0b733e5..d011e546f44b 100644
--- a/apps/microtvm/reference-vm/zephyr/Vagrantfile
+++ b/apps/microtvm/reference-vm/zephyr/Vagrantfile
@@ -17,6 +17,7 @@
 
 Vagrant.configure("2") do |config|
   config.vm.box = "tlcpack/microtvm-zephyr-2.5"
+  config.vm.box_version = "0.0.2"
 
   if ENV.has_key?("TVM_RVM_NUM_CORES")
     num_cores = ENV["TVM_RVM_NUM_CORES"]

From dd6421645ec0d1a43c92738fed1f06fcc86e1913 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 1 Jun 2021 11:57:54 -0700
Subject: [PATCH 21/40] add commit option

---
 .../reference-vm/zephyr/base-box/setup.sh     |  2 +-
 .../reference-vm/zephyr/rebuild-tvm.sh        |  4 +--
 docker/install/ubuntu_init_zephyr_project.sh  | 26 +++++++++++++++++--
 3 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/apps/microtvm/reference-vm/zephyr/base-box/setup.sh b/apps/microtvm/reference-vm/zephyr/base-box/setup.sh
index 8f7ed41af337..34a489da68ca 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/base-box/setup.sh
@@ -82,7 +82,7 @@ echo 'export PATH=$HOME/.local/bin:"$PATH"' >> ~/.profile
 source ~/.profile
 echo PATH=$PATH
 REPO_ROOT=$(git rev-parse --show-toplevel)
-${REPO_ROOT}/docker/install/ubuntu_init_zephyr_project.sh ~/zephyr v2.5.0
+${REPO_ROOT}/docker/install/ubuntu_init_zephyr_project.sh ~/zephyr v2.5-branch dabf23758417fd041fec2a2a821d8f526afac29d
 
 cd ~
 echo "Downloading zephyr SDK..."
diff --git a/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh b/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
index bb24302f96a6..379b002de654 100755
--- a/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
+++ b/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
@@ -19,8 +19,8 @@
 set -e
 
 # Get number of cores for build
-if [[ "${CI_NUM_CORE}" ]]; then
-  num_cores=${CI_NUM_CORE}
+if [[ "${TVM_CI_NUM_CORES}" ]]; then
+  num_cores=${TVM_CI_NUM_CORES}
 else
   num_cores=2
 fi
diff --git a/docker/install/ubuntu_init_zephyr_project.sh b/docker/install/ubuntu_init_zephyr_project.sh
index 2116a4d981f5..c7be46b5ecb9 100755
--- a/docker/install/ubuntu_init_zephyr_project.sh
+++ b/docker/install/ubuntu_init_zephyr_project.sh
@@ -16,10 +16,32 @@
 # specific language governing permissions and limitations
 # under the License.
 
+#
+# Initialize Zephyr Project.
+#
+# Usage: ubuntu_init_zephyr_project.sh path branch [--commit hash]
+#   path is the installation path for the repository.
+#   branch is the zephyr branch.
+#   --commit is the commit hash number of zephyrproject repository. If not specified, it uses the latest commit.
+#
+
 DOWNLOAD_DIR=$1
-ZEPHYR_BRANCH=$2
+shift
+ZEPHYR_BRANCH=$1
+shift
+
+commit="0"
+if [ "$1" == "--commit" ]; then
+    commit=$1
+fi
 
 west init --mr ${ZEPHYR_BRANCH} ${DOWNLOAD_DIR}
-cd ${DOWNLOAD_DIR}
+
+if [ "$commit" != "0"]; then
+    cd ${DOWNLOAD_DIR}/zephyr
+    git checkout ${commit}
+fi
+
+cd ${DOWNLOAD_DIR}/zephyr
 west update
 west zephyr-export

From 53a130f87c729566c9b9bd859d3e3bee939e603e Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 1 Jun 2021 14:39:39 -0700
Subject: [PATCH 22/40] fixed qemu isntall

---
 apps/microtvm/reference-vm/README.md          |  6 ++--
 apps/microtvm/reference-vm/base-box-tool.py   | 36 ++++++++++++++-----
 apps/microtvm/reference-vm/zephyr/Vagrantfile |  2 +-
 .../base-box/Vagrantfile.packer-template      |  4 +--
 .../zephyr/base-box/base_box_provision.sh     | 34 ++++++++++++++++++
 .../base-box/{setup.sh => base_box_setup.sh}  |  5 ++-
 .../zephyr/base-box/test-config.json          |  4 +--
 .../zephyr/{setup.sh => provision_setup.sh}   |  5 +--
 docker/install/ubuntu_init_zephyr_project.sh  | 11 +++---
 docker/install/ubuntu_install_qemu.sh         | 33 ++++++++---------
 10 files changed, 95 insertions(+), 45 deletions(-)
 create mode 100644 apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh
 rename apps/microtvm/reference-vm/zephyr/base-box/{setup.sh => base_box_setup.sh} (95%)
 rename apps/microtvm/reference-vm/zephyr/{setup.sh => provision_setup.sh} (92%)

diff --git a/apps/microtvm/reference-vm/README.md b/apps/microtvm/reference-vm/README.md
index 7ef7900c3e05..874facbcba2d 100644
--- a/apps/microtvm/reference-vm/README.md
+++ b/apps/microtvm/reference-vm/README.md
@@ -49,10 +49,10 @@ Reference VMs are organized as follows:
 
 ## Creating Releases
 
-1. Build the base box for the given platform: `$ ./base-box-tool.py build <platform>`
+1. Build the base box for the given platform: `$ ./base-box-tool.py <--provider=> build <platform>`
 2. Run release tests for each platform:
     1. Connect any needed hardware to the VM host machine.
-    2. Run tests: `$ ./base-box-tool.py test <platform> [--test-device-serial=<serial>]`. This
+    2. Run tests: `$ ./base-box-tool.py <--provider=> test <--microtvm-platform=> <platform> [--test-device-serial=<serial>]`. This
        command does the following for each provider:
         1. Copies all files inside `./<platform>` except `.vagrant` and `base-box` to
            `./release-test`. This is done to avoid reusing any VM the developer may have started.
@@ -63,5 +63,5 @@ Reference VMs are organized as follows:
         4. SSHs to the VM, `cd` to the TVM root directory, and runs `test_cmd` from
            `test-config.json`. Nonzero status means failure.
 3. If release tests fail, fix them and restart from step 1.
-4. If release tests pass: `$ ./base-box-tool.py release <platform> <version>`. Be sure you've logged
+4. If release tests pass: `$ ./base-box-tool.py <--provider=> release <--release-version=> <--platform-version=> <platform>`. Be sure you've logged
    in to Vagrant Cloud using the `vagrant` tool.
diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py
index 5b9e8ad86356..a478ff19deac 100755
--- a/apps/microtvm/reference-vm/base-box-tool.py
+++ b/apps/microtvm/reference-vm/base-box-tool.py
@@ -34,7 +34,6 @@
 
 THIS_DIR = os.path.realpath(os.path.dirname(__file__) or ".")
 
-
 # List of vagrant providers supported by this tool
 ALL_PROVIDERS = (
     "parallels",
@@ -174,12 +173,20 @@ def attach_vmware(uuid, vid_hex=None, pid_hex=None, serial=None):
     "vmware_desktop": attach_vmware,
 }
 
+EXTRA_SCRIPTS = (
+    "../../../../../docker/install/ubuntu_init_zephyr_project.sh",
+    "../../../../../docker/install/ubuntu_install_qemu.sh",
+)
+
 
 def generate_packer_config(file_path, providers):
     builders = []
+    provisioners = []
+
     for provider_name in providers:
         builders.append(
             {
+                "name": f"{provider_name}",
                 "type": "vagrant",
                 "box_name": f"microtvm-base-{provider_name}",
                 "output_dir": f"output-packer-{provider_name}",
@@ -190,13 +197,26 @@ def generate_packer_config(file_path, providers):
             }
         )
 
+    for script in EXTRA_SCRIPTS:
+        filename = os.path.basename(script)
+        provisioners.append({"type": "file", "source": script, "destination": f"~/{filename}"})
+
+    provisioners.append(
+        {
+            "type": "shell",
+            "script": "base_box_provision.sh",
+            "execute_command": f"chmod +x {{{{ .Path }}}}; {{{{ .Vars }}}} {{{{ .Path }}}}",
+        }
+    )
+
     with open(file_path, "w") as f:
         json.dump(
             {
                 "builders": builders,
+                "provisioners": provisioners,
             },
             f,
-            sort_keys=True,
+            sort_keys=False,
             indent=2,
         )
 
@@ -286,7 +306,6 @@ def do_build_release_test_vm(release_test_dir, user_box_dir, base_box_dir, provi
     return_code = subprocess.call(remove_args, cwd=release_test_dir)
     assert return_code in (0, 1), f'{" ".join(remove_args)} returned exit code {return_code}'
     subprocess.check_call(["vagrant", "up", f"--provider={provider_name}"], cwd=release_test_dir)
-
     return True
 
 
@@ -297,7 +316,7 @@ def do_run_release_test(release_test_dir, provider_name, test_config, test_devic
         machine_uuid = f.read()
 
     # Check if target is not QEMU
-    if test_config["vid_hex"] >= 0 and test_config["pid_hex"] >= 0:
+    if test_config["vid_hex"] and test_config["pid_hex"]:
         ATTACH_USB_DEVICE[provider_name](
             machine_uuid,
             vid_hex=test_config["vid_hex"],
@@ -404,10 +423,6 @@ def parse_args():
         description="Automates building, testing, and releasing a base box"
     )
     subparsers = parser.add_subparsers(help="Action to perform.")
-    parser.add_argument(
-        "platform",
-        help="Name of the platform VM to act on. Must be a sub-directory of this directory.",
-    )
     parser.add_argument(
         "--provider",
         choices=ALL_PROVIDERS,
@@ -415,6 +430,11 @@ def parse_args():
         help="Name of the provider or providers to act on; if not specified, act on all.",
     )
 
+    parser.add_argument(
+        "platform",
+        help="Name of the platform VM to act on. Must be a sub-directory of this directory.",
+    )
+
     parser_build = subparsers.add_parser("build", help="Build a base box.")
     parser_build.set_defaults(func=build_command)
     parser_test = subparsers.add_parser("test", help="Test a base box before release.")
diff --git a/apps/microtvm/reference-vm/zephyr/Vagrantfile b/apps/microtvm/reference-vm/zephyr/Vagrantfile
index b24e6a1bcf94..18bbe7854f1d 100644
--- a/apps/microtvm/reference-vm/zephyr/Vagrantfile
+++ b/apps/microtvm/reference-vm/zephyr/Vagrantfile
@@ -34,7 +34,7 @@ Vagrant.configure("2") do |config|
     end
   end
 
-  config.vm.provision "shell", path: "setup.sh", env: {"TVM_HOME": dirs_to_mount[0]}, privileged: true
+  config.vm.provision "shell", path: "provision_setup.sh", env: {"TVM_HOME": dirs_to_mount[0]}, privileged: false
 
   # Enable USB Controller on VirtualBox
   vm_name = "microtvm-#{Time.now.tv_sec}"
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/Vagrantfile.packer-template b/apps/microtvm/reference-vm/zephyr/base-box/Vagrantfile.packer-template
index 38f9a20b56cf..b43596bb83c1 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/Vagrantfile.packer-template
+++ b/apps/microtvm/reference-vm/zephyr/base-box/Vagrantfile.packer-template
@@ -41,7 +41,7 @@ Vagrant.configure("2") do |config|
   config.vm.provision "shell", inline: "touch ~/skip_zeroing_disk", privileged: false
   {{- end}}
 
-  # NOTE: setup.sh resides in the parent directory (../) because this template is expanded into a
+  # NOTE: base_box_setup.sh resides in the parent directory (../) because this template is expanded into a
   # sub-directory of base-box (output-packer-*).
-  config.vm.provision "shell", path: "../setup.sh", privileged: false
+  config.vm.provision "shell", path: "../base_box_setup.sh", privileged: false
 end
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh
new file mode 100644
index 000000000000..af8e36f41ca1
--- /dev/null
+++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh
@@ -0,0 +1,34 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+set -x
+
+source ~/.profile
+
+# Init Zephyr
+cd ~
+~/ubuntu_init_zephyr_project.sh ~/zephyrproject v2.5-branch --commit dabf23758417fd041fec2a2a821d8f526afac29d
+
+# Build QEMU
+cd ~
+sudo ~/ubuntu_install_qemu.sh --target-list arm-softmmu
+
+# Cleanup
+cd ~
+rm -f *.sh
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/setup.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh
similarity index 95%
rename from apps/microtvm/reference-vm/zephyr/base-box/setup.sh
rename to apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh
index 34a489da68ca..31efdb851d19 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh
@@ -17,6 +17,7 @@
 # under the License.
 
 set -e
+set -x
 
 skip_zeroing_disk=0
 if [ -e "$HOME/skip_zeroing_disk" ]; then
@@ -81,8 +82,6 @@ pip3 install --user -U west
 echo 'export PATH=$HOME/.local/bin:"$PATH"' >> ~/.profile
 source ~/.profile
 echo PATH=$PATH
-REPO_ROOT=$(git rev-parse --show-toplevel)
-${REPO_ROOT}/docker/install/ubuntu_init_zephyr_project.sh ~/zephyr v2.5-branch dabf23758417fd041fec2a2a821d8f526afac29d
 
 cd ~
 echo "Downloading zephyr SDK..."
@@ -104,7 +103,7 @@ sudo udevadm control --reload
 # Poetry
 curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python3
 sed -i "/^# If not running interactively,/ i source \$HOME/.poetry/env" ~/.bashrc
-sed -i "/^# If not running interactively,/ i export ZEPHYR_BASE=$HOME/zephyr/zephyr" ~/.bashrc
+sed -i "/^# If not running interactively,/ i export ZEPHYR_BASE=$HOME/zephyrproject/zephyr" ~/.bashrc
 sed -i "/^# If not running interactively,/ i\\ " ~/.bashrc
 
 # Clean box for packaging as a base box
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/test-config.json b/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
index 6b5214b0ef03..e196d0a8b18a 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
+++ b/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
@@ -10,8 +10,8 @@
         "test_cmd": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=nrf5340dk"]
         },
     "mps2_an521": {
-        "vid_hex": "-1",
-        "pid_hex": "-1",
+        "vid_hex": "",
+        "pid_hex": "",
         "test_cmd": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=mps2_an521"]
         }
 }
diff --git a/apps/microtvm/reference-vm/zephyr/setup.sh b/apps/microtvm/reference-vm/zephyr/provision_setup.sh
similarity index 92%
rename from apps/microtvm/reference-vm/zephyr/setup.sh
rename to apps/microtvm/reference-vm/zephyr/provision_setup.sh
index 470b2e52170b..6ed9d576dfbe 100644
--- a/apps/microtvm/reference-vm/zephyr/setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/provision_setup.sh
@@ -24,9 +24,6 @@ cd "${TVM_HOME}"
 
 apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
 
-# Install QEMU
-sudo docker/install/ubuntu_install_qemu.sh --target-list arm-softmmu
-
 # Build poetry
 cd apps/microtvm/reference-vm/zephyr
 poetry env use 3.6
@@ -44,7 +41,7 @@ echo "downloaded and cached for future use."
 echo "------------------------------[ TVM Message ]------------------------------"
 poetry lock -vvv
 poetry install
-poetry run pip3 install -r ~/zephyr/zephyr/scripts/requirements.txt
+poetry run pip3 install -r ~/zephyrproject/zephyr/scripts/requirements.txt
 
 echo "export TVM_LIBRARY_PATH=\"$TVM_HOME\"/build-microtvm" >>~/.profile
 echo "VENV_PATH=\$((cd \"$TVM_HOME\"/apps/microtvm/reference-vm/zephyr && poetry env list --full-path) | sed -E 's/^(.*)[[:space:]]\(Activated\)\$/\1/g')" >>~/.profile
diff --git a/docker/install/ubuntu_init_zephyr_project.sh b/docker/install/ubuntu_init_zephyr_project.sh
index c7be46b5ecb9..2ec60a0efeb0 100755
--- a/docker/install/ubuntu_init_zephyr_project.sh
+++ b/docker/install/ubuntu_init_zephyr_project.sh
@@ -25,21 +25,24 @@
 #   --commit is the commit hash number of zephyrproject repository. If not specified, it uses the latest commit.
 #
 
+set -x
+
 DOWNLOAD_DIR=$1
 shift
 ZEPHYR_BRANCH=$1
 shift
 
-commit="0"
+commit_hash="0"
 if [ "$1" == "--commit" ]; then
-    commit=$1
+    shift
+    commit_hash=$1
 fi
 
 west init --mr ${ZEPHYR_BRANCH} ${DOWNLOAD_DIR}
 
-if [ "$commit" != "0"]; then
+if [ "$commit_hash" != "0" ]; then
     cd ${DOWNLOAD_DIR}/zephyr
-    git checkout ${commit}
+    git checkout ${commit_hash}
 fi
 
 cd ${DOWNLOAD_DIR}/zephyr
diff --git a/docker/install/ubuntu_install_qemu.sh b/docker/install/ubuntu_install_qemu.sh
index be53e5aa388f..b31d0c748de9 100755
--- a/docker/install/ubuntu_install_qemu.sh
+++ b/docker/install/ubuntu_install_qemu.sh
@@ -26,7 +26,12 @@
 set -e
 set -o pipefail
 
-QEMU_DIR=qemu-5.1.0
+QEMU_NAME=qemu-5.1.0
+QEMU_SIG_FILE=${QEMU_NAME}.tar.xz.sig
+QEMU_TAR_FILE=${QEMU_NAME}.tar.xz
+
+# Clean previous build
+rm -rf ${QEMU_NAME} ${QEMU_SIG_FILE} ${QEMU_TAR_FILE}
 
 # Get number of cores for build
 if [[ "${TVM_CI_NUM_CORES}" ]]; then
@@ -44,19 +49,12 @@ else
     target_list="aarch64-softmmu,arm-softmmu,i386-softmmu,riscv32-softmmu,riscv64-softmmu,x86_64-softmmu"
 fi
 
-# Check if QEMU already built
-rebuild=0
-if [ -d ${QEMU_DIR} ]; then
-    rebuild=1
-fi
-
-if [ $rebuild -eq 0 ]; then
-    sudo sed -i '/deb-src/s/^# //' /etc/apt/sources.list
-    apt update
-    apt-get -y build-dep qemu
+sudo sed -i '/deb-src/s/^# //' /etc/apt/sources.list
+apt update
+apt-get -y build-dep qemu
 
-    gpg --keyserver keys.gnupg.net --recv-keys 0x3353C9CEF108B584
-    cat <<EOF | gpg --dearmor >qemu-5.1.0.tar.xz.sig
+gpg --keyserver keys.gnupg.net --recv-keys 0x3353C9CEF108B584
+cat <<EOF | gpg --dearmor >qemu-5.1.0.tar.xz.sig
 -----BEGIN PGP ARMORED FILE-----
 Comment: Use "gpg --dearmor" for unpacking
 
@@ -70,13 +68,12 @@ p5ez/+2k4VAIwIQoP5DoO06waLBffvLIAdPPKYsx71K67OoGG2svc7duC/+5qf1x
 =hCS7
 -----END PGP ARMORED FILE-----
 EOF
-    curl -OLs https://download.qemu.org/qemu-5.1.0.tar.xz
-    gpg --verify qemu-5.1.0.tar.xz.sig
+curl -OLs https://download.qemu.org/qemu-5.1.0.tar.xz
+gpg --verify ${QEMU_SIG_FILE}
 
-    tar -xf qemu-5.1.0.tar.xz
-fi
+tar -xf ${QEMU_TAR_FILE}
 
-cd qemu-5.1.0
+cd ${QEMU_NAME}
 ./configure --target-list=${target_list}
 make -j${num_cores}
 sudo make install

From 38cc261faeb7e3a888fb427ebf2ffb4f0efe8521 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Fri, 4 Jun 2021 09:08:36 -0700
Subject: [PATCH 23/40] add docker import

---
 docker/bash.sh | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/docker/bash.sh b/docker/bash.sh
index 3c4be21c9b9e..6d2f1153c063 100755
--- a/docker/bash.sh
+++ b/docker/bash.sh
@@ -20,7 +20,7 @@
 #
 # Start a bash, mount /workspace to be current directory.
 #
-# Usage: bash.sh <CONTAINER_TYPE> [-i] [--net=host] <CONTAINER_NAME>  <COMMAND>
+# Usage: bash.sh <CONTAINER_TYPE> [-i] [--net=host] [--mount path] <CONTAINER_NAME>  <COMMAND>
 #
 # Usage: docker/bash.sh <CONTAINER_NAME>
 #     Starts an interactive session
@@ -46,6 +46,14 @@ if [[ "$1" == "--net=host" ]]; then
     shift 1
 fi
 
+# Mount external directory to the docker
+CI_DOCKER_MOUNT_DIR=( )
+if [[ "$1" == "--mount" ]]; then
+    shift 1
+    CI_DOCKER_MOUNT_DIR+="$1"
+    shift 1
+fi
+
 if [ "$#" -lt 1 ]; then
     echo "Usage: docker/bash.sh [-i] [--net=host] <CONTAINER_NAME> [COMMAND]"
     exit -1
@@ -154,6 +162,7 @@ ${DOCKER_BINARY} run --rm --pid=host\
     ${WORKSPACE_VOLUMES}\
     -v ${WORKSPACE}:/workspace \
     -v ${SCRIPT_DIR}:/docker \
+    -v ${CI_DOCKER_MOUNT_DIR}:${CI_DOCKER_MOUNT_DIR} \
     "${EXTRA_MOUNTS[@]}" \
     -w /workspace \
     -e "CI_BUILD_HOME=/workspace" \

From b8591f6e19d20f59142e594e89d22766901b0895 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Fri, 4 Jun 2021 12:57:59 -0700
Subject: [PATCH 24/40] cleanup

---
 apps/microtvm/reference-vm/zephyr/Vagrantfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/apps/microtvm/reference-vm/zephyr/Vagrantfile b/apps/microtvm/reference-vm/zephyr/Vagrantfile
index d011e546f44b..be41c0b733e5 100644
--- a/apps/microtvm/reference-vm/zephyr/Vagrantfile
+++ b/apps/microtvm/reference-vm/zephyr/Vagrantfile
@@ -17,7 +17,6 @@
 
 Vagrant.configure("2") do |config|
   config.vm.box = "tlcpack/microtvm-zephyr-2.5"
-  config.vm.box_version = "0.0.2"
 
   if ENV.has_key?("TVM_RVM_NUM_CORES")
     num_cores = ENV["TVM_RVM_NUM_CORES"]

From 76fe98cea1975c0a93dc760a77fcff80607fdc07 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Mon, 7 Jun 2021 10:27:06 -0700
Subject: [PATCH 25/40] fix

---
 apps/microtvm/reference-vm/zephyr/provision_setup.sh | 1 +
 apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh     | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/apps/microtvm/reference-vm/zephyr/provision_setup.sh b/apps/microtvm/reference-vm/zephyr/provision_setup.sh
index 6ed9d576dfbe..c40f17849442 100644
--- a/apps/microtvm/reference-vm/zephyr/provision_setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/provision_setup.sh
@@ -26,6 +26,7 @@ apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
 
 # Build poetry
 cd apps/microtvm/reference-vm/zephyr
+
 poetry env use 3.6
 # NOTE: due to https://github.com/python-poetry/poetry/issues/2247, download torch here.
 poetry run pip3 install torch==1.4.0 torchvision==0.5.0
diff --git a/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh b/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
index 379b002de654..526cae5086ec 100755
--- a/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
+++ b/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
@@ -19,10 +19,10 @@
 set -e
 
 # Get number of cores for build
-if [[ "${TVM_CI_NUM_CORES}" ]]; then
+if [ -n "${TVM_CI_NUM_CORES}" ]; then
   num_cores=${TVM_CI_NUM_CORES}
 else
-  num_cores=2
+  num_cores=4
 fi
 
 cd "$(dirname $0)"

From aa0732907b3e84930551098b93127fe6d2b7f7da Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Mon, 7 Jun 2021 11:25:40 -0700
Subject: [PATCH 26/40] cleanup

---
 docker/bash.sh | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/docker/bash.sh b/docker/bash.sh
index 6d2f1153c063..3c4be21c9b9e 100755
--- a/docker/bash.sh
+++ b/docker/bash.sh
@@ -20,7 +20,7 @@
 #
 # Start a bash, mount /workspace to be current directory.
 #
-# Usage: bash.sh <CONTAINER_TYPE> [-i] [--net=host] [--mount path] <CONTAINER_NAME>  <COMMAND>
+# Usage: bash.sh <CONTAINER_TYPE> [-i] [--net=host] <CONTAINER_NAME>  <COMMAND>
 #
 # Usage: docker/bash.sh <CONTAINER_NAME>
 #     Starts an interactive session
@@ -46,14 +46,6 @@ if [[ "$1" == "--net=host" ]]; then
     shift 1
 fi
 
-# Mount external directory to the docker
-CI_DOCKER_MOUNT_DIR=( )
-if [[ "$1" == "--mount" ]]; then
-    shift 1
-    CI_DOCKER_MOUNT_DIR+="$1"
-    shift 1
-fi
-
 if [ "$#" -lt 1 ]; then
     echo "Usage: docker/bash.sh [-i] [--net=host] <CONTAINER_NAME> [COMMAND]"
     exit -1
@@ -162,7 +154,6 @@ ${DOCKER_BINARY} run --rm --pid=host\
     ${WORKSPACE_VOLUMES}\
     -v ${WORKSPACE}:/workspace \
     -v ${SCRIPT_DIR}:/docker \
-    -v ${CI_DOCKER_MOUNT_DIR}:${CI_DOCKER_MOUNT_DIR} \
     "${EXTRA_MOUNTS[@]}" \
     -w /workspace \
     -e "CI_BUILD_HOME=/workspace" \

From 721a18a707ce9ab32c2b25664e7fb60de187ee28 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Mon, 7 Jun 2021 11:28:38 -0700
Subject: [PATCH 27/40] fix

---
 docker/install/ubuntu_install_qemu.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/install/ubuntu_install_qemu.sh b/docker/install/ubuntu_install_qemu.sh
index b31d0c748de9..b1d375253e05 100755
--- a/docker/install/ubuntu_install_qemu.sh
+++ b/docker/install/ubuntu_install_qemu.sh
@@ -34,7 +34,7 @@ QEMU_TAR_FILE=${QEMU_NAME}.tar.xz
 rm -rf ${QEMU_NAME} ${QEMU_SIG_FILE} ${QEMU_TAR_FILE}
 
 # Get number of cores for build
-if [[ "${TVM_CI_NUM_CORES}" ]]; then
+if [ -n "${TVM_CI_NUM_CORES}" ]; then
   num_cores=${TVM_CI_NUM_CORES}
 else
   num_cores=2

From 5fd7f7334f1613e2d4a0095ed69ea4c999df2777 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Mon, 7 Jun 2021 13:19:27 -0700
Subject: [PATCH 28/40] fix zephyr path

---
 apps/microtvm/reference-vm/zephyr/provision_setup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/microtvm/reference-vm/zephyr/provision_setup.sh b/apps/microtvm/reference-vm/zephyr/provision_setup.sh
index c40f17849442..f95c7e24f5aa 100644
--- a/apps/microtvm/reference-vm/zephyr/provision_setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/provision_setup.sh
@@ -42,7 +42,7 @@ echo "downloaded and cached for future use."
 echo "------------------------------[ TVM Message ]------------------------------"
 poetry lock -vvv
 poetry install
-poetry run pip3 install -r ~/zephyrproject/zephyr/scripts/requirements.txt
+poetry run pip3 install -r ${ZEPHYR_BASE}/scripts/requirements.txt
 
 echo "export TVM_LIBRARY_PATH=\"$TVM_HOME\"/build-microtvm" >>~/.profile
 echo "VENV_PATH=\$((cd \"$TVM_HOME\"/apps/microtvm/reference-vm/zephyr && poetry env list --full-path) | sed -E 's/^(.*)[[:space:]]\(Activated\)\$/\1/g')" >>~/.profile

From ed9f0c455b0ea3cb181427d17eb944b33ae0b9c2 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Mon, 7 Jun 2021 13:31:01 -0700
Subject: [PATCH 29/40] fix

---
 docker/install/ubuntu_init_zephyr_project.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/install/ubuntu_init_zephyr_project.sh b/docker/install/ubuntu_init_zephyr_project.sh
index 2ec60a0efeb0..a2433deffccf 100755
--- a/docker/install/ubuntu_init_zephyr_project.sh
+++ b/docker/install/ubuntu_init_zephyr_project.sh
@@ -45,6 +45,6 @@ if [ "$commit_hash" != "0" ]; then
     git checkout ${commit_hash}
 fi
 
-cd ${DOWNLOAD_DIR}/zephyr
+cd ${DOWNLOAD_DIR}
 west update
 west zephyr-export

From 153aeac3413bfae5d3d0aa1cc72979f3905f5ee1 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 8 Jun 2021 15:01:57 -0700
Subject: [PATCH 30/40] fix

---
 apps/microtvm/reference-vm/base-box-tool.py               | 1 -
 .../reference-vm/zephyr/base-box/test-config.json         | 8 +++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py
index 99fdba518b69..6c7d7e8ac06f 100755
--- a/apps/microtvm/reference-vm/base-box-tool.py
+++ b/apps/microtvm/reference-vm/base-box-tool.py
@@ -204,7 +204,6 @@ def generate_packer_config(file_path, providers):
         {
             "type": "shell",
             "script": "base_box_provision.sh",
-            "execute_command": f"chmod +x {{{{ .Path }}}}; {{{{ .Vars }}}} {{{{ .Path }}}}",
         }
     )
 
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/test-config.json b/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
index e196d0a8b18a..9b9e0c19c852 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
+++ b/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
@@ -2,16 +2,18 @@
     "stm32f746xx": {
         "vid_hex": "0483",
         "pid_hex": "374b",
-        "test_cmd": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=stm32f746xx"]
+        "test_cmd_zephyr": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=stm32f746xx"]
         },
     "nrf5340dk": {
         "vid_hex": "1366",
         "pid_hex": "1055",
-        "test_cmd": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=nrf5340dk"]
+        "test_cmd_zephyr": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=nrf5340dk"],
+        "test_cmd_zephyr_aot": ["pytest", "tests/micro/zephyr/test_zephyr_aot.py", "--microtvm-platforms=nrf5340dk"]
         },
     "mps2_an521": {
         "vid_hex": "",
         "pid_hex": "",
-        "test_cmd": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=mps2_an521"]
+        "test_cmd_zephyr": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=mps2_an521"],
+        "test_cmd_zephyr_aot": ["pytest", "tests/micro/zephyr/test_zephyr_aot.py", "--microtvm-platforms=mps2_an521"]
         }
 }

From b3dcdc83d300a662f66fd92006530803403ad227 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 8 Jun 2021 18:19:39 -0700
Subject: [PATCH 31/40] address comments

---
 apps/microtvm/reference-vm/README.md           |  8 ++++----
 apps/microtvm/reference-vm/base-box-tool.py    | 18 ++++++++++--------
 .../zephyr/base-box/base_box_provision.sh      |  2 --
 .../reference-vm/zephyr/rebuild-tvm.sh         |  3 ++-
 4 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/apps/microtvm/reference-vm/README.md b/apps/microtvm/reference-vm/README.md
index 874facbcba2d..7ff75c75b4f9 100644
--- a/apps/microtvm/reference-vm/README.md
+++ b/apps/microtvm/reference-vm/README.md
@@ -49,19 +49,19 @@ Reference VMs are organized as follows:
 
 ## Creating Releases
 
-1. Build the base box for the given platform: `$ ./base-box-tool.py <--provider=> build <platform>`
+1. Build the base box for the given platform: `$ ./base-box-tool.py [--provider=<provider>] build <platform>`
 2. Run release tests for each platform:
     1. Connect any needed hardware to the VM host machine.
-    2. Run tests: `$ ./base-box-tool.py <--provider=> test <--microtvm-platform=> <platform> [--test-device-serial=<serial>]`. This
+    2. Run tests: `$ ./base-box-tool.py [--provider=<provider>] test [--microtvm-platform=<platform>] <platform> [--test-device-serial=<serial>]`. This
        command does the following for each provider:
         1. Copies all files inside `./<platform>` except `.vagrant` and `base-box` to
            `./release-test`. This is done to avoid reusing any VM the developer may have started.
-        2. Executes `$ vagrant up --provider=<provider>`.
+        2. Executes `$ vagrant up [--provider=<provider>]`.
         3. Finds an attached USB device matching the VID and PID specified in `test-config.json`,
            and if `--test-device-serial` was given, that serial number (as reported to USB). Creates
            a rule to autoconnect this device to the VM, and also attaches it to the VM>
         4. SSHs to the VM, `cd` to the TVM root directory, and runs `test_cmd` from
            `test-config.json`. Nonzero status means failure.
 3. If release tests fail, fix them and restart from step 1.
-4. If release tests pass: `$ ./base-box-tool.py <--provider=> release <--release-version=> <--platform-version=> <platform>`. Be sure you've logged
+4. If release tests pass: `$ ./base-box-tool.py [--provider=<provider>] release <--release-version=<version>> <--platform-version=<version>> <platform>`. Be sure you've logged
    in to Vagrant Cloud using the `vagrant` tool.
diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py
index 99fdba518b69..6f93a2e6f258 100755
--- a/apps/microtvm/reference-vm/base-box-tool.py
+++ b/apps/microtvm/reference-vm/base-box-tool.py
@@ -48,6 +48,7 @@
     "mps2_an521",
 )
 
+PACKER_FILE_NAME = "packer.json"
 
 def parse_virtualbox_devices():
     output = subprocess.check_output(["VBoxManage", "list", "usbhost"], encoding="utf-8")
@@ -174,8 +175,8 @@ def attach_vmware(uuid, vid_hex=None, pid_hex=None, serial=None):
 }
 
 EXTRA_SCRIPTS = (
-    "../../../../../docker/install/ubuntu_init_zephyr_project.sh",
-    "../../../../../docker/install/ubuntu_install_qemu.sh",
+    "docker/install/ubuntu_init_zephyr_project.sh",
+    "docker/install/ubuntu_install_qemu.sh",
 )
 
 
@@ -196,9 +197,11 @@ def generate_packer_config(file_path, providers):
             }
         )
 
+    repo_root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], cwd=os.path.dirname(__file__))
     for script in EXTRA_SCRIPTS:
-        filename = os.path.basename(script)
-        provisioners.append({"type": "file", "source": script, "destination": f"~/{filename}"})
+        file_path = os.path.join(repo_root, script)
+        filename = os.path.basename(file_path)
+        provisioners.append({"type": "file", "source": file_path, "destination": f"~/{filename}"})
 
     provisioners.append(
         {
@@ -215,15 +218,14 @@ def generate_packer_config(file_path, providers):
                 "provisioners": provisioners,
             },
             f,
-            sort_keys=False,
+            sort_keys=True,
             indent=2,
         )
 
 
 def build_command(args):
-    packer_file_name = "packer.json"
     generate_packer_config(
-        os.path.join(THIS_DIR, args.platform, "base-box", packer_file_name),
+        os.path.join(THIS_DIR, args.platform, "base-box", PACKER_FILE_NAME),
         args.provider or ALL_PROVIDERS,
     )
     env = copy.copy(os.environ)
@@ -233,7 +235,7 @@ def build_command(args):
     if args.debug_packer:
         packer_args += ["-debug"]
 
-    packer_args += [packer_file_name]
+    packer_args += [PACKER_FILE_NAME]
     subprocess.check_call(
         packer_args, cwd=os.path.join(THIS_DIR, args.platform, "base-box"), env=env
     )
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh
index af8e36f41ca1..34ea82888e69 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh
+++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh
@@ -26,9 +26,7 @@ cd ~
 ~/ubuntu_init_zephyr_project.sh ~/zephyrproject v2.5-branch --commit dabf23758417fd041fec2a2a821d8f526afac29d
 
 # Build QEMU
-cd ~
 sudo ~/ubuntu_install_qemu.sh --target-list arm-softmmu
 
 # Cleanup
-cd ~
 rm -f *.sh
diff --git a/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh b/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
index 526cae5086ec..1cebcf7166af 100755
--- a/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
+++ b/apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh
@@ -22,7 +22,8 @@ set -e
 if [ -n "${TVM_CI_NUM_CORES}" ]; then
   num_cores=${TVM_CI_NUM_CORES}
 else
-  num_cores=4
+  # default setup for Vagrantfile
+  num_cores=2
 fi
 
 cd "$(dirname $0)"

From 3a2af66f4b2a1cebc8b22c7dd283ed2f86336d37 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 8 Jun 2021 18:27:24 -0700
Subject: [PATCH 32/40] fix test

---
 apps/microtvm/reference-vm/base-box-tool.py           | 11 ++++++++---
 .../reference-vm/zephyr/base-box/test-config.json     |  3 ++-
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py
index d01e3c9f5291..ccc50e3841e1 100755
--- a/apps/microtvm/reference-vm/base-box-tool.py
+++ b/apps/microtvm/reference-vm/base-box-tool.py
@@ -243,7 +243,8 @@ def build_command(args):
 REQUIRED_TEST_CONFIG_KEYS = {
     "vid_hex": str,
     "pid_hex": str,
-    "test_cmd": list,
+    "test_cmd_zephyr": list,
+    "test_cmd_zephyr_aot": list,
 }
 
 
@@ -328,8 +329,12 @@ def do_run_release_test(release_test_dir, provider_name, test_config, test_devic
     def _quote_cmd(cmd):
         return " ".join(shlex.quote(a) for a in cmd)
 
-    test_cmd = _quote_cmd(["cd", tvm_home]) + " && " + _quote_cmd(test_config["test_cmd"])
-    subprocess.check_call(["vagrant", "ssh", "-c", f"bash -ec '{test_cmd}'"], cwd=release_test_dir)
+    test_cmd_zephyr = _quote_cmd(["cd", tvm_home]) + " && " + _quote_cmd(test_config["test_cmd_zephyr"])
+    test_cmd_zephyr_aot = _quote_cmd(["cd", tvm_home]) + " && " + _quote_cmd(test_config["test_cmd_zephyr_aot"])
+    if test_cmd_zephyr:
+        subprocess.check_call(["vagrant", "ssh", "-c", f"bash -ec '{test_cmd_zephyr}'"], cwd=release_test_dir)
+    if test_cmd_zephyr_aot:
+        subprocess.check_call(["vagrant", "ssh", "-c", f"bash -ec '{test_cmd_zephyr_aot}'"], cwd=release_test_dir)
 
 
 def test_command(args):
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/test-config.json b/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
index 9b9e0c19c852..19de3f31ea89 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
+++ b/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
@@ -2,7 +2,8 @@
     "stm32f746xx": {
         "vid_hex": "0483",
         "pid_hex": "374b",
-        "test_cmd_zephyr": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=stm32f746xx"]
+        "test_cmd_zephyr": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=stm32f746xx"],
+        "test_cmd_zephyr_aot": []
         },
     "nrf5340dk": {
         "vid_hex": "1366",

From 21c35df41af744b98679c5b9ff76e24850e1795d Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 8 Jun 2021 18:58:15 -0700
Subject: [PATCH 33/40] fix

---
 apps/microtvm/reference-vm/base-box-tool.py | 27 +++++++++++++++------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py
index ccc50e3841e1..d76033b0821e 100755
--- a/apps/microtvm/reference-vm/base-box-tool.py
+++ b/apps/microtvm/reference-vm/base-box-tool.py
@@ -50,6 +50,7 @@
 
 PACKER_FILE_NAME = "packer.json"
 
+
 def parse_virtualbox_devices():
     output = subprocess.check_output(["VBoxManage", "list", "usbhost"], encoding="utf-8")
     devices = []
@@ -197,11 +198,13 @@ def generate_packer_config(file_path, providers):
             }
         )
 
-    repo_root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], cwd=os.path.dirname(__file__))
+    repo_root = subprocess.check_output(
+        ["git", "rev-parse", "--show-toplevel"], cwd=os.path.dirname(__file__), encoding="utf-8"
+    ).strip()
     for script in EXTRA_SCRIPTS:
-        file_path = os.path.join(repo_root, script)
-        filename = os.path.basename(file_path)
-        provisioners.append({"type": "file", "source": file_path, "destination": f"~/{filename}"})
+        script_path = os.path.join(repo_root, script)
+        filename = os.path.basename(script_path)
+        provisioners.append({"type": "file", "source": script_path, "destination": f"~/{filename}"})
 
     provisioners.append(
         {
@@ -329,12 +332,20 @@ def do_run_release_test(release_test_dir, provider_name, test_config, test_devic
     def _quote_cmd(cmd):
         return " ".join(shlex.quote(a) for a in cmd)
 
-    test_cmd_zephyr = _quote_cmd(["cd", tvm_home]) + " && " + _quote_cmd(test_config["test_cmd_zephyr"])
-    test_cmd_zephyr_aot = _quote_cmd(["cd", tvm_home]) + " && " + _quote_cmd(test_config["test_cmd_zephyr_aot"])
+    test_cmd_zephyr = (
+        _quote_cmd(["cd", tvm_home]) + " && " + _quote_cmd(test_config["test_cmd_zephyr"])
+    )
+    test_cmd_zephyr_aot = (
+        _quote_cmd(["cd", tvm_home]) + " && " + _quote_cmd(test_config["test_cmd_zephyr_aot"])
+    )
     if test_cmd_zephyr:
-        subprocess.check_call(["vagrant", "ssh", "-c", f"bash -ec '{test_cmd_zephyr}'"], cwd=release_test_dir)
+        subprocess.check_call(
+            ["vagrant", "ssh", "-c", f"bash -ec '{test_cmd_zephyr}'"], cwd=release_test_dir
+        )
     if test_cmd_zephyr_aot:
-        subprocess.check_call(["vagrant", "ssh", "-c", f"bash -ec '{test_cmd_zephyr_aot}'"], cwd=release_test_dir)
+        subprocess.check_call(
+            ["vagrant", "ssh", "-c", f"bash -ec '{test_cmd_zephyr_aot}'"], cwd=release_test_dir
+        )
 
 
 def test_command(args):

From 70f3c7d840028d81823b99ae12ae6969b80d9a91 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Tue, 8 Jun 2021 15:57:18 -0700
Subject: [PATCH 34/40] add wait

---
 python/tvm/micro/contrib/zephyr.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/python/tvm/micro/contrib/zephyr.py b/python/tvm/micro/contrib/zephyr.py
index b7d7496b7440..c3cda3717a48 100644
--- a/python/tvm/micro/contrib/zephyr.py
+++ b/python/tvm/micro/contrib/zephyr.py
@@ -30,6 +30,8 @@
 import shutil
 import subprocess
 import sys
+import threading
+import queue
 
 import yaml
 
@@ -630,6 +632,8 @@ def __init__(self, base_dir, startup_timeout_sec=5.0, qemu_debugger=None, **kwar
         self.fd_transport = None
         self.pipe_dir = None
         self.qemu_debugger = qemu_debugger
+        self._queue = queue.Queue()
+        self._qemu_ready_msg = "Qready"
 
     def timeouts(self):
         return TransportTimeouts(
@@ -658,7 +662,9 @@ def open(self):
             ["make", "run", f"QEMU_PIPE={self.pipe}"],
             cwd=self.base_dir,
             **self.kwargs,
+            stdout=subprocess.PIPE,
         )
+        self._wait_for_qemu()
 
         if self.qemu_debugger is not None:
             self.qemu_debugger.start()
@@ -703,6 +709,21 @@ def write(self, data, timeout_sec):
             raise TransportClosedError()
         return self.fd_transport.write(data, timeout_sec)
 
+    def _qemu_check_stdout(self):
+        for line in self.proc.stdout:
+            line = str(line)
+            _LOG.debug(line)
+            if "[QEMU] CPU" in line:
+                self._queue.put(self._qemu_ready_msg)
+                return
+
+    def _wait_for_qemu(self):
+        threading.Thread(target=self._qemu_check_stdout, daemon=True).start()
+        while True:
+            item = self._queue.get()
+            if item == self._qemu_ready_msg:
+                return True
+
 
 class ZephyrDebugger(debugger.GdbDebugger):
     """A Zephyr debugger implementation."""

From 87b4721e496ae1387317b6a8284102bb2a043662 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Thu, 10 Jun 2021 16:20:21 -0700
Subject: [PATCH 35/40] comments

---
 .../reference-vm/zephyr/base-box/base_box_provision.sh        | 2 +-
 docker/install/ubuntu_init_zephyr_project.sh                  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh
index 34ea82888e69..08678dcd4fc5 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh
+++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh
@@ -23,7 +23,7 @@ source ~/.profile
 
 # Init Zephyr
 cd ~
-~/ubuntu_init_zephyr_project.sh ~/zephyrproject v2.5-branch --commit dabf23758417fd041fec2a2a821d8f526afac29d
+~/ubuntu_init_zephyr_project.sh ~/zephyr v2.5-branch --commit dabf23758417fd041fec2a2a821d8f526afac29d
 
 # Build QEMU
 sudo ~/ubuntu_install_qemu.sh --target-list arm-softmmu
diff --git a/docker/install/ubuntu_init_zephyr_project.sh b/docker/install/ubuntu_init_zephyr_project.sh
index a2433deffccf..573ff30c38a8 100755
--- a/docker/install/ubuntu_init_zephyr_project.sh
+++ b/docker/install/ubuntu_init_zephyr_project.sh
@@ -32,7 +32,7 @@ shift
 ZEPHYR_BRANCH=$1
 shift
 
-commit_hash="0"
+commit_hash=
 if [ "$1" == "--commit" ]; then
     shift
     commit_hash=$1
@@ -40,7 +40,7 @@ fi
 
 west init --mr ${ZEPHYR_BRANCH} ${DOWNLOAD_DIR}
 
-if [ "$commit_hash" != "0" ]; then
+if [ -n "$commit_hash" ]; then
     cd ${DOWNLOAD_DIR}/zephyr
     git checkout ${commit_hash}
 fi

From ca9981aa1bc657fcc7b85dbbfa8257a381d6d1f4 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Thu, 10 Jun 2021 16:49:07 -0700
Subject: [PATCH 36/40] changed test to script

---
 apps/microtvm/reference-vm/base-box-tool.py   | 26 +++++++--------
 .../zephyr/base-box/base_box_test.sh          | 33 +++++++++++++++++++
 .../zephyr/base-box/test-config.json          | 12 ++-----
 3 files changed, 47 insertions(+), 24 deletions(-)
 create mode 100755 apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh

diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py
index d76033b0821e..997f739b3b5d 100755
--- a/apps/microtvm/reference-vm/base-box-tool.py
+++ b/apps/microtvm/reference-vm/base-box-tool.py
@@ -246,8 +246,6 @@ def build_command(args):
 REQUIRED_TEST_CONFIG_KEYS = {
     "vid_hex": str,
     "pid_hex": str,
-    "test_cmd_zephyr": list,
-    "test_cmd_zephyr_aot": list,
 }
 
 
@@ -332,20 +330,17 @@ def do_run_release_test(release_test_dir, provider_name, test_config, test_devic
     def _quote_cmd(cmd):
         return " ".join(shlex.quote(a) for a in cmd)
 
-    test_cmd_zephyr = (
-        _quote_cmd(["cd", tvm_home]) + " && " + _quote_cmd(test_config["test_cmd_zephyr"])
-    )
-    test_cmd_zephyr_aot = (
-        _quote_cmd(["cd", tvm_home]) + " && " + _quote_cmd(test_config["test_cmd_zephyr_aot"])
-    )
-    if test_cmd_zephyr:
-        subprocess.check_call(
-            ["vagrant", "ssh", "-c", f"bash -ec '{test_cmd_zephyr}'"], cwd=release_test_dir
-        )
-    if test_cmd_zephyr_aot:
-        subprocess.check_call(
-            ["vagrant", "ssh", "-c", f"bash -ec '{test_cmd_zephyr_aot}'"], cwd=release_test_dir
+    test_cmd = (
+        _quote_cmd(["cd", tvm_home])
+        + " && "
+        + _quote_cmd(
+            [
+                "apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh",
+                test_config["microtvm_platform"],
+            ]
         )
+    )
+    subprocess.check_call(["vagrant", "ssh", "-c", f"bash -ec '{test_cmd}'"], cwd=release_test_dir)
 
 
 def test_command(args):
@@ -365,6 +360,7 @@ def test_command(args):
 
         microtvm_test_platform["vid_hex"] = microtvm_test_platform["vid_hex"].lower()
         microtvm_test_platform["pid_hex"] = microtvm_test_platform["pid_hex"].lower()
+        microtvm_test_platform["microtvm_platform"] = args.microtvm_platform
 
     providers = args.provider
     provider_passed = {p: False for p in providers}
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh
new file mode 100755
index 000000000000..8fb80f20d757
--- /dev/null
+++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh
@@ -0,0 +1,33 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Usage: base_box_test.sh <TARGET>
+#     Execute microTVM Zephyr tests.
+#
+
+set -e
+set -x
+
+target=$1
+
+pytest tests/micro/zephyr/test_zephyr.py --microtvm-platforms=${target}
+
+if [ $target == "nrf5340dk" ] || [ $target == "mps2_an521" ]; then
+    # It breaks for stm32f746xx.
+    pytest tests/micro/zephyr/test_zephyr_aot.py --microtvm-platforms=${target}
+fi
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/test-config.json b/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
index 19de3f31ea89..48b6915a10f4 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
+++ b/apps/microtvm/reference-vm/zephyr/base-box/test-config.json
@@ -1,20 +1,14 @@
 {
     "stm32f746xx": {
         "vid_hex": "0483",
-        "pid_hex": "374b",
-        "test_cmd_zephyr": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=stm32f746xx"],
-        "test_cmd_zephyr_aot": []
+        "pid_hex": "374b"
         },
     "nrf5340dk": {
         "vid_hex": "1366",
-        "pid_hex": "1055",
-        "test_cmd_zephyr": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=nrf5340dk"],
-        "test_cmd_zephyr_aot": ["pytest", "tests/micro/zephyr/test_zephyr_aot.py", "--microtvm-platforms=nrf5340dk"]
+        "pid_hex": "1055"
         },
     "mps2_an521": {
         "vid_hex": "",
-        "pid_hex": "",
-        "test_cmd_zephyr": ["pytest", "tests/micro/zephyr/test_zephyr.py", "--microtvm-platforms=mps2_an521"],
-        "test_cmd_zephyr_aot": ["pytest", "tests/micro/zephyr/test_zephyr_aot.py", "--microtvm-platforms=mps2_an521"]
+        "pid_hex": ""
         }
 }

From 510e4fad434dcda985179e99bef768f9f626ffce Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Thu, 10 Jun 2021 16:58:37 -0700
Subject: [PATCH 37/40] add checks

---
 apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh
index 8fb80f20d757..eae7d6a0432a 100755
--- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh
+++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh
@@ -23,6 +23,11 @@
 set -e
 set -x
 
+if [ "$#" -lt 1 ]; then
+    echo "Usage: base_box_test.sh <TARGET>"
+    exit -1
+fi
+
 target=$1
 
 pytest tests/micro/zephyr/test_zephyr.py --microtvm-platforms=${target}

From 635db461bdd5da35e973006cb5dac061c50b072a Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Thu, 10 Jun 2021 22:34:17 -0700
Subject: [PATCH 38/40] fix zephyr

---
 apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh
index 31efdb851d19..e8385af9f663 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh
+++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh
@@ -103,7 +103,7 @@ sudo udevadm control --reload
 # Poetry
 curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python3
 sed -i "/^# If not running interactively,/ i source \$HOME/.poetry/env" ~/.bashrc
-sed -i "/^# If not running interactively,/ i export ZEPHYR_BASE=$HOME/zephyrproject/zephyr" ~/.bashrc
+sed -i "/^# If not running interactively,/ i export ZEPHYR_BASE=$HOME/zephyr/zephyr" ~/.bashrc
 sed -i "/^# If not running interactively,/ i\\ " ~/.bashrc
 
 # Clean box for packaging as a base box

From b535e84fb637573a8f578c52753cdf2df1df7882 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Fri, 11 Jun 2021 09:32:26 -0700
Subject: [PATCH 39/40] Revert "add wait"

This reverts commit 70f3c7d840028d81823b99ae12ae6969b80d9a91.
---
 python/tvm/micro/contrib/zephyr.py | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/python/tvm/micro/contrib/zephyr.py b/python/tvm/micro/contrib/zephyr.py
index c3cda3717a48..b7d7496b7440 100644
--- a/python/tvm/micro/contrib/zephyr.py
+++ b/python/tvm/micro/contrib/zephyr.py
@@ -30,8 +30,6 @@
 import shutil
 import subprocess
 import sys
-import threading
-import queue
 
 import yaml
 
@@ -632,8 +630,6 @@ def __init__(self, base_dir, startup_timeout_sec=5.0, qemu_debugger=None, **kwar
         self.fd_transport = None
         self.pipe_dir = None
         self.qemu_debugger = qemu_debugger
-        self._queue = queue.Queue()
-        self._qemu_ready_msg = "Qready"
 
     def timeouts(self):
         return TransportTimeouts(
@@ -662,9 +658,7 @@ def open(self):
             ["make", "run", f"QEMU_PIPE={self.pipe}"],
             cwd=self.base_dir,
             **self.kwargs,
-            stdout=subprocess.PIPE,
         )
-        self._wait_for_qemu()
 
         if self.qemu_debugger is not None:
             self.qemu_debugger.start()
@@ -709,21 +703,6 @@ def write(self, data, timeout_sec):
             raise TransportClosedError()
         return self.fd_transport.write(data, timeout_sec)
 
-    def _qemu_check_stdout(self):
-        for line in self.proc.stdout:
-            line = str(line)
-            _LOG.debug(line)
-            if "[QEMU] CPU" in line:
-                self._queue.put(self._qemu_ready_msg)
-                return
-
-    def _wait_for_qemu(self):
-        threading.Thread(target=self._qemu_check_stdout, daemon=True).start()
-        while True:
-            item = self._queue.get()
-            if item == self._qemu_ready_msg:
-                return True
-
 
 class ZephyrDebugger(debugger.GdbDebugger):
     """A Zephyr debugger implementation."""

From b43c3daa8525a86496fe9e7b50e536c9e742bc22 Mon Sep 17 00:00:00 2001
From: Mehrdad Hessar <mhessar@octoml.ai>
Date: Fri, 11 Jun 2021 09:49:56 -0700
Subject: [PATCH 40/40] address comments

---
 apps/microtvm/reference-vm/base-box-tool.py       |  2 ++
 .../zephyr/base-box/base_box_provision.sh         |  5 +++++
 .../reference-vm/zephyr/base-box/base_box_test.sh | 15 ++++++++-------
 3 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py
index 997f739b3b5d..c22eff4cdbad 100755
--- a/apps/microtvm/reference-vm/base-box-tool.py
+++ b/apps/microtvm/reference-vm/base-box-tool.py
@@ -175,6 +175,8 @@ def attach_vmware(uuid, vid_hex=None, pid_hex=None, serial=None):
     "vmware_desktop": attach_vmware,
 }
 
+# Extra scripts required to execute on provisioning
+# in zephyr/base-box/base_box_provision.sh
 EXTRA_SCRIPTS = (
     "docker/install/ubuntu_init_zephyr_project.sh",
     "docker/install/ubuntu_install_qemu.sh",
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh
index 08678dcd4fc5..69e6171d06dd 100644
--- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh
+++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh
@@ -15,6 +15,10 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+#
+#   Using this script we can reuse docker/install scripts to configure the reference 
+#   virtual machine similar to CI QEMU setup.
+#
 
 set -e
 set -x
@@ -23,6 +27,7 @@ source ~/.profile
 
 # Init Zephyr
 cd ~
+# Using most recent commit that passes all the tests.
 ~/ubuntu_init_zephyr_project.sh ~/zephyr v2.5-branch --commit dabf23758417fd041fec2a2a821d8f526afac29d
 
 # Build QEMU
diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh
index eae7d6a0432a..8eba63e9e331 100755
--- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh
+++ b/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh
@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Usage: base_box_test.sh <TARGET>
+# Usage: base_box_test.sh <MICROTVM_PLATFORM>
 #     Execute microTVM Zephyr tests.
 #
 
@@ -24,15 +24,16 @@ set -e
 set -x
 
 if [ "$#" -lt 1 ]; then
-    echo "Usage: base_box_test.sh <TARGET>"
+    echo "Usage: base_box_test.sh <MICROTVM_PLATFORM>"
     exit -1
 fi
 
-target=$1
+microtvm_platform=$1
 
-pytest tests/micro/zephyr/test_zephyr.py --microtvm-platforms=${target}
+pytest tests/micro/zephyr/test_zephyr.py --microtvm-platforms=${microtvm_platform}
 
-if [ $target == "nrf5340dk" ] || [ $target == "mps2_an521" ]; then
-    # It breaks for stm32f746xx.
-    pytest tests/micro/zephyr/test_zephyr_aot.py --microtvm-platforms=${target}
+if [ $microtvm_platform == "stm32f746xx" ]; then
+    echo "NOTE: skipped test_zephyr_aot.py on $microtvm_platform -- known failure"
+else
+    pytest tests/micro/zephyr/test_zephyr_aot.py --microtvm-platforms=${microtvm_platform}
 fi