From 5596e7bdc69792fbe66337e09ba5b1bf2e1b972a Mon Sep 17 00:00:00 2001 From: Ramana Radhakrishnan Date: Thu, 23 Apr 2020 11:54:35 +0100 Subject: [PATCH 1/7] [RFC] Pass pytest options globally. In many places having a global pytest flag is useful . For me with the build and test of tvm , I would like to be able to globally pass in pytest options as part of development flow or CI flows where one would like to measure other things regularly that need measurements including pytest coverage data that I would like to experiment with across the stack. This has been achieved with an additional setup-pytest-env.sh file in tests/scripts rather than putting in something in every single task test script and something I would like to avoid. This now means the -v option to pytest is superfluous. I did consider having a pytest.ini file but that doesn't allow me to pass any old environment variable in and this seems to be the compromise. --- docker/bash.sh | 1 + docker/build.sh | 1 + docker/with_the_same_user | 2 ++ tests/scripts/setup-pytest-env.sh | 21 +++++++++++++++++++++ tests/scripts/task_python_frontend.sh | 18 +++++++++--------- tests/scripts/task_python_integration.sh | 18 +++++++++--------- tests/scripts/task_python_nightly.sh | 2 +- tests/scripts/task_python_topi.sh | 2 +- tests/scripts/task_python_unittest.sh | 4 ++-- tests/scripts/task_python_vta_fsim.sh | 4 ++-- tests/scripts/task_python_vta_tsim.sh | 4 ++-- 11 files changed, 51 insertions(+), 26 deletions(-) create mode 100755 tests/scripts/setup-pytest-env.sh diff --git a/docker/bash.sh b/docker/bash.sh index 61823f9b6700..a6aab53fc1aa 100755 --- a/docker/bash.sh +++ b/docker/bash.sh @@ -89,6 +89,7 @@ ${DOCKER_BINARY} run --rm --pid=host\ -e "CI_BUILD_GROUP=$(id -g -n)" \ -e "CI_BUILD_GID=$(id -g)" \ -e "PYTHONPATH=python:topi/python"\ + -e "CI_PYTEST_ADD_OPTIONS=$CI_PYTEST_ADD_OPTIONS" \ ${CUDA_ENV}\ ${CI_DOCKER_EXTRA_PARAMS[@]} \ ${DOCKER_IMAGE_NAME}\ diff --git a/docker/build.sh b/docker/build.sh index defa28245544..d5925dcae288 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -162,6 +162,7 @@ ${DOCKER_BINARY} run --rm --pid=host \ -e "CI_BUILD_UID=$(id -u)" \ -e "CI_BUILD_GROUP=$(id -g -n)" \ -e "CI_BUILD_GID=$(id -g)" \ + -e "CI_PYTEST_ADD_OPTIONS=$CI_PYTEST_ADD_OPTIONS" \ ${CUDA_ENV}\ ${CI_DOCKER_EXTRA_PARAMS[@]} \ ${DOCKER_IMG_NAME} \ diff --git a/docker/with_the_same_user b/docker/with_the_same_user index 1288afd006c0..bb37148f9503 100644 --- a/docker/with_the_same_user +++ b/docker/with_the_same_user @@ -41,6 +41,8 @@ getent passwd "${CI_BUILD_UID}" || adduser --gid "${CI_BUILD_GID}" --uid "${CI_B --gecos "${CI_BUILD_USER} (generated by with_the_same_user script)" \ --disabled-password --home "${CI_BUILD_HOME}" --quiet "${CI_BUILD_USER}" usermod -a -G sudo "${CI_BUILD_USER}" +# This is a grotesque hack to get PYTEST_ADD_OPTS available to all task scripts. +source tests/scripts/setup-pytest-env.sh echo "${CI_BUILD_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-nopasswd-sudo if [[ ! -z $CUDA_VISIBLE_DEVICES ]]; then diff --git a/tests/scripts/setup-pytest-env.sh b/tests/scripts/setup-pytest-env.sh new file mode 100755 index 000000000000..73b9b3a7d5da --- /dev/null +++ b/tests/scripts/setup-pytest-env.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -u +set -e +export PYTEST_ADDOPTS="-v $CI_PYTEST_ADD_OPTIONS" diff --git a/tests/scripts/task_python_frontend.sh b/tests/scripts/task_python_frontend.sh index 862de5a81c73..3f5e350317fb 100755 --- a/tests/scripts/task_python_frontend.sh +++ b/tests/scripts/task_python_frontend.sh @@ -30,28 +30,28 @@ find . -type f -path "*.pyc" | xargs rm -f make cython3 echo "Running relay TFLite frontend test..." -python3 -m pytest -v tests/python/frontend/tflite +python3 -m pytest tests/python/frontend/tflite echo "Running relay MXNet frontend test..." -python3 -m pytest -v tests/python/frontend/mxnet +python3 -m pytest tests/python/frontend/mxnet echo "Running relay Keras frontend test..." -python3 -m pytest -v tests/python/frontend/keras +python3 -m pytest tests/python/frontend/keras echo "Running relay ONNX frontend test..." -python3 -m pytest -v tests/python/frontend/onnx +python3 -m pytest tests/python/frontend/onnx echo "Running relay CoreML frontend test..." -python3 -m pytest -v tests/python/frontend/coreml +python3 -m pytest tests/python/frontend/coreml echo "Running relay Tensorflow frontend test..." -python3 -m pytest -v tests/python/frontend/tensorflow +python3 -m pytest tests/python/frontend/tensorflow echo "Running relay caffe2 frontend test..." -python3 -m pytest -v tests/python/frontend/caffe2 +python3 -m pytest tests/python/frontend/caffe2 echo "Running relay DarkNet frontend test..." -python3 -m pytest -v tests/python/frontend/darknet +python3 -m pytest tests/python/frontend/darknet echo "Running relay PyTorch frontend test..." -python3 -m pytest -v tests/python/frontend/pytorch +python3 -m pytest tests/python/frontend/pytorch diff --git a/tests/scripts/task_python_integration.sh b/tests/scripts/task_python_integration.sh index cfd6bd042e82..2adbebf04041 100755 --- a/tests/scripts/task_python_integration.sh +++ b/tests/scripts/task_python_integration.sh @@ -42,26 +42,26 @@ rm -rf lib make cd ../.. -TVM_FFI=cython python3 -m pytest -v apps/extension/tests -TVM_FFI=ctypes python3 -m pytest -v apps/extension/tests +TVM_FFI=cython python3 -m pytest apps/extension/tests +TVM_FFI=ctypes python3 -m pytest apps/extension/tests # Test dso plugin cd apps/dso_plugin_module rm -rf lib make cd ../.. -TVM_FFI=cython python3 -m pytest -v apps/dso_plugin_module -TVM_FFI=ctypes python3 -m pytest -v apps/dso_plugin_module +TVM_FFI=cython python3 -m pytest apps/dso_plugin_module +TVM_FFI=ctypes python3 -m pytest apps/dso_plugin_module # Do not enable TensorFlow op # TVM_FFI=cython sh prepare_and_test_tfop_module.sh # TVM_FFI=ctypes sh prepare_and_test_tfop_module.sh -TVM_FFI=ctypes python3 -m pytest -v tests/python/integration -TVM_FFI=ctypes python3 -m pytest -v tests/python/contrib +TVM_FFI=ctypes python3 -m pytest tests/python/integration +TVM_FFI=ctypes python3 -m pytest tests/python/contrib -TVM_FFI=ctypes python3 -m pytest -v tests/python/relay +TVM_FFI=ctypes python3 -m pytest tests/python/relay # Do not enable OpenGL -# TVM_FFI=cython python -m pytest -v tests/webgl -# TVM_FFI=ctypes python3 -m pytest -v tests/webgl +# TVM_FFI=cython python -m pytest tests/webgl +# TVM_FFI=ctypes python3 -m pytest tests/webgl diff --git a/tests/scripts/task_python_nightly.sh b/tests/scripts/task_python_nightly.sh index 6ada8af31c5c..ee212a908c24 100755 --- a/tests/scripts/task_python_nightly.sh +++ b/tests/scripts/task_python_nightly.sh @@ -27,4 +27,4 @@ make cython3 # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f -python3 -m pytest -v topi/tests/python/nightly +python3 -m pytest topi/tests/python/nightly diff --git a/tests/scripts/task_python_topi.sh b/tests/scripts/task_python_topi.sh index 5e5fcb87c51b..7139affa3f9f 100755 --- a/tests/scripts/task_python_topi.sh +++ b/tests/scripts/task_python_topi.sh @@ -27,4 +27,4 @@ make cython3 # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f -python3 -m pytest -v topi/tests/python +python3 -m pytest topi/tests/python diff --git a/tests/scripts/task_python_unittest.sh b/tests/scripts/task_python_unittest.sh index a5ad5cae3287..1ed885099bac 100755 --- a/tests/scripts/task_python_unittest.sh +++ b/tests/scripts/task_python_unittest.sh @@ -24,6 +24,6 @@ export PYTHONPATH=python:topi/python # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f -TVM_FFI=ctypes python3 -m pytest -v tests/python/unittest +TVM_FFI=ctypes python3 -m pytest tests/python/unittest make cython3 -TVM_FFI=cython python3 -m pytest -v tests/python/unittest +TVM_FFI=cython python3 -m pytest tests/python/unittest diff --git a/tests/scripts/task_python_vta_fsim.sh b/tests/scripts/task_python_vta_fsim.sh index f269866c39e7..0a9df2a5f788 100755 --- a/tests/scripts/task_python_vta_fsim.sh +++ b/tests/scripts/task_python_vta_fsim.sh @@ -36,8 +36,8 @@ cp ${VTA_HW_PATH}/config/fsim_sample.json ${VTA_HW_PATH}/config/vta_config.json # Run unit tests in functional/fast simulator echo "Running unittest in fsim..." -python3 -m pytest -v ${TVM_PATH}/vta/tests/python/unittest +python3 -m pytest ${TVM_PATH}/vta/tests/python/unittest # Run unit tests in functional/fast simulator echo "Running integration test in fsim..." -python3 -m pytest -v ${TVM_PATH}/vta/tests/python/integration +python3 -m pytest ${TVM_PATH}/vta/tests/python/integration diff --git a/tests/scripts/task_python_vta_tsim.sh b/tests/scripts/task_python_vta_tsim.sh index 49366748b895..87624a4658f9 100755 --- a/tests/scripts/task_python_vta_tsim.sh +++ b/tests/scripts/task_python_vta_tsim.sh @@ -51,11 +51,11 @@ make -C ${VTA_HW_PATH}/hardware/chisel USE_THREADS=0 lib # Run unit tests in cycle accurate simulator echo "Running unittest in tsim..." -python3 -m pytest -v ${TVM_PATH}/vta/tests/python/unittest +python3 -m pytest ${TVM_PATH}/vta/tests/python/unittest # Run unit tests in cycle accurate simulator echo "Running integration test in tsim..." -python3 -m pytest -v ${TVM_PATH}/vta/tests/python/integration +python3 -m pytest ${TVM_PATH}/vta/tests/python/integration # Reset default fsim simulation cp ${VTA_HW_PATH}/config/fsim_sample.json ${VTA_HW_PATH}/config/vta_config.json From 859748af5848f6dd6539b3345552cf21ff927f89 Mon Sep 17 00:00:00 2001 From: Ramana Radhakrishnan Date: Thu, 23 Apr 2020 15:02:19 +0100 Subject: [PATCH 2/7] Improve other use case documentation --- docs/contribute/pull_request.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/contribute/pull_request.rst b/docs/contribute/pull_request.rst index 51626a16eb1a..8700103f3e4a 100644 --- a/docs/contribute/pull_request.rst +++ b/docs/contribute/pull_request.rst @@ -118,3 +118,6 @@ If you want to run a single test: rm -rf python/tvm/*.pyc python/tvm/*/*.pyc python/tvm/*/*/*.pyc TVM_FFI=ctypes python -m pytest -v tests/python/unittest/test_pass_storage_rewrite.py + + #Additionally if you want to run a single test, for example test_all_elemwise inside a file. + TVM_FFI=ctypes python -m pytest -v -k "test_all_elemwise" tests/python/frontend/tflite/test_forward.py From 9209df736b4ddb4c29ee0f3b923ac4e09cbe1f48 Mon Sep 17 00:00:00 2001 From: Ramana Radhakrishnan Date: Thu, 23 Apr 2020 16:35:51 +0100 Subject: [PATCH 3/7] Rationalize pytest environment. * Remove the setting from docker/with_same_user. * Take the opportunity to migrate common PYTHONPATH and TVM_PATH into the common environment setting. --- docker/with_the_same_user | 1 - tests/scripts/setup-pytest-env.sh | 2 ++ tests/scripts/task_python_docs.sh | 1 + tests/scripts/task_python_frontend.sh | 2 +- tests/scripts/task_python_integration.sh | 3 ++- tests/scripts/task_python_nightly.sh | 2 +- tests/scripts/task_python_topi.sh | 3 +-- tests/scripts/task_python_unittest.sh | 2 +- tests/scripts/task_python_vta_fsim.sh | 2 +- tests/scripts/task_python_vta_tsim.sh | 4 ++-- 10 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docker/with_the_same_user b/docker/with_the_same_user index bb37148f9503..2338f6351e82 100644 --- a/docker/with_the_same_user +++ b/docker/with_the_same_user @@ -42,7 +42,6 @@ getent passwd "${CI_BUILD_UID}" || adduser --gid "${CI_BUILD_GID}" --uid "${CI_B --disabled-password --home "${CI_BUILD_HOME}" --quiet "${CI_BUILD_USER}" usermod -a -G sudo "${CI_BUILD_USER}" # This is a grotesque hack to get PYTEST_ADD_OPTS available to all task scripts. -source tests/scripts/setup-pytest-env.sh echo "${CI_BUILD_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-nopasswd-sudo if [[ ! -z $CUDA_VISIBLE_DEVICES ]]; then diff --git a/tests/scripts/setup-pytest-env.sh b/tests/scripts/setup-pytest-env.sh index 73b9b3a7d5da..58921cb2c568 100755 --- a/tests/scripts/setup-pytest-env.sh +++ b/tests/scripts/setup-pytest-env.sh @@ -18,4 +18,6 @@ set -u set -e +export TVM_PATH=`pwd` +export PYTHONPATH=python:topi/python export PYTEST_ADDOPTS="-v $CI_PYTEST_ADD_OPTIONS" diff --git a/tests/scripts/task_python_docs.sh b/tests/scripts/task_python_docs.sh index d24ed1af63ea..819961dc6ebd 100755 --- a/tests/scripts/task_python_docs.sh +++ b/tests/scripts/task_python_docs.sh @@ -19,6 +19,7 @@ set -e set -u +source tests/scripts/setup-pytest-env.sh # cleanup old states rm -rf docs/_build mkdir -p docs/_build/html diff --git a/tests/scripts/task_python_frontend.sh b/tests/scripts/task_python_frontend.sh index 3f5e350317fb..46cb1bbb363d 100755 --- a/tests/scripts/task_python_frontend.sh +++ b/tests/scripts/task_python_frontend.sh @@ -19,7 +19,7 @@ set -e set -u -export PYTHONPATH=python:topi/python +source tests/scripts/setup-pytest-env.sh # to avoid openblas threading error export TVM_BIND_THREADS=0 export OMP_NUM_THREADS=1 diff --git a/tests/scripts/task_python_integration.sh b/tests/scripts/task_python_integration.sh index 2adbebf04041..296f9338bfb6 100755 --- a/tests/scripts/task_python_integration.sh +++ b/tests/scripts/task_python_integration.sh @@ -19,7 +19,8 @@ set -e set -u -export PYTHONPATH=`pwd`/python:`pwd`/topi/python:`pwd`/apps/extension/python +source tests/scripts/setup-pytest-env.sh +export PYTHONPATH=${PYTHONPATH}:${TVM_PATH}/apps/extension/python export LD_LIBRARY_PATH="build:${LD_LIBRARY_PATH:-}" export TVM_BIND_THREADS=0 export TVM_NUM_THREADS=2 diff --git a/tests/scripts/task_python_nightly.sh b/tests/scripts/task_python_nightly.sh index ee212a908c24..c2c0eab7d26a 100755 --- a/tests/scripts/task_python_nightly.sh +++ b/tests/scripts/task_python_nightly.sh @@ -19,7 +19,7 @@ set -e set -u -export PYTHONPATH=python:topi/python +source tests/scripts/setup-pytest-env.sh # Rebuild cython make cython3 diff --git a/tests/scripts/task_python_topi.sh b/tests/scripts/task_python_topi.sh index 7139affa3f9f..66585d56d1db 100755 --- a/tests/scripts/task_python_topi.sh +++ b/tests/scripts/task_python_topi.sh @@ -19,8 +19,7 @@ set -e set -u -export PYTHONPATH=python:topi/python - +source tests/scripts/setup-pytest-env.sh # Rebuild cython make cython3 diff --git a/tests/scripts/task_python_unittest.sh b/tests/scripts/task_python_unittest.sh index 1ed885099bac..622646b76189 100755 --- a/tests/scripts/task_python_unittest.sh +++ b/tests/scripts/task_python_unittest.sh @@ -19,7 +19,7 @@ set -e set -u -export PYTHONPATH=python:topi/python +source tests/scripts/setup-pytest-env.sh # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f diff --git a/tests/scripts/task_python_vta_fsim.sh b/tests/scripts/task_python_vta_fsim.sh index 0a9df2a5f788..f4fa307dafa4 100755 --- a/tests/scripts/task_python_vta_fsim.sh +++ b/tests/scripts/task_python_vta_fsim.sh @@ -20,7 +20,7 @@ set -e set -u export TVM_PATH=`pwd` -export PYTHONPATH=${TVM_PATH}/python:${TVM_PATH}/vta/python:${TVM_PATH}/topi/python +export PYTHONPATH=${TVM_PATH}/python:${TVM_PATH}/vta/python export VTA_HW_PATH=`pwd`/3rdparty/vta-hw # cleanup pycache diff --git a/tests/scripts/task_python_vta_tsim.sh b/tests/scripts/task_python_vta_tsim.sh index 87624a4658f9..8e9a5eafbd9a 100755 --- a/tests/scripts/task_python_vta_tsim.sh +++ b/tests/scripts/task_python_vta_tsim.sh @@ -19,8 +19,8 @@ set -e set -u -export TVM_PATH=`pwd` -export PYTHONPATH=${TVM_PATH}/python:${TVM_PATH}/vta/python:${TVM_PATH}/topi/python +source tests/scripts/setup-pytest-env.sh +export PYTHONPATH=${PYTHONPATH}:${TVM_PATH}/vta/python export VTA_HW_PATH=`pwd`/3rdparty/vta-hw # cleanup pycache From 6067aadd1d422ac37d8c32b755e5b362beb98778 Mon Sep 17 00:00:00 2001 From: Ramana Radhakrishnan Date: Thu, 23 Apr 2020 17:11:55 +0100 Subject: [PATCH 4/7] Fixup vta fsim --- tests/scripts/task_python_vta_fsim.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/task_python_vta_fsim.sh b/tests/scripts/task_python_vta_fsim.sh index f4fa307dafa4..2d368cea769e 100755 --- a/tests/scripts/task_python_vta_fsim.sh +++ b/tests/scripts/task_python_vta_fsim.sh @@ -19,7 +19,7 @@ set -e set -u -export TVM_PATH=`pwd` +source tests/scripts/setup-pytest-env.sh export PYTHONPATH=${TVM_PATH}/python:${TVM_PATH}/vta/python export VTA_HW_PATH=`pwd`/3rdparty/vta-hw From 91f39d6d45843739f599bb33be6e171f5b027160 Mon Sep 17 00:00:00 2001 From: Ramana Radhakrishnan Date: Thu, 23 Apr 2020 17:41:50 +0100 Subject: [PATCH 5/7] Be more explicit with common PYTHONPATH --- tests/scripts/setup-pytest-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/setup-pytest-env.sh b/tests/scripts/setup-pytest-env.sh index 58921cb2c568..36d095692ec8 100755 --- a/tests/scripts/setup-pytest-env.sh +++ b/tests/scripts/setup-pytest-env.sh @@ -19,5 +19,5 @@ set -u set -e export TVM_PATH=`pwd` -export PYTHONPATH=python:topi/python +export PYTHONPATH=${TVM_PATH}/python:${TVM_PATH}/topi/python export PYTEST_ADDOPTS="-v $CI_PYTEST_ADD_OPTIONS" From f37c650b5b49839d0ebac19e81db5b3fc611fa6f Mon Sep 17 00:00:00 2001 From: Ramana Radhakrishnan Date: Thu, 23 Apr 2020 18:16:48 +0100 Subject: [PATCH 6/7] Fix python path for task_python_vta_fsim.sh properly --- tests/scripts/task_python_vta_fsim.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/task_python_vta_fsim.sh b/tests/scripts/task_python_vta_fsim.sh index 2d368cea769e..f48b7d6d1594 100755 --- a/tests/scripts/task_python_vta_fsim.sh +++ b/tests/scripts/task_python_vta_fsim.sh @@ -20,7 +20,7 @@ set -e set -u source tests/scripts/setup-pytest-env.sh -export PYTHONPATH=${TVM_PATH}/python:${TVM_PATH}/vta/python +export PYTHONPATH=${PYTHONPATH}:${TVM_PATH}/vta/python export VTA_HW_PATH=`pwd`/3rdparty/vta-hw # cleanup pycache From 99e5d4e4c9f3aba101c77b7c6e3e4382810e940d Mon Sep 17 00:00:00 2001 From: Ramana Radhakrishnan Date: Thu, 23 Apr 2020 19:40:55 +0100 Subject: [PATCH 7/7] Fix nit in documentation. --- docs/contribute/pull_request.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contribute/pull_request.rst b/docs/contribute/pull_request.rst index 8700103f3e4a..25dc0daca362 100644 --- a/docs/contribute/pull_request.rst +++ b/docs/contribute/pull_request.rst @@ -119,5 +119,5 @@ If you want to run a single test: TVM_FFI=ctypes python -m pytest -v tests/python/unittest/test_pass_storage_rewrite.py - #Additionally if you want to run a single test, for example test_all_elemwise inside a file. + # Additionally if you want to run a single test, for example test_all_elemwise inside a file. TVM_FFI=ctypes python -m pytest -v -k "test_all_elemwise" tests/python/frontend/tflite/test_forward.py