diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 61b8814857..f7024f1a08 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -11,7 +11,7 @@ A few sentences describing the changes proposed in this pull request. - [x] Non-breaking change (fix or new feature that would not break existing functionality). - [ ] Breaking change (fix or new feature that would cause existing functionality to change). - [ ] New tests added to cover the changes. -- [ ] Integration tests passed locally by running `./runtests.sh --codeformat --coverage`. -- [ ] Quick tests passed locally by running `./runtests.sh --quick`. +- [ ] Integration tests passed locally by running `./runtests.sh -f -u --net --coverage`. +- [ ] Quick tests passed locally by running `./runtests.sh --quick --unittests`. - [ ] In-line docstrings updated. - [ ] Documentation updated, tested `make html` command in the `docs/` folder. diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 681331aae1..90abb5d7e4 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -40,9 +40,16 @@ jobs: nvidia-smi export CUDA_VISIBLE_DEVICES=$(python -m tests.utils) echo $CUDA_VISIBLE_DEVICES + stop_time=$((LAUNCH_DELAY + $(date +%s))) + while [ $(date +%s) -lt $stop_time ]; do + python -c 'import torch; torch.rand(5, 3, device=torch.device("cuda:0"))'; + done python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))" - python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))' - BUILD_MONAI=1 ./runtests.sh --coverage + python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))' + BUILD_MONAI=1 ./runtests.sh --coverage --unittests # unit tests with coverage report + export CUDA_VISIBLE_DEVICES=$(python -m tests.utils) + echo $CUDA_VISIBLE_DEVICES + BUILD_MONAI=1 ./runtests.sh --coverage --net # integration tests with coverage report coverage xml - name: Upload coverage uses: codecov/codecov-action@v1 @@ -76,9 +83,16 @@ jobs: nvidia-smi export CUDA_VISIBLE_DEVICES=$(python -m tests.utils) echo $CUDA_VISIBLE_DEVICES + stop_time=$((LAUNCH_DELAY + $(date +%s))) + while [ $(date +%s) -lt $stop_time ]; do + python -c 'import torch; torch.rand(5, 3, device=torch.device("cuda:0"))'; + done python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))" - python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))' - BUILD_MONAI=1 ./runtests.sh --coverage + python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))' + BUILD_MONAI=1 ./runtests.sh --coverage --unittests # unit tests with coverage report + export CUDA_VISIBLE_DEVICES=$(python -m tests.utils) + echo $CUDA_VISIBLE_DEVICES + BUILD_MONAI=1 ./runtests.sh --coverage --net # integration tests with coverage report coverage xml - name: Upload coverage uses: codecov/codecov-action@v1 @@ -104,7 +118,10 @@ jobs: python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))" python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))' ngc --version - BUILD_MONAI=1 ./runtests.sh --coverage --pytype + BUILD_MONAI=1 ./runtests.sh --coverage --pytype --unittests # unit tests with pytype checks, coverage report + export CUDA_VISIBLE_DEVICES=$(python -m tests.utils) + echo $CUDA_VISIBLE_DEVICES + BUILD_MONAI=1 ./runtests.sh --coverage --net # integration tests with coverage report coverage xml - name: Upload coverage uses: codecov/codecov-action@v1 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 66f6c2956d..ac3efbb751 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -44,7 +44,7 @@ jobs: echo $CUDA_VISIBLE_DEVICES python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))" python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))' - BUILD_MONAI=1 ./runtests.sh --net + BUILD_MONAI=1 ./runtests.sh --unittests --net - name: Add reaction uses: peter-evans/create-or-update-comment@v1 with: diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 227d55a082..8ed1f6d21e 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -39,7 +39,7 @@ jobs: # clean up temporary files $(pwd)/runtests.sh --clean # Git hub actions have 2 cores, so parallize pytype - $(pwd)/runtests.sh --nounittests --codeformat -j 2 + $(pwd)/runtests.sh --codeformat -j 2 quick-py3: # full dependencies installed runs-on: ${{ matrix.os }} @@ -241,7 +241,7 @@ jobs: python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))" python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))' python -c "import monai; monai.config.print_config()" - BUILD_MONAI=1 ./runtests.sh --quick + BUILD_MONAI=1 ./runtests.sh --quick --unittests if [ ${{ matrix.environment }} == "PT18+CUDA112" ]; then # test the clang-format tool downloading once coverage run -m tests.clang_format_utils diff --git a/.github/workflows/setupapp.yml b/.github/workflows/setupapp.yml index e40660c213..e5cb9a7cf1 100644 --- a/.github/workflows/setupapp.yml +++ b/.github/workflows/setupapp.yml @@ -41,12 +41,22 @@ jobs: - name: Run unit tests report coverage run: | python -m pip list + export LAUNCH_DELAY=$[ $RANDOM % 16 * 60 ] + echo "Sleep $LAUNCH_DELAY" + sleep $LAUNCH_DELAY nvidia-smi export CUDA_VISIBLE_DEVICES=$(python -m tests.utils) echo $CUDA_VISIBLE_DEVICES + stop_time=$((LAUNCH_DELAY + $(date +%s))) + while [ $(date +%s) -lt $stop_time ]; do + python -c 'import torch; torch.rand(5, 3, device=torch.device("cuda:0"))'; + done python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))" - python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))' - BUILD_MONAI=1 ./runtests.sh --coverage + python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))' + BUILD_MONAI=1 ./runtests.sh --coverage --unittests # unit tests with coverage report + export CUDA_VISIBLE_DEVICES=$(python -m tests.utils) + echo $CUDA_VISIBLE_DEVICES + BUILD_MONAI=1 ./runtests.sh --coverage --net # integration tests with coverage report coverage xml - name: Upload coverage uses: codecov/codecov-action@v1 @@ -88,7 +98,7 @@ jobs: run: | python -m pip list python -c 'import torch; print(torch.__version__); print(torch.rand(5,3))' - BUILD_MONAI=1 ./runtests.sh --quick + BUILD_MONAI=1 ./runtests.sh --quick --unittests coverage xml - name: Upload coverage uses: codecov/codecov-action@v1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 01a4773b5a..325f81b127 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,10 +52,10 @@ Before submitting a pull request, we recommend that all linting should pass, by ```bash pip install -U -r requirements-dev.txt # install the latest tools -./runtests.sh --codeformat --nounittests # runs the linting tools only +./runtests.sh --codeformat # runs the linting tools # try to fix the coding style errors automatically -./runtests.sh --autofix --nounittests +./runtests.sh --autofix ``` License information: all source code files should start with this paragraph: @@ -98,12 +98,12 @@ Before submitting a pull request, we recommend that all linting and unit tests should pass, by running the following command locally: ```bash -./runtests.sh --codeformat --coverage +./runtests.sh --codeformat --coverage --unittests ``` or (for new features that would not break existing functionality): ```bash -./runtests.sh --quick +./runtests.sh --quick --unittests ``` It is recommended that the new test `test_[module_name].py` is constructed by using only diff --git a/runtests.sh b/runtests.sh index 1395ccdcfd..0d3551291a 100755 --- a/runtests.sh +++ b/runtests.sh @@ -36,9 +36,7 @@ doQuickTests=false doNetTests=false doDryRun=false doZooTests=false - -doUnitTests=true - +doUnitTests=false doBlackFormat=false doBlackFix=false doIsortFormat=false @@ -55,16 +53,17 @@ PY_EXE=${MONAI_PY_EXE:-$(which python)} function print_usage { echo "runtests.sh [--codeformat] [--autofix] [--black] [--isort] [--flake8] [--clangformat] [--pytype] [--mypy]" - echo " [--nounittests] [--coverage] [--quick] [--net] [--dryrun] [-j number] [--clean] [--help] [--version]" + echo " [--unittests] [--coverage] [--quick] [--net] [--dryrun] [-j number] [--clean] [--help] [--version]" echo "" echo "MONAI unit testing utilities." echo "" echo "Examples:" - echo "./runtests.sh --codeformat --coverage # run full tests (${green}recommended before making pull requests${noColor})." - echo "./runtests.sh --codeformat --nounittests # run coding style and static type checking." - echo "./runtests.sh --quick # run minimal unit tests, for quick verification during code developments." - echo "./runtests.sh --autofix --nounittests # run automatic code formatting using \"isort\" and \"black\"." - echo "./runtests.sh --clean # clean up temporary files and run \"${PY_EXE} setup.py develop --uninstall\"." + echo "./runtests.sh -f -u --net --coverage # run style checks, full tests, print code coverage (${green}recommended for pull requests${noColor})." + echo "./runtests.sh -f -u # run style checks and unit tests." + echo "./runtests.sh -f # run coding style and static type checking." + echo "./runtests.sh --quick --unittests # run minimal unit tests, for quick verification during code developments." + echo "./runtests.sh --autofix # run automatic code formatting using \"isort\" and \"black\"." + echo "./runtests.sh --clean # clean up temporary files and run \"${PY_EXE} setup.py develop --uninstall\"." echo "" echo "Code style check options:" echo " --black : perform \"black\" code format checks" @@ -79,11 +78,11 @@ function print_usage { echo " -j, --jobs : number of parallel jobs to run \"pytype\" (default $NUM_PARALLEL)" echo "" echo "MONAI unit testing options:" - echo " --nounittests : skip doing unit testing (i.e. only format lint testers)" - echo " --coverage : peforms coverage analysis of code for tests run" - echo " -q, --quick : disable long running tests" - echo " --net : perform training/inference/eval integration testing" - echo " --list_tests : list tests and exit" + echo " -u, --unittests : perform unit testing" + echo " --coverage : report testing code coverage, to be used with \"--net\", \"--unittests\"" + echo " -q, --quick : skip long running unit tests and integration tests" + echo " --net : perform integration testing" + echo " --list_tests : list unit tests and exit" echo "" echo "Misc. options:" echo " --dryrun : display the commands to the screen without running" @@ -92,7 +91,7 @@ function print_usage { echo " -h, --help : show this help message and exit" echo " -v, --version : show MONAI and system version information and exit" echo "" - echo "${separator}For bug reports, questions, and discussions, please file an issue at:" + echo "${separator}For bug reports and feature requests, please file an issue at:" echo " https://github.com/Project-MONAI/MONAI/issues/new/choose" echo "" echo "To choose an alternative python executable, set the environmental variable, \"MONAI_PY_EXE\"." @@ -220,8 +219,8 @@ do --dryrun) doDryRun=true ;; - --nou*) # allow --nounittest | --nounittests | --nounittesting etc. - doUnitTests=false + -u|--u*) # allow --unittest | --unittests | --unittesting etc. + doUnitTests=true ;; -f|--codeformat) doBlackFormat=true @@ -268,6 +267,10 @@ do print_version exit 1 ;; + --nou*) # allow --nounittest | --nounittests | --nounittesting etc. + print_error_msg "nounittest option is deprecated, no unit tests is the default setting" + print_usage + ;; *) print_error_msg "Incorrect commandline provided, invalid key: $key" print_usage @@ -511,7 +514,7 @@ if [ $doUnitTests = true ] then echo "${separator}${blue}unittests${noColor}" torch_validate - ${cmdPrefix}${cmd} ./tests/runner.py + ${cmdPrefix}${cmd} ./tests/runner.py -p "test_[!integration]*py" fi # network training/inference/eval integration tests diff --git a/tests/runner.py b/tests/runner.py index b5d1de5fc1..f7be96cfb3 100644 --- a/tests/runner.py +++ b/tests/runner.py @@ -126,6 +126,7 @@ def get_default_pattern(loader): tests = loader.discover(args.path, args.pattern) discovery_time = pc.total_time print(f"time to discover tests: {discovery_time}s") + print(tests) test_runner = unittest.runner.TextTestRunner( resultclass=TimeLoggingTestResult, verbosity=args.verbosity, failfast=args.failfast