From a390d46edc97fea8fa90ac43224d991f639a4eb7 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 04:52:40 -0400 Subject: [PATCH 01/20] fixes https://github.com/Project-MONAI/research-contributions/issues/235 Signed-off-by: Wenqi Li --- monai/networks/layers/factories.py | 7 +++---- monai/networks/utils.py | 28 ++++++++++++++++++++++++++-- tests/test_dynunet.py | 9 --------- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/monai/networks/layers/factories.py b/monai/networks/layers/factories.py index 8c3d9766a4..bb56b0c0c5 100644 --- a/monai/networks/layers/factories.py +++ b/monai/networks/layers/factories.py @@ -67,10 +67,9 @@ def use_factory(fact_args): import torch.nn as nn +from monai.networks.utils import has_nvfuser_instance_norm from monai.utils import look_up_option, optional_import -InstanceNorm3dNVFuser, has_nvfuser = optional_import("apex.normalization", name="InstanceNorm3dNVFuser") - __all__ = ["LayerFactory", "Dropout", "Norm", "Act", "Conv", "Pool", "Pad", "split_args"] @@ -267,12 +266,12 @@ def instance_nvfuser_factory(dim): warnings.warn(f"`InstanceNorm3dNVFuser` only supports 3d cases, use {types[dim - 1]} instead.") return types[dim - 1] - if not has_nvfuser: + if not has_nvfuser_instance_norm(): warnings.warn( "`apex.normalization.InstanceNorm3dNVFuser` is not installed properly, use nn.InstanceNorm3d instead." ) return nn.InstanceNorm3d - return InstanceNorm3dNVFuser + return optional_import("apex.normalization", name="InstanceNorm3dNVFuser")[0] Act.add_factory_callable("elu", lambda: nn.modules.ELU) diff --git a/monai/networks/utils.py b/monai/networks/utils.py index 57b59b3f64..01126419db 100644 --- a/monai/networks/utils.py +++ b/monai/networks/utils.py @@ -27,10 +27,10 @@ import torch import torch.nn as nn -from monai.apps.utils import get_logger, optional_import +from monai.apps.utils import get_logger from monai.config import PathLike from monai.utils.misc import ensure_tuple, save_obj, set_determinism -from monai.utils.module import look_up_option, pytorch_after +from monai.utils.module import look_up_option, optional_import, pytorch_after from monai.utils.type_conversion import convert_to_dst_type, convert_to_tensor onnx, _ = optional_import("onnx") @@ -59,10 +59,34 @@ "replace_modules_temp", "look_up_named_module", "set_named_module", + "has_nvfuser_instance_norm", ] logger = get_logger(module_name=__name__) +global _has_nvfuser +_has_nvfuser = None + + +def has_nvfuser_instance_norm(): + """whether the current environment has InstanceNorm3dNVFuser + https://github.com/NVIDIA/apex/blob/23.05-devel/apex/normalization/instance_norm.py#L15-L16 + """ + global _has_nvfuser + if _has_nvfuser is not None: + return _has_nvfuser + + _, _has_nvfuser = optional_import("apex.normalization", name="InstanceNorm3dNVFuser") + if not _has_nvfuser: + return False + try: + import importlib + + importlib.import_module("instance_norm_nvfuser_cuda") + except ImportError: + _has_nvfuser = False + return _has_nvfuser + def look_up_named_module(name: str, mod, print_all_options=False): """ diff --git a/tests/test_dynunet.py b/tests/test_dynunet.py index 247da14b7d..56bfa0ee3d 100644 --- a/tests/test_dynunet.py +++ b/tests/test_dynunet.py @@ -128,15 +128,6 @@ def test_script(self): @skip_if_no_cuda @skip_if_windows class TestDynUNetWithInstanceNorm3dNVFuser(unittest.TestCase): - def setUp(self): - try: - layer = InstanceNorm3dNVFuser(num_features=1, affine=False).to("cuda:0") - inp = torch.randn([1, 1, 1, 1, 1]).to("cuda:0") - out = layer(inp) - del inp, out, layer - except Exception: - self.skipTest("NVFuser not available") - @parameterized.expand([TEST_CASE_DYNUNET_3D[0]]) def test_consistency(self, input_param, input_shape, _): for eps in [1e-4, 1e-5]: From 783ebf7302ed1029e94e57a165104be5bd240c8b Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 11:22:16 +0100 Subject: [PATCH 02/20] update based on comments Signed-off-by: Wenqi Li --- tests/test_dynunet.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_dynunet.py b/tests/test_dynunet.py index 56bfa0ee3d..c37158ded4 100644 --- a/tests/test_dynunet.py +++ b/tests/test_dynunet.py @@ -19,11 +19,8 @@ from monai.networks import eval_mode from monai.networks.nets import DynUNet -from monai.utils import optional_import from tests.utils import assert_allclose, skip_if_no_cuda, skip_if_windows, test_script_save -InstanceNorm3dNVFuser, _ = optional_import("apex.normalization", name="InstanceNorm3dNVFuser") - device = "cuda" if torch.cuda.is_available() else "cpu" strides: Sequence[Sequence[int] | int] From 93052057e3d976a61a5616ca5311a1b26e4a1e71 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 11:35:26 +0100 Subject: [PATCH 03/20] remove unused variables Signed-off-by: Wenqi Li --- monai/apps/detection/transforms/box_ops.py | 4 +--- monai/apps/pathology/transforms/post/dictionary.py | 1 - monai/inferers/utils.py | 4 ++-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/monai/apps/detection/transforms/box_ops.py b/monai/apps/detection/transforms/box_ops.py index 15de8a4342..d63a155e42 100644 --- a/monai/apps/detection/transforms/box_ops.py +++ b/monai/apps/detection/transforms/box_ops.py @@ -21,12 +21,10 @@ from monai.data.box_utils import COMPUTE_DTYPE, TO_REMOVE, get_spatial_dims from monai.transforms import Resize from monai.transforms.utils import create_scale -from monai.utils import look_up_option, optional_import +from monai.utils import look_up_option from monai.utils.misc import ensure_tuple, ensure_tuple_rep from monai.utils.type_conversion import convert_data_type, convert_to_dst_type -scipy, _ = optional_import("scipy") - def _apply_affine_to_points(points: torch.Tensor, affine: torch.Tensor, include_shift: bool = True) -> torch.Tensor: """ diff --git a/monai/apps/pathology/transforms/post/dictionary.py b/monai/apps/pathology/transforms/post/dictionary.py index a95bdfd48f..1e2540daee 100644 --- a/monai/apps/pathology/transforms/post/dictionary.py +++ b/monai/apps/pathology/transforms/post/dictionary.py @@ -35,7 +35,6 @@ from monai.utils.enums import HoVerNetBranch find_contours, _ = optional_import("skimage.measure", name="find_contours") -moments, _ = optional_import("skimage.measure", name="moments") __all__ = [ "WatershedD", diff --git a/monai/inferers/utils.py b/monai/inferers/utils.py index b79d119414..92f267e8a2 100644 --- a/monai/inferers/utils.py +++ b/monai/inferers/utils.py @@ -280,7 +280,7 @@ def sliding_window_inference( else: sw_device_buffer[ss] *= w_t sw_device_buffer[ss] = sw_device_buffer[ss].to(device) - _compute_coords(sw_batch_size, unravel_slice, z_scale, output_image_list[ss], sw_device_buffer[ss]) + _compute_coords(unravel_slice, z_scale, output_image_list[ss], sw_device_buffer[ss]) sw_device_buffer = [] if buffered: b_s += 1 @@ -342,7 +342,7 @@ def _create_buffered_slices(slices, batch_size, sw_batch_size, buffer_dim, buffe return slices, n_per_batch, b_slices, windows_range -def _compute_coords(sw, coords, z_scale, out, patch): +def _compute_coords(coords, z_scale, out, patch): """sliding window batch spatial scaling indexing for multi-resolution outputs.""" for original_idx, p in zip(coords, patch): idx_zm = list(original_idx) # 4D for 2D image, 5D for 3D image From 12cb977382d9cf906865adbed27bcaeece4d38b7 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 12:40:34 +0100 Subject: [PATCH 04/20] remove duplidated tests (in test_lmdbdataset_dist) Signed-off-by: Wenqi Li --- tests/test_lmdbdataset.py | 42 +-------------------------------------- 1 file changed, 1 insertion(+), 41 deletions(-) diff --git a/tests/test_lmdbdataset.py b/tests/test_lmdbdataset.py index 080868b0dd..155b4eb0fc 100644 --- a/tests/test_lmdbdataset.py +++ b/tests/test_lmdbdataset.py @@ -12,7 +12,6 @@ from __future__ import annotations import os -import shutil import tempfile import unittest @@ -22,7 +21,7 @@ from monai.data import LMDBDataset, json_hashing from monai.transforms import Compose, LoadImaged, SimulateDelayd, Transform -from tests.utils import DistCall, DistTestCase, skip_if_windows +from tests.utils import skip_if_windows TEST_CASE_1 = [ Compose( @@ -204,44 +203,5 @@ def test_shape(self, transform, expected_shape, kwargs=None): dataset_postcached.set_data(data=test_data_new) # filename list updated, files do not exist -@skip_if_windows -class TestMPLMDBDataset(DistTestCase): - def setUp(self): - self.tempdir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.tempdir) - - @DistCall(nnodes=1, nproc_per_node=1) - def test_mp_cache(self): - items = [[list(range(i))] for i in range(5)] - - ds = LMDBDataset(items, transform=_InplaceXform(), cache_dir=self.tempdir, lmdb_kwargs={"map_size": 10 * 1024}) - self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]]) - ds1 = LMDBDataset(items, transform=_InplaceXform(), cache_dir=self.tempdir, lmdb_kwargs={"map_size": 10 * 1024}) - self.assertEqual(list(ds1), list(ds)) - self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]]) - - ds = LMDBDataset( - items, - transform=_InplaceXform(), - cache_dir=self.tempdir, - lmdb_kwargs={"map_size": 10 * 1024}, - hash_func=json_hashing, - ) - self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]]) - ds1 = LMDBDataset( - items, - transform=_InplaceXform(), - cache_dir=self.tempdir, - lmdb_kwargs={"map_size": 10 * 1024}, - hash_func=json_hashing, - ) - self.assertEqual(list(ds1), list(ds)) - self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]]) - - self.assertTrue(isinstance(ds1.info(), dict)) - - if __name__ == "__main__": unittest.main() From 4d75aabb7b62d3136b7442628b5312528c5b26c7 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 13:01:22 +0100 Subject: [PATCH 05/20] update tests Signed-off-by: Wenqi Li --- .github/workflows/integration.yml | 82 ++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 13 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 5b5e32c159..7a2b2276b2 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -6,10 +6,10 @@ on: type: [integration-test-command] jobs: - integration-py3: + integration-auto3dseg: container: image: nvcr.io/nvidia/pytorch:22.04-py3 # CUDA 11.6 py38 - options: --gpus "device=0" --ipc host # shm-size 4g works fine + options: --gpus "device=1" --ipc host # shm-size 4g works fine runs-on: [self-hosted, linux, x64, command] steps: # checkout the pull request branch @@ -45,7 +45,7 @@ jobs: run: | python -m pip list git config --global --add safe.directory /__w/MONAI/MONAI - git clean -ffdx + git clean -ffdx && git reset --hard HEAD nvidia-smi export CUDA_VISIBLE_DEVICES=$(python -m tests.utils -c 1 | tail -n 1) echo $CUDA_VISIBLE_DEVICES @@ -63,6 +63,70 @@ jobs: python -m tests.test_integration_autorunner python -m tests.test_integration_gpu_customization + - name: Integration tests + shell: bash + env: + BUILD_MONAI: 1 + run: | + pwd + ls -ll + ./runtests.sh --build --net + + - name: Add reaction + uses: peter-evans/create-or-update-comment@v2 + if: github.event.pull_request.number != '' + with: + token: ${{ secrets.PR_MAINTAIN }} + repository: ${{ github.event.client_payload.github.payload.repository.full_name }} + comment-id: ${{ github.event.client_payload.github.payload.comment.id }} + reaction-type: rocket + + integration-full-unit: + container: + image: nvcr.io/nvidia/pytorch:22.04-py3 # CUDA 11.6 py38 + options: --gpus "device=2" --ipc host # shm-size 4g works fine + runs-on: [self-hosted, linux, x64, command] + steps: + # checkout the pull request branch + - uses: actions/checkout@v3 + with: + token: ${{ secrets.PR_MAINTAIN }} + repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} + ref: ${{ github.event.client_payload.pull_request.head.ref }} + - name: cache weekly timestamp + id: pip-cache + run: | + echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT + - name: cache for pip + uses: actions/cache@v3 + id: cache + with: + path: | + ~/.cache/pip + ~/.cache/torch + key: docker-py3-pip-${{ steps.pip-cache.outputs.datew }} + - name: Install the dependencies + run: | + which python + python -m pip install --upgrade pip wheel + pip uninstall -y monai + pip uninstall -y monai + pip uninstall -y monai-weekly + pip uninstall -y monai-weekly + python -m pip install --upgrade torch torchvision torchaudio + python -m pip install -r requirements-dev.txt + rm -rf /github/home/.cache/torch/hub/mmars/ + - name: Clean directory + run: | + python -m pip list + git config --global --add safe.directory /__w/MONAI/MONAI + git clean -ffdx && git reset --hard HEAD + nvidia-smi + export CUDA_VISIBLE_DEVICES=$(python -m tests.utils -c 1 | tail -n 1) + echo $CUDA_VISIBLE_DEVICES + python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))" + python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))' + - name: Auto3dseg latest algo shell: bash env: @@ -70,6 +134,7 @@ jobs: run: | # test latest template echo "test latest algo" + pwd cd ../ rm -rf research-contributions rm -rf algorithm_templates @@ -87,15 +152,6 @@ jobs: python -m tests.test_integration_autorunner python -m tests.test_integration_gpu_customization - - name: Integration tests - shell: bash - env: - BUILD_MONAI: 1 - run: | - pwd - ls -ll - ./runtests.sh --build --net - - name: Unit tests shell: bash env: @@ -112,4 +168,4 @@ jobs: token: ${{ secrets.PR_MAINTAIN }} repository: ${{ github.event.client_payload.github.payload.repository.full_name }} comment-id: ${{ github.event.client_payload.github.payload.comment.id }} - reaction-type: rocket + reaction-type: +1 From a07d6470877fe35dd41e37ff9ab4977023076b9e Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 13:01:40 +0100 Subject: [PATCH 06/20] temp test Signed-off-by: Wenqi Li --- .github/workflows/integration.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7a2b2276b2..b0e84a9957 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -2,6 +2,9 @@ name: integration on: + push: + branches: + - temp-test repository_dispatch: type: [integration-test-command] From a784655ca62bed33e6b45233766cf00710337f9a Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 13:03:11 +0100 Subject: [PATCH 07/20] update tag Signed-off-by: Wenqi Li --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b0e84a9957..208b20c0a8 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -88,7 +88,7 @@ jobs: container: image: nvcr.io/nvidia/pytorch:22.04-py3 # CUDA 11.6 py38 options: --gpus "device=2" --ipc host # shm-size 4g works fine - runs-on: [self-hosted, linux, x64, command] + runs-on: [self-hosted, linux, x64, command1] steps: # checkout the pull request branch - uses: actions/checkout@v3 From 2e4f88c828809727b95e4ad94255510392d484ec Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 13:27:33 +0100 Subject: [PATCH 08/20] temp tests Signed-off-by: Wenqi Li --- .github/workflows/integration.yml | 52 +++++++++++++------------------ 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 208b20c0a8..01e3417f05 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,7 +9,7 @@ on: type: [integration-test-command] jobs: - integration-auto3dseg: + setup-folders: container: image: nvcr.io/nvidia/pytorch:22.04-py3 # CUDA 11.6 py38 options: --gpus "device=1" --ipc host # shm-size 4g works fine @@ -54,13 +54,32 @@ jobs: echo $CUDA_VISIBLE_DEVICES python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))" python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))' + BUILD_MONAI=0 ./runtests.sh --build + + integration-auto3dseg: + container: + image: nvcr.io/nvidia/pytorch:22.04-py3 # CUDA 11.6 py38 + options: --gpus "device=1" --ipc host # shm-size 4g works fine + runs-on: [self-hosted, linux, x64, command] + steps: + # checkout the pull request branch + - uses: actions/checkout@v3 + with: + token: ${{ secrets.PR_MAINTAIN }} + repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} + ref: ${{ github.event.client_payload.pull_request.head.ref }} + - name: Install the dependencies + run: | + which python + python -m pip install --upgrade pip wheel + python -m pip install --upgrade torch torchvision torchaudio + python -m pip install -r requirements-dev.txt - name: Auto3dseg tag algo shell: bash env: BUILD_MONAI: 0 run: | - BUILD_MONAI=0 ./runtests.sh --build python -m tests.test_auto3dseg_ensemble python -m tests.test_auto3dseg_hpo python -m tests.test_integration_autorunner @@ -96,39 +115,12 @@ jobs: token: ${{ secrets.PR_MAINTAIN }} repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} ref: ${{ github.event.client_payload.pull_request.head.ref }} - - name: cache weekly timestamp - id: pip-cache - run: | - echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT - - name: cache for pip - uses: actions/cache@v3 - id: cache - with: - path: | - ~/.cache/pip - ~/.cache/torch - key: docker-py3-pip-${{ steps.pip-cache.outputs.datew }} - name: Install the dependencies run: | which python python -m pip install --upgrade pip wheel - pip uninstall -y monai - pip uninstall -y monai - pip uninstall -y monai-weekly - pip uninstall -y monai-weekly python -m pip install --upgrade torch torchvision torchaudio python -m pip install -r requirements-dev.txt - rm -rf /github/home/.cache/torch/hub/mmars/ - - name: Clean directory - run: | - python -m pip list - git config --global --add safe.directory /__w/MONAI/MONAI - git clean -ffdx && git reset --hard HEAD - nvidia-smi - export CUDA_VISIBLE_DEVICES=$(python -m tests.utils -c 1 | tail -n 1) - echo $CUDA_VISIBLE_DEVICES - python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))" - python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))' - name: Auto3dseg latest algo shell: bash @@ -162,7 +154,7 @@ jobs: run: | pwd ls -ll - ./runtests.sh --unittests + ./runtests.sh --build --unittests - name: Add reaction uses: peter-evans/create-or-update-comment@v2 From 3577ff346c1e48c8594bdfb64d7cf617b2500751 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 13:28:52 +0100 Subject: [PATCH 09/20] update dep Signed-off-by: Wenqi Li --- .github/workflows/integration.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 01e3417f05..cfe8f85d86 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -57,6 +57,7 @@ jobs: BUILD_MONAI=0 ./runtests.sh --build integration-auto3dseg: + needs: setup-folders container: image: nvcr.io/nvidia/pytorch:22.04-py3 # CUDA 11.6 py38 options: --gpus "device=1" --ipc host # shm-size 4g works fine @@ -104,6 +105,7 @@ jobs: reaction-type: rocket integration-full-unit: + needs: setup-folders container: image: nvcr.io/nvidia/pytorch:22.04-py3 # CUDA 11.6 py38 options: --gpus "device=2" --ipc host # shm-size 4g works fine From 22a39145751c6177a72ff84cf9c827b6015ec280 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 13:31:38 +0100 Subject: [PATCH 10/20] update temp Signed-off-by: Wenqi Li --- .github/workflows/integration.yml | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index cfe8f85d86..6457f8d3e0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -63,14 +63,11 @@ jobs: options: --gpus "device=1" --ipc host # shm-size 4g works fine runs-on: [self-hosted, linux, x64, command] steps: - # checkout the pull request branch - - uses: actions/checkout@v3 - with: - token: ${{ secrets.PR_MAINTAIN }} - repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} - ref: ${{ github.event.client_payload.pull_request.head.ref }} - name: Install the dependencies run: | + pwd + ls * + git log -1 which python python -m pip install --upgrade pip wheel python -m pip install --upgrade torch torchvision torchaudio @@ -111,14 +108,11 @@ jobs: options: --gpus "device=2" --ipc host # shm-size 4g works fine runs-on: [self-hosted, linux, x64, command1] steps: - # checkout the pull request branch - - uses: actions/checkout@v3 - with: - token: ${{ secrets.PR_MAINTAIN }} - repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} - ref: ${{ github.event.client_payload.pull_request.head.ref }} - name: Install the dependencies run: | + pwd + ls * + git log -1 which python python -m pip install --upgrade pip wheel python -m pip install --upgrade torch torchvision torchaudio @@ -130,7 +124,6 @@ jobs: BUILD_MONAI: 0 run: | # test latest template - echo "test latest algo" pwd cd ../ rm -rf research-contributions From 3b87927475adfaf96e51ea82fb78d6d88ee1393a Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 13:45:49 +0100 Subject: [PATCH 11/20] update temp Signed-off-by: Wenqi Li --- .github/workflows/integration.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6457f8d3e0..24ae39473c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -65,10 +65,7 @@ jobs: steps: - name: Install the dependencies run: | - pwd - ls * - git log -1 - which python + pwd && git log -1 && which python python -m pip install --upgrade pip wheel python -m pip install --upgrade torch torchvision torchaudio python -m pip install -r requirements-dev.txt @@ -78,6 +75,8 @@ jobs: env: BUILD_MONAI: 0 run: | + pwd && git log -1 && which python + ./runtests.sh -b python -m tests.test_auto3dseg_ensemble python -m tests.test_auto3dseg_hpo python -m tests.test_integration_autorunner @@ -88,8 +87,6 @@ jobs: env: BUILD_MONAI: 1 run: | - pwd - ls -ll ./runtests.sh --build --net - name: Add reaction @@ -110,10 +107,7 @@ jobs: steps: - name: Install the dependencies run: | - pwd - ls * - git log -1 - which python + pwd && git log -1 && which python python -m pip install --upgrade pip wheel python -m pip install --upgrade torch torchvision torchaudio python -m pip install -r requirements-dev.txt @@ -137,6 +131,8 @@ jobs: export OMP_NUM_THREADS=4 export MKL_NUM_THREADS=4 export MONAI_TESTING_ALGO_TEMPLATE=algorithm_templates + pwd && git log -1 && which python + ./runtests.sh -b python -m tests.test_auto3dseg_ensemble python -m tests.test_auto3dseg_hpo python -m tests.test_integration_autorunner @@ -147,8 +143,6 @@ jobs: env: BUILD_MONAI: 1 run: | - pwd - ls -ll ./runtests.sh --build --unittests - name: Add reaction From 30491a5ff384556f597e13d9a906fe10519954b7 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 13:56:20 +0100 Subject: [PATCH 12/20] update Signed-off-by: Wenqi Li --- .github/workflows/integration.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 24ae39473c..0d66100924 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -4,7 +4,7 @@ name: integration on: push: branches: - - temp-test + - dev repository_dispatch: type: [integration-test-command] @@ -41,7 +41,7 @@ jobs: pip uninstall -y monai pip uninstall -y monai-weekly pip uninstall -y monai-weekly - python -m pip install --upgrade torch torchvision torchaudio + python -m pip install --upgrade torch torchvision torchaudio torchtext python -m pip install -r requirements-dev.txt rm -rf /github/home/.cache/torch/hub/mmars/ - name: Clean directory @@ -67,7 +67,7 @@ jobs: run: | pwd && git log -1 && which python python -m pip install --upgrade pip wheel - python -m pip install --upgrade torch torchvision torchaudio + python -m pip install --upgrade torch torchvision torchaudio torchtext python -m pip install -r requirements-dev.txt - name: Auto3dseg tag algo @@ -109,7 +109,7 @@ jobs: run: | pwd && git log -1 && which python python -m pip install --upgrade pip wheel - python -m pip install --upgrade torch torchvision torchaudio + python -m pip install --upgrade torch torchvision torchaudio torchtext python -m pip install -r requirements-dev.txt - name: Auto3dseg latest algo From 9bec9092062923369864cc86f57e451f1938294e Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 14:42:45 +0100 Subject: [PATCH 13/20] temp test Signed-off-by: Wenqi Li --- .github/workflows/integration.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 0d66100924..a78e37e102 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -4,7 +4,7 @@ name: integration on: push: branches: - - dev + - temp-test repository_dispatch: type: [integration-test-command] @@ -63,6 +63,12 @@ jobs: options: --gpus "device=1" --ipc host # shm-size 4g works fine runs-on: [self-hosted, linux, x64, command] steps: + # checkout the pull request branch + - uses: actions/checkout@v3 + with: + token: ${{ secrets.PR_MAINTAIN }} + repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} + ref: ${{ github.event.client_payload.pull_request.head.ref }} - name: Install the dependencies run: | pwd && git log -1 && which python @@ -81,7 +87,6 @@ jobs: python -m tests.test_auto3dseg_hpo python -m tests.test_integration_autorunner python -m tests.test_integration_gpu_customization - - name: Integration tests shell: bash env: @@ -105,6 +110,12 @@ jobs: options: --gpus "device=2" --ipc host # shm-size 4g works fine runs-on: [self-hosted, linux, x64, command1] steps: + # checkout the pull request branch + - uses: actions/checkout@v3 + with: + token: ${{ secrets.PR_MAINTAIN }} + repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} + ref: ${{ github.event.client_payload.pull_request.head.ref }} - name: Install the dependencies run: | pwd && git log -1 && which python @@ -137,7 +148,6 @@ jobs: python -m tests.test_auto3dseg_hpo python -m tests.test_integration_autorunner python -m tests.test_integration_gpu_customization - - name: Unit tests shell: bash env: From 5b8064938bd5507d39304057b0d763e2e50f967e Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 14:44:20 +0100 Subject: [PATCH 14/20] update tests Signed-off-by: Wenqi Li --- .github/workflows/integration.yml | 126 +----------------------------- 1 file changed, 4 insertions(+), 122 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a78e37e102..d4938ca242 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -21,40 +21,8 @@ jobs: token: ${{ secrets.PR_MAINTAIN }} repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} ref: ${{ github.event.client_payload.pull_request.head.ref }} - - name: cache weekly timestamp - id: pip-cache - run: | - echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT - - name: cache for pip - uses: actions/cache@v3 - id: cache - with: - path: | - ~/.cache/pip - ~/.cache/torch - key: docker-py3-pip-${{ steps.pip-cache.outputs.datew }} - - name: Install the dependencies - run: | - which python - python -m pip install --upgrade pip wheel - pip uninstall -y monai - pip uninstall -y monai - pip uninstall -y monai-weekly - pip uninstall -y monai-weekly - python -m pip install --upgrade torch torchvision torchaudio torchtext - python -m pip install -r requirements-dev.txt - rm -rf /github/home/.cache/torch/hub/mmars/ - - name: Clean directory - run: | - python -m pip list - git config --global --add safe.directory /__w/MONAI/MONAI - git clean -ffdx && git reset --hard HEAD - nvidia-smi - export CUDA_VISIBLE_DEVICES=$(python -m tests.utils -c 1 | tail -n 1) - echo $CUDA_VISIBLE_DEVICES - python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))" - python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))' - BUILD_MONAI=0 ./runtests.sh --build + - name: make updates + run: touch hello.txt integration-auto3dseg: needs: setup-folders @@ -63,45 +31,9 @@ jobs: options: --gpus "device=1" --ipc host # shm-size 4g works fine runs-on: [self-hosted, linux, x64, command] steps: - # checkout the pull request branch - - uses: actions/checkout@v3 - with: - token: ${{ secrets.PR_MAINTAIN }} - repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} - ref: ${{ github.event.client_payload.pull_request.head.ref }} - name: Install the dependencies run: | - pwd && git log -1 && which python - python -m pip install --upgrade pip wheel - python -m pip install --upgrade torch torchvision torchaudio torchtext - python -m pip install -r requirements-dev.txt - - - name: Auto3dseg tag algo - shell: bash - env: - BUILD_MONAI: 0 - run: | - pwd && git log -1 && which python - ./runtests.sh -b - python -m tests.test_auto3dseg_ensemble - python -m tests.test_auto3dseg_hpo - python -m tests.test_integration_autorunner - python -m tests.test_integration_gpu_customization - - name: Integration tests - shell: bash - env: - BUILD_MONAI: 1 - run: | - ./runtests.sh --build --net - - - name: Add reaction - uses: peter-evans/create-or-update-comment@v2 - if: github.event.pull_request.number != '' - with: - token: ${{ secrets.PR_MAINTAIN }} - repository: ${{ github.event.client_payload.github.payload.repository.full_name }} - comment-id: ${{ github.event.client_payload.github.payload.comment.id }} - reaction-type: rocket + pwd && ls -a && git log -1 && which python integration-full-unit: needs: setup-folders @@ -110,56 +42,6 @@ jobs: options: --gpus "device=2" --ipc host # shm-size 4g works fine runs-on: [self-hosted, linux, x64, command1] steps: - # checkout the pull request branch - - uses: actions/checkout@v3 - with: - token: ${{ secrets.PR_MAINTAIN }} - repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} - ref: ${{ github.event.client_payload.pull_request.head.ref }} - name: Install the dependencies run: | - pwd && git log -1 && which python - python -m pip install --upgrade pip wheel - python -m pip install --upgrade torch torchvision torchaudio torchtext - python -m pip install -r requirements-dev.txt - - - name: Auto3dseg latest algo - shell: bash - env: - BUILD_MONAI: 0 - run: | - # test latest template - pwd - cd ../ - rm -rf research-contributions - rm -rf algorithm_templates - git clone --depth 1 --branch main --single-branch https://github.com/Project-MONAI/research-contributions.git - ls research-contributions/ - cp -r research-contributions/auto3dseg/algorithm_templates MONAI/ - cd research-contributions && git log -1 && cd ../MONAI - pwd - ls -ll - export OMP_NUM_THREADS=4 - export MKL_NUM_THREADS=4 - export MONAI_TESTING_ALGO_TEMPLATE=algorithm_templates - pwd && git log -1 && which python - ./runtests.sh -b - python -m tests.test_auto3dseg_ensemble - python -m tests.test_auto3dseg_hpo - python -m tests.test_integration_autorunner - python -m tests.test_integration_gpu_customization - - name: Unit tests - shell: bash - env: - BUILD_MONAI: 1 - run: | - ./runtests.sh --build --unittests - - - name: Add reaction - uses: peter-evans/create-or-update-comment@v2 - if: github.event.pull_request.number != '' - with: - token: ${{ secrets.PR_MAINTAIN }} - repository: ${{ github.event.client_payload.github.payload.repository.full_name }} - comment-id: ${{ github.event.client_payload.github.payload.comment.id }} - reaction-type: +1 + pwd && ls -a && git log -1 && which python From 2eb0fc1f2a39dc44c6c04d2a5fa0cb7b1bae2151 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 14:56:30 +0100 Subject: [PATCH 15/20] update Signed-off-by: Wenqi Li --- .github/workflows/integration.yml | 146 +++++++++++++++++++++++++++--- 1 file changed, 133 insertions(+), 13 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d4938ca242..275ab020fb 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,7 +9,7 @@ on: type: [integration-test-command] jobs: - setup-folders: + integration-auto3dseg: container: image: nvcr.io/nvidia/pytorch:22.04-py3 # CUDA 11.6 py38 options: --gpus "device=1" --ipc host # shm-size 4g works fine @@ -21,19 +21,65 @@ jobs: token: ${{ secrets.PR_MAINTAIN }} repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} ref: ${{ github.event.client_payload.pull_request.head.ref }} - - name: make updates - run: touch hello.txt - - integration-auto3dseg: - needs: setup-folders - container: - image: nvcr.io/nvidia/pytorch:22.04-py3 # CUDA 11.6 py38 - options: --gpus "device=1" --ipc host # shm-size 4g works fine - runs-on: [self-hosted, linux, x64, command] - steps: + - name: cache weekly timestamp + id: pip-cache + run: echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT + - name: cache for pip + uses: actions/cache@v3 + id: cache + with: + path: | + ~/.cache/pip + ~/.cache/torch + key: docker-py3-pip-${{ steps.pip-cache.outputs.datew }} - name: Install the dependencies run: | - pwd && ls -a && git log -1 && which python + pwd && git log -1 && which python + python -m pip install --upgrade pip wheel + pip uninstall -y monai + pip uninstall -y monai + pip uninstall -y monai-weekly + pip uninstall -y monai-weekly + python -m pip install --upgrade torch torchvision torchaudio torchtext + python -m pip install -r requirements-dev.txt + rm -rf /github/home/.cache/torch/hub/mmars/ + - name: Clean directory + run: | + python -m pip list + git config --global --add safe.directory /__w/MONAI/MONAI + git clean -ffdx && git reset --hard HEAD + nvidia-smi + export CUDA_VISIBLE_DEVICES=$(python -m tests.utils -c 1 | tail -n 1) + echo $CUDA_VISIBLE_DEVICES + python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))" + python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))' + + - name: Auto3dseg tag algo + shell: bash + env: + BUILD_MONAI: 0 + run: | + pwd && git log -1 && which python + ./runtests.sh -b + python -m tests.test_auto3dseg_ensemble + python -m tests.test_auto3dseg_hpo + python -m tests.test_integration_autorunner + python -m tests.test_integration_gpu_customization + - name: Integration tests + shell: bash + env: + BUILD_MONAI: 1 + run: ./runtests.sh --build --net + + - name: Add reaction + uses: peter-evans/create-or-update-comment@v2 + if: github.event.pull_request.number != '' + with: + token: ${{ secrets.PR_MAINTAIN }} + repository: ${{ github.event.client_payload.github.payload.repository.full_name }} + comment-id: ${{ github.event.client_payload.github.payload.comment.id }} + reaction-type: rocket + integration-full-unit: needs: setup-folders @@ -42,6 +88,80 @@ jobs: options: --gpus "device=2" --ipc host # shm-size 4g works fine runs-on: [self-hosted, linux, x64, command1] steps: + # checkout the pull request branch + - uses: actions/checkout@v3 + with: + token: ${{ secrets.PR_MAINTAIN }} + repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} + ref: ${{ github.event.client_payload.pull_request.head.ref }} + - name: cache weekly timestamp + id: pip-cache + run: echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT + - name: cache for pip + uses: actions/cache@v3 + id: cache + with: + path: | + ~/.cache/pip + ~/.cache/torch + key: docker-py3-pip-${{ steps.pip-cache.outputs.datew }} - name: Install the dependencies run: | - pwd && ls -a && git log -1 && which python + pwd && git log -1 && which python + python -m pip install --upgrade pip wheel + pip uninstall -y monai + pip uninstall -y monai + pip uninstall -y monai-weekly + pip uninstall -y monai-weekly + python -m pip install --upgrade torch torchvision torchaudio torchtext + python -m pip install -r requirements-dev.txt + rm -rf /github/home/.cache/torch/hub/mmars/ + - name: Clean directory + run: | + python -m pip list + git config --global --add safe.directory /__w/MONAI/MONAI + git clean -ffdx && git reset --hard HEAD + nvidia-smi + export CUDA_VISIBLE_DEVICES=$(python -m tests.utils -c 1 | tail -n 1) + echo $CUDA_VISIBLE_DEVICES + python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))" + python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))' + + - name: Auto3dseg latest algo + shell: bash + env: + BUILD_MONAI: 0 + run: | + pwd + cd ../ + rm -rf research-contributions + rm -rf algorithm_templates + git clone --depth 1 --branch main --single-branch https://github.com/Project-MONAI/research-contributions.git + ls research-contributions/ + cp -r research-contributions/auto3dseg/algorithm_templates MONAI/ + cd research-contributions && git log -1 && cd ../MONAI + pwd + ls -ll + export OMP_NUM_THREADS=4 + export MKL_NUM_THREADS=4 + export MONAI_TESTING_ALGO_TEMPLATE=algorithm_templates + pwd && git log -1 && which python + ./runtests.sh -b + python -m tests.test_auto3dseg_ensemble + python -m tests.test_auto3dseg_hpo + python -m tests.test_integration_autorunner + python -m tests.test_integration_gpu_customization + - name: Unit tests + shell: bash + env: + BUILD_MONAI: 1 + run: ./runtests.sh --build --unittests + + - name: Add reaction + uses: peter-evans/create-or-update-comment@v2 + if: github.event.pull_request.number != '' + with: + token: ${{ secrets.PR_MAINTAIN }} + repository: ${{ github.event.client_payload.github.payload.repository.full_name }} + comment-id: ${{ github.event.client_payload.github.payload.comment.id }} + reaction-type: +1 From 2770194d80ee0402f311d4da4e0cac4670715647 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 14:56:55 +0100 Subject: [PATCH 16/20] remove needs Signed-off-by: Wenqi Li --- .github/workflows/integration.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 275ab020fb..fd31b0885f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -82,7 +82,6 @@ jobs: integration-full-unit: - needs: setup-folders container: image: nvcr.io/nvidia/pytorch:22.04-py3 # CUDA 11.6 py38 options: --gpus "device=2" --ipc host # shm-size 4g works fine From e625ee0d16e26cd63ea04ded1b11e610a8916c92 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 14:58:23 +0100 Subject: [PATCH 17/20] dev tests Signed-off-by: Wenqi Li --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index fd31b0885f..7d22a9503e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -4,7 +4,7 @@ name: integration on: push: branches: - - temp-test + - dev repository_dispatch: type: [integration-test-command] From 040720fb0cc75cbb490bd8f110b9c12f509c0aa8 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 16:30:31 +0100 Subject: [PATCH 18/20] update Signed-off-by: Wenqi Li --- .github/workflows/integration.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7d22a9503e..0ed89c64ab 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -2,9 +2,6 @@ name: integration on: - push: - branches: - - dev repository_dispatch: type: [integration-test-command] From 51794c9666ded084c4673133620fb2ad9b532691 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 16:33:11 +0100 Subject: [PATCH 19/20] update Signed-off-by: Wenqi Li --- .github/workflows/integration.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 0ed89c64ab..e373c6ac34 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -78,7 +78,7 @@ jobs: reaction-type: rocket - integration-full-unit: + integration-unit: container: image: nvcr.io/nvidia/pytorch:22.04-py3 # CUDA 11.6 py38 options: --gpus "device=2" --ipc host # shm-size 4g works fine @@ -151,6 +151,7 @@ jobs: shell: bash env: BUILD_MONAI: 1 + QUICKTEST: True run: ./runtests.sh --build --unittests - name: Add reaction From 767bb1cf50609319e45f278422131c480186277b Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 19 Apr 2023 17:16:19 +0100 Subject: [PATCH 20/20] update template hash Signed-off-by: Wenqi Li --- monai/apps/auto3dseg/bundle_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/apps/auto3dseg/bundle_gen.py b/monai/apps/auto3dseg/bundle_gen.py index 98b6b2186d..27aed9a757 100644 --- a/monai/apps/auto3dseg/bundle_gen.py +++ b/monai/apps/auto3dseg/bundle_gen.py @@ -36,7 +36,7 @@ from monai.utils.enums import AlgoKeys logger = get_logger(module_name=__name__) -ALGO_HASH = os.environ.get("MONAI_ALGO_HASH", "5b4e379") +ALGO_HASH = os.environ.get("MONAI_ALGO_HASH", "e619d26") __all__ = ["BundleAlgo", "BundleGen"]