From 6a3bed8d45ce519b7e762cc6e4748828a2eb1331 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Tue, 6 Apr 2021 17:15:33 +0100 Subject: [PATCH 1/6] refactor docker building Signed-off-by: Wenqi Li --- .dockerignore | 1 - .github/workflows/docker.yml | 113 +++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 44 ++++++++++++- .github/workflows/setupapp.yml | 43 ------------- Dockerfile | 3 +- 5 files changed, 155 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/docker.yml diff --git a/.dockerignore b/.dockerignore index 262da4d0dd..4e1161bfb2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,7 +8,6 @@ docs/ .coverage/ coverage.xml .readthedocs.yml -*.md *.toml !README.md diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000000..089d460b89 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,113 @@ +name: docker +# versioning: compute a static version file +# local_docker: use the version file to build docker images +# docker_test_latest: test the latest internal docker image (has flake) +# docker_test_dockerhub: test the latest dockerhub release (no flake) +on: + # master only docker deployment and quick tests + push: + branches: + - master + - fixes-dockerfile + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + versioning: + # compute versioning file from python setup.py + # upload as artifact + # (also used in release.yml) + if: github.repository == 'Project-MONAI/MONAI' + container: + image: localhost:5000/local_monai:latest + runs-on: [self-hosted, linux, x64, build_only] + steps: + - uses: actions/checkout@v2 + # full history so that we can git describe + with: + ref: master + fetch-depth: 0 + - shell: bash + run: | + git describe + python setup.py build + cat build/lib/monai/_version.py + - name: Upload version + uses: actions/upload-artifact@v2 + with: + name: _version.py + path: build/lib/monai/_version.py + - name: Clean up directory + shell: bash + run: | + ls -al + rm -rf {*,.[^.]*} + + local_docker: + # builds two versions: local_monai:latest and local_monai:dockerhub + # latest: used for local tests + # dockerhub: release, no flake package + if: github.repository == 'Project-MONAI/MONAI' + needs: versioning + runs-on: [self-hosted, linux, x64, build_only] + steps: + - uses: actions/checkout@v2 + with: + ref: master + - name: Download version + uses: actions/download-artifact@v2 + with: + name: _version.py + - name: docker_build + shell: bash + run: | + # get tag info for versioning + cat _version.py + mv _version.py monai/ + # build and run original docker image for local registry + docker build -t localhost:5000/local_monai:latest -f Dockerfile . + docker push localhost:5000/local_monai:latest + # build once more w/ tag "latest": remove flake package as it is not needed on hub.docker.com + sed -i '/flake/d' requirements-dev.txt + docker build -t projectmonai/monai:latest -f Dockerfile . + # also push as tag "dockerhub" to local registry + docker image tag projectmonai/monai:latest localhost:5000/local_monai:dockerhub + docker push localhost:5000/local_monai:dockerhub + # distribute as always w/ tag "latest" to hub.docker.com + echo "${{ secrets.DOCKER_PW }}" | docker login -u projectmonai --password-stdin + docker push projectmonai/monai:latest + docker logout + + docker_test_latest: + if: github.repository == 'Project-MONAI/MONAI' + needs: local_docker + container: + image: localhost:5000/local_monai:latest + runs-on: [self-hosted, linux, x64, common] + steps: + - name: Import + run: | + python -c 'import monai; monai.config.print_config()' + cd /opt/monai + ls -al + ngc --version + python -m tests.min_tests + env: + QUICKTEST: True + + docker_test_dockerhub: + if: github.repository == 'Project-MONAI/MONAI' + needs: local_docker + container: + image: localhost:5000/local_monai:dockerhub + runs-on: [self-hosted, linux, x64, common] + steps: + - name: Import + run: | + python -c 'import monai; monai.config.print_config()' + cd /opt/monai + ls -al + ngc --version + python -m tests.min_tests + env: + QUICKTEST: True diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f36abc9fcf..00e28ecd52 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -83,14 +83,49 @@ jobs: password: ${{ secrets.TEST_PYPI }} repository_url: https://test.pypi.org/legacy/ - release_docker: + versioning: + # compute versioning file from python setup.py + # upload as artifact + # (also used in docker.yml) if: github.repository == 'Project-MONAI/MONAI' needs: packaging - runs-on: [ self-hosted, linux, x64, build_only ] + container: + image: localhost:5000/local_monai:latest + runs-on: [self-hosted, linux, x64, build_only] steps: - uses: actions/checkout@v2 + # full history so that we can git describe with: ref: master + fetch-depth: 0 + - shell: bash + run: | + git describe + python setup.py build + cat build/lib/monai/_version.py + - name: Upload version + uses: actions/upload-artifact@v2 + with: + name: _version.py + path: build/lib/monai/_version.py + - name: Clean up directory + shell: bash + run: | + ls -al + rm -rf {*,.[^.]*} + + release_tag_docker: + if: github.repository == 'Project-MONAI/MONAI' + needs: versioning + runs-on: [self-hosted, linux, x64, build_only] + steps: + - uses: actions/checkout@v2 + with: + ref: master + - name: Download version + uses: actions/download-artifact@v2 + with: + name: _version.py - name: Set tag id: versioning run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} @@ -99,12 +134,15 @@ jobs: RELEASE_VERSION: ${{ steps.versioning.outputs.tag }} run: | echo "$RELEASE_VERSION" + cat _version.py - if: startsWith(github.ref, 'refs/tags/') name: build with the tag env: RELEASE_VERSION: ${{ steps.versioning.outputs.tag }} + shell: bash run: | - git fetch --depth=1 origin +refs/tags/*:refs/tags/* + # get tag info for versioning + mv _version.py monai/ # remove flake package as it is not needed on hub.docker.com sed -i '/flake/d' requirements-dev.txt docker build -t projectmonai/monai:"$RELEASE_VERSION" -f Dockerfile . diff --git a/.github/workflows/setupapp.yml b/.github/workflows/setupapp.yml index 450be403a0..dc65141fe8 100644 --- a/.github/workflows/setupapp.yml +++ b/.github/workflows/setupapp.yml @@ -148,46 +148,3 @@ jobs: python -m tests.min_tests env: QUICKTEST: True - - local_docker: - if: github.repository == 'Project-MONAI/MONAI' - runs-on: [self-hosted, linux, x64, build_only] - # we only push built container if it is built from master branch - steps: - - uses: actions/checkout@v2 - with: - ref: master - - name: docker_build - run: | - # get tag info for versioning - git fetch --depth=1 origin +refs/tags/*:refs/tags/* - # build and run original docker image for local registry - docker build -t localhost:5000/local_monai:latest -f Dockerfile . - docker push localhost:5000/local_monai:latest - # build once more w/ tag "latest": remove flake package as it is not needed on hub.docker.com - sed -i '/flake/d' requirements-dev.txt - docker build -t projectmonai/monai:latest -f Dockerfile . - # also push as tag "dockerhub" to local registry - docker image tag projectmonai/monai:latest localhost:5000/local_monai:dockerhub - docker push localhost:5000/local_monai:dockerhub - # distribute as always w/ tag "latest" to hub.docker.com - echo "${{ secrets.DOCKER_PW }}" | docker login -u projectmonai --password-stdin - docker push projectmonai/monai:latest - docker logout - - docker: - if: github.repository == 'Project-MONAI/MONAI' - needs: local_docker - container: - image: localhost:5000/local_monai:latest - runs-on: [self-hosted, linux, x64, common] - steps: - - name: Import - run: | - python -c 'import monai; monai.config.print_config()' - cd /opt/monai - ls -al - ngc --version - python -m tests.min_tests - env: - QUICKTEST: True diff --git a/Dockerfile b/Dockerfile index 57ea567869..23be9ae1c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,10 +30,9 @@ RUN cp /tmp/requirements.txt /tmp/req.bak \ # please specify exact files and folders to be copied -- else, basically always, the Docker build process cannot cache # this or anything below it and always will build from at most here; one file change leads to no caching from here on... -COPY LICENSE setup.py setup.cfg versioneer.py runtests.sh .gitignore .gitattributes README.md MANIFEST.in ./ +COPY LICENSE CHANGELOG.md CODE_OF_CONDUCT.md CONTRIBUTING.md README.md versioneer.py setup.py setup.cfg runtests.sh MANIFEST.in ./ COPY tests ./tests COPY monai ./monai -COPY .git ./.git RUN BUILD_MONAI=1 FORCE_CUDA=1 python setup.py develop \ && rm -rf build __pycache__ From 3aae3616f716b02374e9221dd400ab5d292c8d16 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Tue, 6 Apr 2021 20:27:02 +0100 Subject: [PATCH 2/6] temp tests Signed-off-by: Wenqi Li --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 089d460b89..a731bb27b6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v2 # full history so that we can git describe with: - ref: master + ref: fixes-dockerfile fetch-depth: 0 - shell: bash run: | @@ -53,7 +53,7 @@ jobs: steps: - uses: actions/checkout@v2 with: - ref: master + ref: fixes-dockerfile - name: Download version uses: actions/download-artifact@v2 with: From c67228937e7b6e7f553e8a9f24f7304d4740eeec Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Tue, 6 Apr 2021 20:48:26 +0100 Subject: [PATCH 3/6] Revert "temp tests" This reverts commit 3aae3616f716b02374e9221dd400ab5d292c8d16. Signed-off-by: Wenqi Li --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a731bb27b6..089d460b89 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v2 # full history so that we can git describe with: - ref: fixes-dockerfile + ref: master fetch-depth: 0 - shell: bash run: | @@ -53,7 +53,7 @@ jobs: steps: - uses: actions/checkout@v2 with: - ref: fixes-dockerfile + ref: master - name: Download version uses: actions/download-artifact@v2 with: From 6fa1c506e93b8d4a2874bf7358fc8e159194e4ad Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Tue, 6 Apr 2021 20:58:10 +0100 Subject: [PATCH 4/6] remove temp test Signed-off-by: Wenqi Li --- .github/workflows/docker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 089d460b89..2745d4169f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -8,7 +8,6 @@ on: push: branches: - master - - fixes-dockerfile # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From 0fa6b7ef91007a8d4e25ad387602923b707ac25b Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Tue, 6 Apr 2021 21:00:35 +0100 Subject: [PATCH 5/6] temp tests Signed-off-by: Wenqi Li --- .github/workflows/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 00e28ecd52..cebfc1c0dc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,7 @@ on: push: branches: - 'releases/*' + - fixes-dockerfile tags: - '*' @@ -96,7 +97,7 @@ jobs: - uses: actions/checkout@v2 # full history so that we can git describe with: - ref: master + ref: fixes-dockerfile fetch-depth: 0 - shell: bash run: | @@ -121,7 +122,7 @@ jobs: steps: - uses: actions/checkout@v2 with: - ref: master + ref: fixes-dockerfile - name: Download version uses: actions/download-artifact@v2 with: @@ -135,8 +136,7 @@ jobs: run: | echo "$RELEASE_VERSION" cat _version.py - - if: startsWith(github.ref, 'refs/tags/') - name: build with the tag + - name: build with the tag env: RELEASE_VERSION: ${{ steps.versioning.outputs.tag }} shell: bash From 9e3c3b18ad413dbbb7d60b40c4be2e894720fd6b Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Tue, 6 Apr 2021 21:16:01 +0100 Subject: [PATCH 6/6] Revert "temp tests" This reverts commit 0fa6b7ef91007a8d4e25ad387602923b707ac25b. Signed-off-by: Wenqi Li --- .github/workflows/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cebfc1c0dc..00e28ecd52 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,7 +4,6 @@ on: push: branches: - 'releases/*' - - fixes-dockerfile tags: - '*' @@ -97,7 +96,7 @@ jobs: - uses: actions/checkout@v2 # full history so that we can git describe with: - ref: fixes-dockerfile + ref: master fetch-depth: 0 - shell: bash run: | @@ -122,7 +121,7 @@ jobs: steps: - uses: actions/checkout@v2 with: - ref: fixes-dockerfile + ref: master - name: Download version uses: actions/download-artifact@v2 with: @@ -136,7 +135,8 @@ jobs: run: | echo "$RELEASE_VERSION" cat _version.py - - name: build with the tag + - if: startsWith(github.ref, 'refs/tags/') + name: build with the tag env: RELEASE_VERSION: ${{ steps.versioning.outputs.tag }} shell: bash