From 6284070db214e0ff995620eaa3b0443202286daf Mon Sep 17 00:00:00 2001 From: FrankLeeeee Date: Thu, 12 Jan 2023 10:32:02 +0800 Subject: [PATCH 1/4] [workflow] automated bdist wheel build --- .bdist.json | 24 +++++++++ .github/workflows/auto_release_bdist.yml | 69 ++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 .bdist.json create mode 100644 .github/workflows/auto_release_bdist.yml diff --git a/.bdist.json b/.bdist.json new file mode 100644 index 000000000000..8693bca489e8 --- /dev/null +++ b/.bdist.json @@ -0,0 +1,24 @@ +{ + "build": [ + { + "torch_version": "1.11.0", + "cuda_image": "hpcaitech/cuda-conda:10.2" + }, + { + "torch_version": "1.11.0", + "cuda_image": "hpcaitech/cuda-conda:11.3" + }, + { + "torch_version": "1.12.1", + "cuda_image": "hpcaitech/cuda-conda:10.2" + }, + { + "torch_version": "1.12.1", + "cuda_image": "hpcaitech/cuda-conda:11.3" + }, + { + "torch_version": "1.12.1", + "cuda_image": "hpcaitech/cuda-conda:11.6" + } + ] +} diff --git a/.github/workflows/auto_release_bdist.yml b/.github/workflows/auto_release_bdist.yml new file mode 100644 index 000000000000..02f3eed19e42 --- /dev/null +++ b/.github/workflows/auto_release_bdist.yml @@ -0,0 +1,69 @@ +name: Auto Release bdist wheel + +on: + pull_request: + # paths: + # - 'version.txt' + # types: + # - closed + +jobs: + matrix_preparation: + name: Prepare Container List + # if: ( github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true ) && github.repository == 'hpcaitech/ColossalAI' + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v3 + - id: set-matrix + run: | + bdist=$(cat .bdist.json | tr '\n' ' ') + echo "matrix=${bdist}" >> $GITHUB_OUTPUT + + build: + name: Release bdist wheels + needs: matrix_preparation + runs-on: [self-hosted, gpu] + strategy: + fail-fast: false + matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}} + container: + image: ${{ matrix.build.cuda_image }} + options: --gpus all --rm + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + # cub is for cuda 10.2 + - name: Copy scripts + run: | + cp -r ./.github/workflows/scripts/* ./ + + # link the cache diretories to current path + ln -s /github/home/conda_pkgs ./conda_pkgs + ln -s /github/home/pip_wheels ./pip_wheels + + # set the conda package path + echo "pkgs_dirs:\n - $PWD/conda_pkgs" > ~/.condarc + + # set safe directory + git config --global --add safe.directory /__w/ColossalAI/ColossalAI + + # get cub package for cuda 10.2 + wget https://github.com/NVIDIA/cub/archive/refs/tags/1.8.0.zip + unzip 1.8.0.zip + - name: Build bdist wheel + run: | + pip install beautifulsoup4 requests packaging + python ./build_colossalai_wheel.py --torch_version $TORCH_VERSIONS + env: + TORCH_VERSIONS: ${{ matrix.build.torch_version }} + - name: 🚀 Deploy + uses: garygrossgarten/github-action-scp@release + with: + local: all_dist + remote: ${{ secrets.PRIVATE_PYPI_DIR }} + host: ${{ secrets.PRIVATE_PYPI_HOST }} + username: ${{ secrets.PRIVATE_PYPI_USER }} + password: ${{ secrets.PRIVATE_PYPI_PASSWD }} From fd4bd7d7f8da82f1668a611c71c3957fd0958b12 Mon Sep 17 00:00:00 2001 From: FrankLeeeee Date: Thu, 12 Jan 2023 10:51:03 +0800 Subject: [PATCH 2/4] polish workflow --- .github/workflows/README.md | 1 + .github/workflows/auto_release_bdist.yml | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index bc1f8504df3c..a3631d07fbab 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -55,6 +55,7 @@ In the section below, we will dive into the details of different workflows avail | `Release Nightly to PyPI` | `release_nightly.yml` | Build and release the nightly wheel to PyPI as `colossalai-nightly`. Automatically executed every Sunday. | | `Release Docker` | `release_docker.yml` | Build and release the Docker image to DockerHub. Triggered when the change of `version.txt` is merged. | | `Release bdist wheel` | `release_bdist.yml` | Build binary wheels with pre-built PyTorch extensions. Manually dispatched. See more details in the next section. | +| `Auto Release bdist wheel` | `auto_release_bdist.yml` | Build binary wheels with pre-built PyTorch extensions.Triggered when the change of `version.txt` is merged. | | `Auto Compatibility Test` | `auto_compatibility_test.yml` | Check Colossal-AI's compatiblity against the PyTorch and CUDA version specified in `.compatibility`. Triggered when `version.txt` is changed in a PR. | ### Manual Dispatch diff --git a/.github/workflows/auto_release_bdist.yml b/.github/workflows/auto_release_bdist.yml index 02f3eed19e42..56a3036f8c94 100644 --- a/.github/workflows/auto_release_bdist.yml +++ b/.github/workflows/auto_release_bdist.yml @@ -1,16 +1,17 @@ name: Auto Release bdist wheel on: + workflow_dispatch: pull_request: - # paths: - # - 'version.txt' - # types: - # - closed + paths: + - 'version.txt' + types: + - closed jobs: matrix_preparation: name: Prepare Container List - # if: ( github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true ) && github.repository == 'hpcaitech/ColossalAI' + if: ( github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true ) && github.repository == 'hpcaitech/ColossalAI' runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} From fc1a276dbf07f4a89c5d0bbe622a29185e3a2e1d Mon Sep 17 00:00:00 2001 From: FrankLeeeee Date: Thu, 12 Jan 2023 10:52:13 +0800 Subject: [PATCH 3/4] polish readme --- .github/workflows/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index a3631d07fbab..b6cb2e20f709 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -48,15 +48,15 @@ In the section below, we will dive into the details of different workflows avail ### Release -| Workflow Name | File name | Description | -| --------------------------- | ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -| `Draft GitHub Release Post` | `draft_github_release_post.yml` | Compose a GitHub release post draft based on the commit history. Triggered when the change of `version.txt` is merged. | -| `Release to PyPI` | `release_pypi.yml` | Build and release the wheel to PyPI. Triggered when the change of `version.txt` is merged. | -| `Release Nightly to PyPI` | `release_nightly.yml` | Build and release the nightly wheel to PyPI as `colossalai-nightly`. Automatically executed every Sunday. | -| `Release Docker` | `release_docker.yml` | Build and release the Docker image to DockerHub. Triggered when the change of `version.txt` is merged. | -| `Release bdist wheel` | `release_bdist.yml` | Build binary wheels with pre-built PyTorch extensions. Manually dispatched. See more details in the next section. | -| `Auto Release bdist wheel` | `auto_release_bdist.yml` | Build binary wheels with pre-built PyTorch extensions.Triggered when the change of `version.txt` is merged. | -| `Auto Compatibility Test` | `auto_compatibility_test.yml` | Check Colossal-AI's compatiblity against the PyTorch and CUDA version specified in `.compatibility`. Triggered when `version.txt` is changed in a PR. | +| Workflow Name | File name | Description | +| --------------------------- | ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Draft GitHub Release Post` | `draft_github_release_post.yml` | Compose a GitHub release post draft based on the commit history. Triggered when the change of `version.txt` is merged. | +| `Release to PyPI` | `release_pypi.yml` | Build and release the wheel to PyPI. Triggered when the change of `version.txt` is merged. | +| `Release Nightly to PyPI` | `release_nightly.yml` | Build and release the nightly wheel to PyPI as `colossalai-nightly`. Automatically executed every Sunday. | +| `Release Docker` | `release_docker.yml` | Build and release the Docker image to DockerHub. Triggered when the change of `version.txt` is merged. | +| `Release bdist wheel` | `release_bdist.yml` | Build binary wheels with pre-built PyTorch extensions. Manually dispatched. See more details in the next section. | +| `Auto Release bdist wheel` | `auto_release_bdist.yml` | Build binary wheels with pre-built PyTorch extensions.Triggered when the change of `version.txt` is merged. Build specificatons are stored in `.bdist.json` | +| `Auto Compatibility Test` | `auto_compatibility_test.yml` | Check Colossal-AI's compatiblity against the PyTorch and CUDA version specified in `.compatibility`. Triggered when `version.txt` is changed in a PR. | ### Manual Dispatch From 68ecb93e94935d687dec5466a7ddd768815d9108 Mon Sep 17 00:00:00 2001 From: FrankLeeeee Date: Thu, 12 Jan 2023 10:54:26 +0800 Subject: [PATCH 4/4] polish readme --- .github/workflows/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index b6cb2e20f709..cda6a3139a1b 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -106,6 +106,21 @@ This section lists the files used to configure the workflow. This `.compatibility` file is to tell GitHub Actions which PyTorch and CUDA versions to test against. Each line in the file is in the format `${torch-version}-${cuda-version}`, which is a tag for Docker image. Thus, this tag must be present in the [docker registry](https://hub.docker.com/r/pytorch/conda-cuda) so as to perform the test. +2. `.bdist.json` + +This file controls what pytorch/cuda compatible pre-built releases will be built and published. You can add a new entry according to the json schema below if there is a new wheel that needs to be built with AOT compilation of PyTorch extensions. + +```json +{ + "build": [ + { + "torch_version": "", + "cuda_image": "" + }, + ] +} +``` + ## Progress Log - [x] unit testing