Gh docker

.dockerignore

@@ -0,0 +1 @@
+.gitignore

.github/workflows/ci-gh-docker-gpu-build-and-test.yml

@@ -0,0 +1,94 @@
+name: GH Docker build and test GPU legate.core
+
+concurrency:
+  group: ci-docker-gpu-on-${{ github.event_name }}-from-${{ github.ref_name }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches:
+      - "pull-request/[0-9]+"
+      - "branch-*"
+
+env:
+  DOCKER_BUILDKIT: 1
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  IMAGE_NAME: legate.core
+
+jobs:
+  build-gpu:
+    permissions:
+      id-token: write # This is required for configure-aws-credentials
+      contents: read  # This is required for actions/checkout
+      packages: write # This is required to push docker image to ghcr.io
+
+    runs-on: ${{ github.repository == 'nv-legate/legate.core' && 'linux-amd64-cpu4' || 'ubuntu-latest' }}
+    steps:
+      - name: Checkout legate.core (= this repo)
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Get AWS credentials for sccache bucket
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-region: us-east-2
+          role-duration-seconds: 28800 # 8 hours
+          role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nv-legate
+
+      - name: Build docker image
+        run: |
+          mkdir -p out conda-build
+          docker build \
+            --build-arg AWS_SESSION_TOKEN=${{env.AWS_SESSION_TOKEN}} \
+            --build-arg AWS_ACCESS_KEY_ID=${{env.AWS_ACCESS_KEY_ID}} \
+            --build-arg AWS_SECRET_ACCESS_KEY=${{env.AWS_SECRET_ACCESS_KEY}} \
+            --build-arg GITHUB_TOKEN=${{env.GITHUB_TOKEN}} \
+            --progress=plain \
+            --tag=$IMAGE_NAME \
+            --label "git-commit=${{github.sha}}" \
+            -f continuous_integration/Dockerfile .
+
+      - name: Log in to container image registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
+
+      - name: Push image
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+
+          # Change all uppercase to lowercase
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+          VERSION=${{github.sha}}
+
+          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION
+
+      - name: Display structure of workdir after docker build
+        run: ls -R
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: "legate.core-gpu-${{ github.sha }}"
+          path: |
+            conda-build
+            out
+
+  test-gpu:
+    needs:
+      - build-gpu
+    runs-on: linux-amd64-gpu-v100-latest-1
+    container:
+      options: -u root
+      image: ghcr.io/nv-legate/legate.core:${{github.sha}}
+      # image: rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04
+      env:
+        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+    steps:
+      - name: Run nvidia-smi to make sure GPU is working
+        run: nvidia-smi
+
+      - name: Run legate.core unit tests
+        shell: su coder {0}
+        run: entrypoint run-all-tests

.gitignore

@@ -38,3 +38,4 @@ _cmake_test_compile
 legate.core.code-workspace
 *.prof
 .legate-test-last-failed
+out/

continuous_integration/Dockerfile

@@ -0,0 +1,61 @@
+ARG BASE_IMAGE=rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04
+FROM ${BASE_IMAGE} as stage0
+
+RUN export DEBIAN_FRONTEND=noninteractive       \
+ && apt update                             \
+ && apt install -y --no-install-recommends \
+    rustc cargo
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV SCCACHE_REGION="us-east-2"
+ENV SCCACHE_BUCKET="rapids-sccache-east"
+ENV SCCACHE_S3_KEY_PREFIX=legate-cunumeric-dev
+ENV VAULT_HOST=https://vault.ops.k8s.rapids.ai
+ENV HISTFILE=/home/coder/.cache/._bash_history
+
+ARG USE_CUDA=ON
+ENV USE_CUDA=${USE_CUDA}
+
+ENV BUILD_MARCH=nocona
+
+ENV PATH="${PATH}:/home/coder/.local/bin"
+
+USER coder
+
+WORKDIR /home/coder/.cache
+WORKDIR /home/coder
+
+COPY --chown=coder:coder continuous_integration/home/coder/.gitconfig /home/coder/
+COPY --chown=coder:coder continuous_integration/home/coder/.local/bin/* /home/coder/.local/bin/
+COPY --chown=coder:coder . /home/coder/legate
+
+RUN chmod a+x /home/coder/.local/bin/* && \
+    mkdir -p /tmp/out && \
+    chown -R coder:coder /tmp/out
+
+FROM stage0 as setup
+
+RUN make-conda-env
+
+#---------------------------------------------------
+FROM setup as build
+
+ARG AWS_SESSION_TOKEN
+ENV AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}
+ARG AWS_ACCESS_KEY_ID
+ENV AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+ARG AWS_SECRET_ACCESS_KEY
+ENV AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+ARG GITHUB_TOKEN
+ENV GITHUB_TOKEN=${GITHUB_TOKEN}
+
+# If .creds exists copy it to /run/secrets
+COPY --chown=coder:coder .cred[s] /run/secrets
+
+RUN entrypoint build-all
+
+#---------------------------------------------------
+FROM stage0 as final
+COPY --from=build /tmp/out /tmp/out
+COPY --from=build /tmp/conda-build /tmp/conda-build
+COPY --from=build /tmp/env_yaml /tmp/env_yaml

continuous_integration/home/coder/.local/bin/activate-conda-env

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+activate_conda_env() {
+    conda activate legate;
+}
+
+. /opt/conda/etc/profile.d/conda.sh;
+. /opt/conda/etc/profile.d/mamba.sh;
+
+activate_conda_env "$@";

continuous_integration/home/coder/.local/bin/build-all

@@ -1,14 +1,14 @@
 #!/usr/bin/env bash
 
 build_all() {
-    set -xeuo pipefail;
-
     cd ~/;
 
-    if type conda >/dev/null 2>&1; then
-        . make-conda-env;
-    fi
+    . activate-conda-env;
+
+    conda info
 
+    set -xeuo pipefail;
+
     build-legate-cpp;
     build-legate-wheel;
     build-legate-conda;

continuous_integration/home/coder/.local/bin/entrypoint

@@ -2,12 +2,35 @@
 
 entrypoint() {
 
-    . devcontainer-utils-post-attach-command;
+    set -x
 
-    sccache --stop-server >/dev/null 2>&1 || true;
-    sccache --show-stats;
+    echo AWS_REGION=${AWS_REGION:-}
+    echo AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-}
+    echo AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
+    echo AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-}
+
+    mkdir -p /home/coder/.cache;
+
+    if [ -d /run/secrets ]; then
+        # disable xtrace and history
+        local xtrace_enabled=$(echo "${SHELLOPTS:-}" | grep -q 'xtrace'; echo $?);
+        local history_enabled=$(echo "${SHELLOPTS:-}" | grep -q 'history'; echo $?);
+        { set +xo history; } 2>/dev/null;
+        eval "export $(find /run/secrets/ -type f -exec bash -c 'echo ${0/\/run\/secrets\//}=$(<${0})' {} \;)";
+        if [ "${history_enabled}" -eq "0" ]; then { set -o history; } 2>/dev/null; fi;
+        if [ "${xtrace_enabled}" -eq "0" ]; then { set -o xtrace; } 2>/dev/null; fi;
+
+        . devcontainer-utils-post-attach-command;
+
+        sleep 10;
+        . devcontainer-utils-vault-s3-test;
+        . devcontainer-utils-vault-s3-export 0;
+    else
+        sccache --stop-server || true && sccache --show-stats;
+    fi
 
     exec "$@";
 }
 
 entrypoint "$@";
+

continuous_integration/home/coder/.local/bin/make-conda-env

@@ -1,8 +1,12 @@
 #!/usr/bin/env bash
 
+yaml_file="";
+
 make_conda_env() {
-    set -xeuo pipefail;
+    mamba env create -v -n "${DEFAULT_CONDA_ENV:-legate}" -f "${yaml_file}";
+}
 
+generate_yaml_file() {
     local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}}";
     cuda_version="$(echo "${cuda_version}" | cut -d'.' -f3 --complement)";
 
@@ -12,7 +16,7 @@ make_conda_env() {
         python_version="$(python3 --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f3 --complement)";
     fi
 
-    local yaml_file=~/"$(         \
+    yaml_file=~/"$(         \
         ~/legate/scripts/generate-conda-envs.py \
             --os linux                          \
             --compilers                         \
@@ -25,13 +29,34 @@ make_conda_env() {
 
     sed -i -re "s/legate-test/${DEFAULT_CONDA_ENV:-legate}/g" "${yaml_file}";
     echo "  - boa" >> "${yaml_file}";
-    cat "${yaml_file}";
 
     cp "${yaml_file}" /tmp/out/
+    mkdir -p /tmp/env_yaml
+    cp "${yaml_file}" /tmp/env_yaml
+}
+
+find_yaml_file() {
+    pattern="/tmp/env_yaml/*.yaml"
+    files=( $pattern )
+    yaml_file="${files[0]}"
+
+    if [ -z "${yaml_file:-}" ] || [ ! -f "$yaml_file" ]; then
+        return 1
+    fi
+
+    return 0
+}
 
-    mamba env create -n "${DEFAULT_CONDA_ENV:-legate}" -f "${yaml_file}";
+main() {
+    set -e;
+    if ! find_yaml_file; then
+        generate_yaml_file;
+    fi
+
+    echo YAML file: ${yaml_file}
+    cat "${yaml_file}";
 
-    { set +x; } 2>/dev/null;
+    make_conda_env;
 }
 
-(make_conda_env "$@");
+main "$@";

continuous_integration/home/coder/.local/bin/run-all-tests

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+run_all_tests() {
+    set -x;
+    cd ~/;
+    mamba create -n "${DEFAULT_CONDA_ENV:-legate}"
+    mamba install -y -n "${DEFAULT_CONDA_ENV:-legate}" -c nvidia -c conda-forge -c /tmp/conda-build/legate_core legate-core
+    mamba install -y --name "${DEFAULT_CONDA_ENV:-legate}" -c conda-forge pytest pytest-mock ipython jupyter_client
+
+    . activate-conda-env;
+
+    conda info
+
+    set -xeuo pipefail;
+
+    cd ~/legate/tests/unit
+    pytest
+}
+
+(run_all_tests "$@");