From 534f6f1d98e05b69c69bd6354ec5fc5a568c8592 Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Tue, 18 Jul 2023 11:51:32 -0700
Subject: [PATCH 01/20] +build-docker-image

---
 .github/workflows/gh-build.yml            | 14 +++-------
 continuous_integration/build-docker-image | 32 +++++++++++++++++++++++
 2 files changed, 36 insertions(+), 10 deletions(-)
 create mode 100644 continuous_integration/build-docker-image

diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml
index 259950c7e1..8182aa1de5 100644
--- a/.github/workflows/gh-build.yml
+++ b/.github/workflows/gh-build.yml
@@ -48,16 +48,10 @@ jobs:
           echo BUILD_TARGET: ${{ inputs.build-target }}
           echo USE_CUDA: ${{ env.USE_CUDA }}
 
-          docker build \
-            --build-arg AWS_SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }} \
-            --build-arg AWS_ACCESS_KEY_ID=${{ env.AWS_ACCESS_KEY_ID }} \
-            --build-arg AWS_SECRET_ACCESS_KEY=${{ env.AWS_SECRET_ACCESS_KEY }} \
-            --build-arg GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} \
-            --build-arg USE_CUDA=${{ env.USE_CUDA }} \
-            --progress=plain \
-            --tag=$IMAGE_NAME:${{ inputs.sha }} \
-            --label "git-commit=${{ inputs.sha }}" \
-            -f continuous_integration/Dockerfile .
+          continuous_integration/build-docker-image \
+            --image-name "$IMAGE_NAME" \
+            --sha ${{ inputs.sha }} \
+            --source-dir .
 
       - name: Log in to container image registry
         run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
diff --git a/continuous_integration/build-docker-image b/continuous_integration/build-docker-image
new file mode 100644
index 0000000000..dd5241a969
--- /dev/null
+++ b/continuous_integration/build-docker-image
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+#Set the options of the getopt command
+format=$(getopt -n "$0" -l "image-name:,sha:,source-dir:" -- -- "$@")
+if [ $# -lt 3 ]; then
+   echo "Wrong number of arguments are passed."
+   exit
+fi
+eval set -- "$format"
+
+#Read the argument values
+while [ $# -gt 0 ]
+do
+     case "$1" in
+          --image-name) IMAGE_NAME="$2"; shift;;
+          --sha) SHA="$2"; shift;;
+          --source) SOURCE_DIR="$2"; shift;;
+          --) shift;;
+     esac
+     shift;
+done
+
+docker build \
+  --build-arg AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN \
+  --build-arg AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+  --build-arg AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+  --build-arg GITHUB_TOKEN=$GITHUB_TOKEN \
+  --build-arg USE_CUDA=$USE_CUDA \
+  --progress=plain \
+  --tag=$IMAGE_NAME:$SHA \
+  --label "git-commit=$SHA" \
+  -f continuous_integration/Dockerfile $SOURCE_DIR

From 56e38cbd67ad566c8c7a7159c4d0501347b672a5 Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Thu, 20 Jul 2023 05:28:37 -0700
Subject: [PATCH 02/20] Minor change.

---
 .github/workflows/gh-build.yml            | 1 +
 continuous_integration/build-docker-image | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml
index 8182aa1de5..3025bf6d6b 100644
--- a/.github/workflows/gh-build.yml
+++ b/.github/workflows/gh-build.yml
@@ -48,6 +48,7 @@ jobs:
           echo BUILD_TARGET: ${{ inputs.build-target }}
           echo USE_CUDA: ${{ env.USE_CUDA }}
 
+          chmod +x continuous_integration/build-docker-image
           continuous_integration/build-docker-image \
             --image-name "$IMAGE_NAME" \
             --sha ${{ inputs.sha }} \
diff --git a/continuous_integration/build-docker-image b/continuous_integration/build-docker-image
index dd5241a969..15c7d63e6d 100644
--- a/continuous_integration/build-docker-image
+++ b/continuous_integration/build-docker-image
@@ -2,8 +2,8 @@
 
 #Set the options of the getopt command
 format=$(getopt -n "$0" -l "image-name:,sha:,source-dir:" -- -- "$@")
-if [ $# -lt 3 ]; then
-   echo "Wrong number of arguments are passed."
+if [ $# -lt 4 ]; then
+   echo "Wrong number of arguments passed."
    exit
 fi
 eval set -- "$format"

From 7c9a36b46106c70dc5b081e4d2c72f01d9d0922e Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Thu, 20 Jul 2023 07:41:28 -0700
Subject: [PATCH 03/20] 1. build-docker-image now takes base image name as an
 argument. 2. Renamed build-all to build-legate-all.

---
 .github/workflows/gh-build.yml                |  2 ++
 continuous_integration/Dockerfile             |  4 ++--
 continuous_integration/build-docker-image     | 24 ++++++++++---------
 .../bin/{build-all => build-legate-all}       |  4 ++--
 4 files changed, 19 insertions(+), 15 deletions(-)
 rename continuous_integration/home/coder/.local/bin/{build-all => build-legate-all} (77%)

diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml
index 3025bf6d6b..123d713c5e 100644
--- a/.github/workflows/gh-build.yml
+++ b/.github/workflows/gh-build.yml
@@ -15,6 +15,7 @@ on:
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  BASE_IMAGE: rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04
   IMAGE_NAME: legate.core-${{ inputs.build-target }}
   USE_CUDA: ${{ (inputs.build-target == 'cpu' && 'OFF') || 'ON' }}
 
@@ -50,6 +51,7 @@ jobs:
 
           chmod +x continuous_integration/build-docker-image
           continuous_integration/build-docker-image \
+            --base-image "$BASE_IMAGE" \
             --image-name "$IMAGE_NAME" \
             --sha ${{ inputs.sha }} \
             --source-dir .
diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile
index bcff43f08c..c459c11834 100644
--- a/continuous_integration/Dockerfile
+++ b/continuous_integration/Dockerfile
@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04
+ARG BASE_IMAGE
 FROM ${BASE_IMAGE} as stage0
 
 ENV PYTHONDONTWRITEBYTECODE=1
@@ -50,7 +50,7 @@ ENV AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
 # If .creds exists copy it to /run/secrets
 COPY --chown=coder:coder .cred[s] /run/secrets
 
-RUN entrypoint build-all
+RUN entrypoint build-legate-all
 
 #---------------------------------------------------
 FROM stage0 as final
diff --git a/continuous_integration/build-docker-image b/continuous_integration/build-docker-image
index 15c7d63e6d..1450c31638 100644
--- a/continuous_integration/build-docker-image
+++ b/continuous_integration/build-docker-image
@@ -1,8 +1,8 @@
 #!/bin/bash
 
 #Set the options of the getopt command
-format=$(getopt -n "$0" -l "image-name:,sha:,source-dir:" -- -- "$@")
-if [ $# -lt 4 ]; then
+format=$(getopt -n "$0" -l "base-image:,image-name:,sha:,source-dir:" -- -- "$@")
+if [ $# -lt 5 ]; then
    echo "Wrong number of arguments passed."
    exit
 fi
@@ -12,6 +12,7 @@ eval set -- "$format"
 while [ $# -gt 0 ]
 do
      case "$1" in
+          --base-image) BASE_IMAGE="$2"; shift;;
           --image-name) IMAGE_NAME="$2"; shift;;
           --sha) SHA="$2"; shift;;
           --source) SOURCE_DIR="$2"; shift;;
@@ -21,12 +22,13 @@ do
 done
 
 docker build \
-  --build-arg AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN \
-  --build-arg AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-  --build-arg AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-  --build-arg GITHUB_TOKEN=$GITHUB_TOKEN \
-  --build-arg USE_CUDA=$USE_CUDA \
-  --progress=plain \
-  --tag=$IMAGE_NAME:$SHA \
-  --label "git-commit=$SHA" \
-  -f continuous_integration/Dockerfile $SOURCE_DIR
+     --build-arg BASE_IMAGE=$BASE_IMAGE \
+     --build-arg AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN \
+     --build-arg AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+     --build-arg AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+     --build-arg GITHUB_TOKEN=$GITHUB_TOKEN \
+     --build-arg USE_CUDA=$USE_CUDA \
+     --progress=plain \
+     --tag=$IMAGE_NAME:$SHA \
+     --label "git-commit=$SHA" \
+     -f $SOURCE_DIR/continuous_integration/Dockerfile $SOURCE_DIR
diff --git a/continuous_integration/home/coder/.local/bin/build-all b/continuous_integration/home/coder/.local/bin/build-legate-all
similarity index 77%
rename from continuous_integration/home/coder/.local/bin/build-all
rename to continuous_integration/home/coder/.local/bin/build-legate-all
index da8d6c0201..46726e1ba4 100755
--- a/continuous_integration/home/coder/.local/bin/build-all
+++ b/continuous_integration/home/coder/.local/bin/build-legate-all
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 
-build_all() {
+build_legate_all() {
     set -x
     cd ~/;
 
@@ -14,4 +14,4 @@ build_all() {
     build-legate-conda;
 }
 
-(build_all "$@");
+(build_legate_all "$@");

From fa5b0a7023392b78c32e880d45bad0dff96e431e Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Thu, 20 Jul 2023 07:49:24 -0700
Subject: [PATCH 04/20] Hardened build-docker-image against failures.

---
 continuous_integration/build-docker-image | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/continuous_integration/build-docker-image b/continuous_integration/build-docker-image
index 1450c31638..0bff7b184d 100644
--- a/continuous_integration/build-docker-image
+++ b/continuous_integration/build-docker-image
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -xeuo pipefail
+
 #Set the options of the getopt command
 format=$(getopt -n "$0" -l "base-image:,image-name:,sha:,source-dir:" -- -- "$@")
 if [ $# -lt 5 ]; then

From 00c59ccefc89435ed161f4e619950e353269724b Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Thu, 20 Jul 2023 07:55:41 -0700
Subject: [PATCH 05/20] Fixed bug in argument parsing.

---
 continuous_integration/build-docker-image | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/continuous_integration/build-docker-image b/continuous_integration/build-docker-image
index 0bff7b184d..8315304c2d 100644
--- a/continuous_integration/build-docker-image
+++ b/continuous_integration/build-docker-image
@@ -17,7 +17,7 @@ do
           --base-image) BASE_IMAGE="$2"; shift;;
           --image-name) IMAGE_NAME="$2"; shift;;
           --sha) SHA="$2"; shift;;
-          --source) SOURCE_DIR="$2"; shift;;
+          --source-dir) SOURCE_DIR="$2"; shift;;
           --) shift;;
      esac
      shift;

From 669ca8afade63f8e550b8d7092019a71e511b1fa Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Thu, 20 Jul 2023 08:02:08 -0700
Subject: [PATCH 06/20] Made build-docker-image executable.

---
 continuous_integration/build-docker-image | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 continuous_integration/build-docker-image

diff --git a/continuous_integration/build-docker-image b/continuous_integration/build-docker-image
old mode 100644
new mode 100755

From 751f90ce14fccdac90115f857b1712d0f4759097 Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Thu, 20 Jul 2023 08:11:19 -0700
Subject: [PATCH 07/20] Delayed the use of set -e flag.

---
 continuous_integration/build-docker-image | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/continuous_integration/build-docker-image b/continuous_integration/build-docker-image
index 8315304c2d..16120fc3d3 100755
--- a/continuous_integration/build-docker-image
+++ b/continuous_integration/build-docker-image
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -xeuo pipefail
+set -xuo pipefail
 
 #Set the options of the getopt command
 format=$(getopt -n "$0" -l "base-image:,image-name:,sha:,source-dir:" -- -- "$@")
@@ -23,6 +23,8 @@ do
      shift;
 done
 
+set -e
+
 docker build \
      --build-arg BASE_IMAGE=$BASE_IMAGE \
      --build-arg AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN \

From 8f47435b7e54fcec2ac4e61184c058727cef4ce5 Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Thu, 20 Jul 2023 08:39:01 -0700
Subject: [PATCH 08/20] Removed the line changing ownership of .artifacts/ in
 Dockerfile.

---
 continuous_integration/Dockerfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile
index c459c11834..bcb5982421 100644
--- a/continuous_integration/Dockerfile
+++ b/continuous_integration/Dockerfile
@@ -28,8 +28,7 @@ COPY --chown=coder:coder . /home/coder/legate
 
 RUN chmod a+x /home/coder/.local/bin/* && \
     mkdir -p /tmp/out && \
-    chown -R coder:coder /tmp/out  && \
-    chown -R coder:coder /home/coder/.artifacts
+    chown -R coder:coder /tmp/out
 
 #---------------------------------------------------
 FROM stage0 as setup

From 72ae4636e0cb24bd891a933cd9e65527322c0f5c Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Thu, 20 Jul 2023 08:59:23 -0700
Subject: [PATCH 09/20] set -x.

---
 .../home/coder/.local/bin/get-yaml-and-make-conda-env           | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/continuous_integration/home/coder/.local/bin/get-yaml-and-make-conda-env b/continuous_integration/home/coder/.local/bin/get-yaml-and-make-conda-env
index 5568c95180..6105eca49e 100755
--- a/continuous_integration/home/coder/.local/bin/get-yaml-and-make-conda-env
+++ b/continuous_integration/home/coder/.local/bin/get-yaml-and-make-conda-env
@@ -2,4 +2,6 @@
 
 . conda-utils
 
+set -x;
+
 get_yaml_and_make_conda_env "$@";

From 563d3d6e6bfaa3c8636a9b0b379581f98b5eed95 Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Thu, 20 Jul 2023 09:08:40 -0700
Subject: [PATCH 10/20] List home dir.

---
 .../home/coder/.local/bin/get-yaml-and-make-conda-env           | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/continuous_integration/home/coder/.local/bin/get-yaml-and-make-conda-env b/continuous_integration/home/coder/.local/bin/get-yaml-and-make-conda-env
index 6105eca49e..3aac0a2652 100755
--- a/continuous_integration/home/coder/.local/bin/get-yaml-and-make-conda-env
+++ b/continuous_integration/home/coder/.local/bin/get-yaml-and-make-conda-env
@@ -4,4 +4,6 @@
 
 set -x;
 
+ls -lah
+
 get_yaml_and_make_conda_env "$@";

From 945c6918f6769b0465d611c341b61c8b59a0fa58 Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Thu, 20 Jul 2023 09:12:31 -0700
Subject: [PATCH 11/20] Change ownership of ~/ to coder:coder.

---
 continuous_integration/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile
index bcb5982421..4a6a47f33c 100644
--- a/continuous_integration/Dockerfile
+++ b/continuous_integration/Dockerfile
@@ -28,7 +28,8 @@ COPY --chown=coder:coder . /home/coder/legate
 
 RUN chmod a+x /home/coder/.local/bin/* && \
     mkdir -p /tmp/out && \
-    chown -R coder:coder /tmp/out
+    chown -R coder:coder /tmp/out && \
+    sudo chown -R coder:coder /home/coder
 
 #---------------------------------------------------
 FROM stage0 as setup

From 465159e7211ae6ea2a1fc78085845cde3c8a860b Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Fri, 21 Jul 2023 03:09:28 -0700
Subject: [PATCH 12/20] Docker build should not fail due to missing .creds
 folder.

---
 continuous_integration/Dockerfile             |  3 +-
 continuous_integration/build-docker-image     |  3 ++
 .../home/coder/.local/bin/entrypoint          | 39 ++++++++++++-------
 3 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile
index 4a6a47f33c..271507bea8 100644
--- a/continuous_integration/Dockerfile
+++ b/continuous_integration/Dockerfile
@@ -47,8 +47,7 @@ ENV AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
 ARG AWS_SECRET_ACCESS_KEY
 ENV AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
 
-# If .creds exists copy it to /run/secrets
-COPY --chown=coder:coder .cred[s] /run/secrets
+COPY --chown=coder:coder .creds /run/secrets
 
 RUN entrypoint build-legate-all
 
diff --git a/continuous_integration/build-docker-image b/continuous_integration/build-docker-image
index 16120fc3d3..6c46ff8397 100755
--- a/continuous_integration/build-docker-image
+++ b/continuous_integration/build-docker-image
@@ -25,6 +25,9 @@ done
 
 set -e
 
+# Avoid build errors due to a missing .creds folder
+mkdir -p $SOURCE_DIR/.creds
+
 docker build \
      --build-arg BASE_IMAGE=$BASE_IMAGE \
      --build-arg AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN \
diff --git a/continuous_integration/home/coder/.local/bin/entrypoint b/continuous_integration/home/coder/.local/bin/entrypoint
index 8124605ae6..8e9178dd97 100755
--- a/continuous_integration/home/coder/.local/bin/entrypoint
+++ b/continuous_integration/home/coder/.local/bin/entrypoint
@@ -1,7 +1,26 @@
 #!/usr/bin/env bash
 
-entrypoint() {
+sccache_stop_server_and_show_stats() {
+    sccache --stop-server || true && sccache --show-stats;
+}
+
+init_devcontainer() {
+    # disable xtrace and history
+    local xtrace_enabled=$(echo "${SHELLOPTS:-}" | grep -q 'xtrace'; echo $?);
+    local history_enabled=$(echo "${SHELLOPTS:-}" | grep -q 'history'; echo $?);
+    { set +xo history; } 2>/dev/null;
+    eval "export $(find /run/secrets/ -type f -exec bash -c 'echo ${0/\/run\/secrets\//}=$(<${0})' {} \;)";
+    if [ "${history_enabled}" -eq "0" ]; then { set -o history; } 2>/dev/null; fi;
+    if [ "${xtrace_enabled}" -eq "0" ]; then { set -o xtrace; } 2>/dev/null; fi;
+
+    . devcontainer-utils-post-attach-command;
 
+    sleep 10;
+    . devcontainer-utils-vault-s3-test;
+    . devcontainer-utils-vault-s3-export 0;
+}
+
+entrypoint() {
     set -x
 
     echo AWS_REGION=${AWS_REGION:-}
@@ -11,22 +30,12 @@ entrypoint() {
 
     mkdir -p /home/coder/.cache;
 
-    if [ -d /run/secrets ]; then
-        # disable xtrace and history
-        local xtrace_enabled=$(echo "${SHELLOPTS:-}" | grep -q 'xtrace'; echo $?);
-        local history_enabled=$(echo "${SHELLOPTS:-}" | grep -q 'history'; echo $?);
-        { set +xo history; } 2>/dev/null;
-        eval "export $(find /run/secrets/ -type f -exec bash -c 'echo ${0/\/run\/secrets\//}=$(<${0})' {} \;)";
-        if [ "${history_enabled}" -eq "0" ]; then { set -o history; } 2>/dev/null; fi;
-        if [ "${xtrace_enabled}" -eq "0" ]; then { set -o xtrace; } 2>/dev/null; fi;
-
-        . devcontainer-utils-post-attach-command;
+    local secrets_dir=/run/secrets
 
-        sleep 10;
-        . devcontainer-utils-vault-s3-test;
-        . devcontainer-utils-vault-s3-export 0;
+    if [ -d "$secrets_dir" ] && [ "$(ls -A $secrets_dir)" ]; then
+        init_devcontainer
     else
-        sccache --stop-server || true && sccache --show-stats;
+        sccache_stop_server_and_show_stats
     fi
 
     exec "$@";

From f4c639518c99fd1ec8c1a37245646e26591395ae Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Fri, 21 Jul 2023 04:26:19 -0700
Subject: [PATCH 13/20] USER coder.

---
 continuous_integration/Dockerfile | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile
index 271507bea8..c448cfcccb 100644
--- a/continuous_integration/Dockerfile
+++ b/continuous_integration/Dockerfile
@@ -34,10 +34,16 @@ RUN chmod a+x /home/coder/.local/bin/* && \
 #---------------------------------------------------
 FROM stage0 as setup
 
+USER coder
+WORKDIR /home/coder
+
 RUN get-yaml-and-make-conda-env
 
 #---------------------------------------------------
 FROM setup as build
+USER coder
+WORKDIR /home/coder
+
 ARG GITHUB_TOKEN
 ENV GITHUB_TOKEN=${GITHUB_TOKEN}
 ARG AWS_SESSION_TOKEN
@@ -53,6 +59,10 @@ RUN entrypoint build-legate-all
 
 #---------------------------------------------------
 FROM stage0 as final
-COPY --from=build /tmp/out /tmp/out
-COPY --from=build /tmp/conda-build /tmp/conda-build
-COPY --from=build /tmp/env_yaml /tmp/env_yaml
+USER coder
+WORKDIR /home/coder
+
+COPY --from=build --chown=coder:coder /tmp/out /tmp/out
+COPY --from=build --chown=coder:coder /tmp/conda-build /tmp/conda-build
+COPY --from=build --chown=coder:coder /tmp/env_yaml /tmp/env_yaml
+

From 777e05cbc6c564883350dc262ab2379c7e1fc363 Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Wed, 26 Jul 2023 04:54:21 -0700
Subject: [PATCH 14/20] +modify_env_for_tests

---
 .github/workflows/gh-build.yml                   |  5 +++++
 .../home/coder/.local/bin/conda-utils            | 16 +++++++++-------
 .../home/coder/.local/bin/run-test-or-analysis   |  6 +++---
 3 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml
index 123d713c5e..fcf577b3d4 100644
--- a/.github/workflows/gh-build.yml
+++ b/.github/workflows/gh-build.yml
@@ -44,6 +44,11 @@ jobs:
           role-duration-seconds: 28800 # 8 hours
           role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nv-legate
 
+      - name: Docker system prune
+        run: |
+          docker version
+          docker system prune --all --force
+
       - name: Build docker image
         run: |
           echo BUILD_TARGET: ${{ inputs.build-target }}
diff --git a/continuous_integration/home/coder/.local/bin/conda-utils b/continuous_integration/home/coder/.local/bin/conda-utils
index 2358b0bad8..d01573ae35 100644
--- a/continuous_integration/home/coder/.local/bin/conda-utils
+++ b/continuous_integration/home/coder/.local/bin/conda-utils
@@ -32,15 +32,15 @@ generate_yaml_file() {
 }
 
 find_yaml_file() {
-    pattern="/tmp/env_yaml/*.yaml"
-    files=( $pattern )
-    yaml_file="${files[0]}"
+    pattern="/tmp/env_yaml/*.yaml";
+    files=( $pattern );
+    yaml_file="${files[0]}";
 
     if [ -z "${yaml_file:-}" ] || [ ! -f "$yaml_file" ]; then
-        return 1
+        return 1;
     fi
 
-    return 0
+    return 0;
 }
 
 get_yaml_and_make_conda_env() {
@@ -58,8 +58,10 @@ get_yaml_and_make_conda_env() {
     make_conda_env_from_yaml;
 }
 
-make_conda_env_using_legate_core() {
-    mamba create -n "${DEFAULT_CONDA_ENV:-legate}"
+modify_env_for_tests() {
+    # WAR: legate-core depends on a different version of numpy than what is already installed.
+    # The correct version will be installed when legate-core is installed below.
+    mamba uninstall -y -n "${DEFAULT_CONDA_ENV:-legate}" numpy
 
     mamba install -y -n "${DEFAULT_CONDA_ENV:-legate}" -c nvidia -c conda-forge -c /tmp/conda-build/legate_core legate-core
 }
diff --git a/continuous_integration/home/coder/.local/bin/run-test-or-analysis b/continuous_integration/home/coder/.local/bin/run-test-or-analysis
index 326830c738..0f69b9c9aa 100644
--- a/continuous_integration/home/coder/.local/bin/run-test-or-analysis
+++ b/continuous_integration/home/coder/.local/bin/run-test-or-analysis
@@ -6,11 +6,11 @@ run_test_or_analysis() {
     set -x
     cd ~/
 
-    make_conda_env_using_legate_core
+    modify_env_for_tests;
 
-    activate_conda_env
+    activate_conda_env;
 
-    conda info
+    conda info;
 
     set -xeuo pipefail
 

From 34408f2877eeb47b1a07eba76fe04ef0d081ba5b Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Wed, 26 Jul 2023 05:25:11 -0700
Subject: [PATCH 15/20] -get-yaml-and-make-conda-env +install_legate_with_war.

---
 continuous_integration/Dockerfile                        | 3 +--
 continuous_integration/home/coder/.local/bin/conda-utils | 6 +++---
 .../home/coder/.local/bin/get-yaml-and-make-conda-env    | 9 ---------
 .../home/coder/.local/bin/run-test-or-analysis           | 2 +-
 4 files changed, 5 insertions(+), 15 deletions(-)
 delete mode 100755 continuous_integration/home/coder/.local/bin/get-yaml-and-make-conda-env

diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile
index c448cfcccb..050312d4fd 100644
--- a/continuous_integration/Dockerfile
+++ b/continuous_integration/Dockerfile
@@ -37,7 +37,7 @@ FROM stage0 as setup
 USER coder
 WORKDIR /home/coder
 
-RUN get-yaml-and-make-conda-env
+RUN set -x && . conda-utils && get_yaml_and_make_conda_env
 
 #---------------------------------------------------
 FROM setup as build
@@ -65,4 +65,3 @@ WORKDIR /home/coder
 COPY --from=build --chown=coder:coder /tmp/out /tmp/out
 COPY --from=build --chown=coder:coder /tmp/conda-build /tmp/conda-build
 COPY --from=build --chown=coder:coder /tmp/env_yaml /tmp/env_yaml
-
diff --git a/continuous_integration/home/coder/.local/bin/conda-utils b/continuous_integration/home/coder/.local/bin/conda-utils
index d01573ae35..2833a4da08 100644
--- a/continuous_integration/home/coder/.local/bin/conda-utils
+++ b/continuous_integration/home/coder/.local/bin/conda-utils
@@ -58,12 +58,12 @@ get_yaml_and_make_conda_env() {
     make_conda_env_from_yaml;
 }
 
-modify_env_for_tests() {
+install_legate_core_with_war() {
     # WAR: legate-core depends on a different version of numpy than what is already installed.
     # The correct version will be installed when legate-core is installed below.
-    mamba uninstall -y -n "${DEFAULT_CONDA_ENV:-legate}" numpy
+    mamba uninstall -y -n "${DEFAULT_CONDA_ENV:-legate}" numpy;
 
-    mamba install -y -n "${DEFAULT_CONDA_ENV:-legate}" -c nvidia -c conda-forge -c /tmp/conda-build/legate_core legate-core
+    mamba install -y -n "${DEFAULT_CONDA_ENV:-legate}" -c nvidia -c conda-forge -c /tmp/conda-build/legate_core legate-core;
 }
 
 activate_conda_env() {
diff --git a/continuous_integration/home/coder/.local/bin/get-yaml-and-make-conda-env b/continuous_integration/home/coder/.local/bin/get-yaml-and-make-conda-env
deleted file mode 100755
index 3aac0a2652..0000000000
--- a/continuous_integration/home/coder/.local/bin/get-yaml-and-make-conda-env
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-
-. conda-utils
-
-set -x;
-
-ls -lah
-
-get_yaml_and_make_conda_env "$@";
diff --git a/continuous_integration/home/coder/.local/bin/run-test-or-analysis b/continuous_integration/home/coder/.local/bin/run-test-or-analysis
index 0f69b9c9aa..155288f613 100644
--- a/continuous_integration/home/coder/.local/bin/run-test-or-analysis
+++ b/continuous_integration/home/coder/.local/bin/run-test-or-analysis
@@ -6,7 +6,7 @@ run_test_or_analysis() {
     set -x
     cd ~/
 
-    modify_env_for_tests;
+    install_legate_core_with_war;
 
     activate_conda_env;
 

From 6549c0985635d289a61ef51f02b07ee3042f8c83 Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Wed, 26 Jul 2023 05:37:06 -0700
Subject: [PATCH 16/20] Changed the shell used in Docker to bash.

---
 continuous_integration/Dockerfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile
index 050312d4fd..c2850f5f9a 100644
--- a/continuous_integration/Dockerfile
+++ b/continuous_integration/Dockerfile
@@ -1,6 +1,8 @@
 ARG BASE_IMAGE
 FROM ${BASE_IMAGE} as stage0
 
+SHELL ["/bin/bash", "-c"]
+
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV SCCACHE_REGION="us-east-2"
 ENV SCCACHE_BUCKET="rapids-sccache-east"

From b70e06ce6e29c93846e9752c435bb963177f1232 Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Wed, 26 Jul 2023 13:41:46 -0700
Subject: [PATCH 17/20] Removed the --sha arg from build-docker-image.

---
 .github/workflows/gh-build.yml            | 27 ++++++++++++++---------
 continuous_integration/build-docker-image | 26 ++++++++++------------
 2 files changed, 29 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml
index fcf577b3d4..c574b2f66a 100644
--- a/.github/workflows/gh-build.yml
+++ b/.github/workflows/gh-build.yml
@@ -54,35 +54,42 @@ jobs:
           echo BUILD_TARGET: ${{ inputs.build-target }}
           echo USE_CUDA: ${{ env.USE_CUDA }}
 
+          IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }}
+
           chmod +x continuous_integration/build-docker-image
           continuous_integration/build-docker-image \
             --base-image "$BASE_IMAGE" \
-            --image-name "$IMAGE_NAME" \
-            --sha ${{ inputs.sha }} \
+            --image-tag "$IMAGE_TAG" \
             --source-dir .
 
+      - name: Dump docker history of image before upload
+        run: |
+          IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }}
+          docker history $IMAGE_TAG
+
       - name: Log in to container image registry
         run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
 
       - name: Push image
         run: |
-          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+          IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }}
+
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}
 
           # Change all uppercase to lowercase
           IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
 
-          VERSION=${{ inputs.sha }}
+          IMAGE_ID=$IMAGE_ID/$IMAGE_TAG
 
-          docker tag $IMAGE_NAME:$VERSION $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION
+          docker tag $IMAGE_TAG $IMAGE_ID
+          docker push $IMAGE_ID
 
       - name: Copy artifacts back to the host
         run: |
+          IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }}
           mkdir -p artifacts
-          docker run  -v "$(pwd)/artifacts:/home/coder/.artifacts" --rm -t $IMAGE_NAME:${{ inputs.sha }} copy-artifacts
-          echo --------- DOCKER HISTORY START -----------
-          docker history $IMAGE_NAME:${{ inputs.sha }}
-          echo --------- DOCKER HISTORY END -----------
+          docker run  -v "$(pwd)/artifacts:/home/coder/.artifacts" --rm -t $IMAGE_TAG copy-artifacts
+
 
       - name: Display structure of workdir
         run: ls -R
diff --git a/continuous_integration/build-docker-image b/continuous_integration/build-docker-image
index 6c46ff8397..e36cf79743 100755
--- a/continuous_integration/build-docker-image
+++ b/continuous_integration/build-docker-image
@@ -3,8 +3,8 @@
 set -xuo pipefail
 
 #Set the options of the getopt command
-format=$(getopt -n "$0" -l "base-image:,image-name:,sha:,source-dir:" -- -- "$@")
-if [ $# -lt 5 ]; then
+format=$(getopt -n "$0" -l "base-image:,image-tag:,source-dir:" -- -- "$@")
+if [ $# -lt 4 ]; then
    echo "Wrong number of arguments passed."
    exit
 fi
@@ -15,8 +15,7 @@ while [ $# -gt 0 ]
 do
      case "$1" in
           --base-image) BASE_IMAGE="$2"; shift;;
-          --image-name) IMAGE_NAME="$2"; shift;;
-          --sha) SHA="$2"; shift;;
+          --image-tag) IMAGE_TAG="$2"; shift;;
           --source-dir) SOURCE_DIR="$2"; shift;;
           --) shift;;
      esac
@@ -26,16 +25,15 @@ done
 set -e
 
 # Avoid build errors due to a missing .creds folder
-mkdir -p $SOURCE_DIR/.creds
+mkdir -p "$SOURCE_DIR/.creds"
 
 docker build \
-     --build-arg BASE_IMAGE=$BASE_IMAGE \
-     --build-arg AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN \
-     --build-arg AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-     --build-arg AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-     --build-arg GITHUB_TOKEN=$GITHUB_TOKEN \
-     --build-arg USE_CUDA=$USE_CUDA \
+     --build-arg BASE_IMAGE="$BASE_IMAGE" \
+     --build-arg AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \
+     --build-arg AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
+     --build-arg AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
+     --build-arg GITHUB_TOKEN="$GITHUB_TOKEN" \
+     --build-arg USE_CUDA="$USE_CUDA" \
      --progress=plain \
-     --tag=$IMAGE_NAME:$SHA \
-     --label "git-commit=$SHA" \
-     -f $SOURCE_DIR/continuous_integration/Dockerfile $SOURCE_DIR
+     --tag="$IMAGE_TAG" \
+     -f "$SOURCE_DIR/continuous_integration/Dockerfile" "$SOURCE_DIR"

From a4c32f669c2806f97a6bb74c53a47f39bd4ca577 Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Fri, 4 Aug 2023 06:58:43 -0700
Subject: [PATCH 18/20] Removed unnecessary call to chmod.

---
 .github/workflows/gh-build.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml
index c574b2f66a..8c5b5976ce 100644
--- a/.github/workflows/gh-build.yml
+++ b/.github/workflows/gh-build.yml
@@ -56,7 +56,6 @@ jobs:
 
           IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }}
 
-          chmod +x continuous_integration/build-docker-image
           continuous_integration/build-docker-image \
             --base-image "$BASE_IMAGE" \
             --image-tag "$IMAGE_TAG" \

From c0100c7089d3c48f43cade4443dd08537f9e27e9 Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Fri, 4 Aug 2023 07:08:14 -0700
Subject: [PATCH 19/20] Removed `sudo chown -R coder:coder /home/code` from
 Dockerfile.

---
 continuous_integration/Dockerfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile
index c2850f5f9a..fbead86fcb 100644
--- a/continuous_integration/Dockerfile
+++ b/continuous_integration/Dockerfile
@@ -30,8 +30,7 @@ COPY --chown=coder:coder . /home/coder/legate
 
 RUN chmod a+x /home/coder/.local/bin/* && \
     mkdir -p /tmp/out && \
-    chown -R coder:coder /tmp/out && \
-    sudo chown -R coder:coder /home/coder
+    chown -R coder:coder /tmp/out
 
 #---------------------------------------------------
 FROM stage0 as setup

From 61d906f39d38a00da63afa38a2b746c4da212711 Mon Sep 17 00:00:00 2001
From: Sandeep Datta <sandeepd@nvidia.com>
Date: Tue, 8 Aug 2023 10:15:21 -0700
Subject: [PATCH 20/20] Adding a reference to the issue being worked around.

---
 continuous_integration/home/coder/.local/bin/conda-utils | 1 +
 1 file changed, 1 insertion(+)

diff --git a/continuous_integration/home/coder/.local/bin/conda-utils b/continuous_integration/home/coder/.local/bin/conda-utils
index 2833a4da08..a7c7b29f7e 100644
--- a/continuous_integration/home/coder/.local/bin/conda-utils
+++ b/continuous_integration/home/coder/.local/bin/conda-utils
@@ -61,6 +61,7 @@ get_yaml_and_make_conda_env() {
 install_legate_core_with_war() {
     # WAR: legate-core depends on a different version of numpy than what is already installed.
     # The correct version will be installed when legate-core is installed below.
+    # See github issue: https://github.com/nv-legate/legate.core/issues/812
     mamba uninstall -y -n "${DEFAULT_CONDA_ENV:-legate}" numpy;
 
     mamba install -y -n "${DEFAULT_CONDA_ENV:-legate}" -c nvidia -c conda-forge -c /tmp/conda-build/legate_core legate-core;