From 3a6591bb88886f90c33dbe9674894eec9aa71873 Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Date: Fri, 2 Jan 2026 10:30:00 -0600 Subject: [PATCH 1/9] Push image directly to ecr to fix "no space left on device" error --- .github/workflows/build_and_push_nf_base_images.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_and_push_nf_base_images.yml b/.github/workflows/build_and_push_nf_base_images.yml index 7971b9d5..40059f45 100644 --- a/.github/workflows/build_and_push_nf_base_images.yml +++ b/.github/workflows/build_and_push_nf_base_images.yml @@ -17,7 +17,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 with: - path: containers + path: containers - name: Checkout other private repository uses: actions/checkout@v3 @@ -46,16 +46,14 @@ jobs: echo "Building an image present in $dir" image_name=nextflow-approved/public tag_name=$(basename "$dir") - docker build -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir - docker push public.ecr.aws/u5x5h6w3/$image_name:$tag_name + docker buildx build --push -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir echo "Built an image with name --> $image_name:$tag_name" for dir in containers/nextflow-base-images/*/;do echo "Building an image present in $dir" image_name=nextflow-approved/public tag_name=$(basename "$dir") - docker build -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir - docker push public.ecr.aws/u5x5h6w3/$image_name:$tag_name + docker buildx build --push -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir echo "Built an image with name --> $image_name:$tag_name" done From 2d134142183cfe9ce8370cf9699449fcffccc5e4 Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Date: Fri, 2 Jan 2026 11:28:43 -0600 Subject: [PATCH 2/9] Update docker files for nextflow base images to reduce layers --- .../Dockerfile | 57 +++++++++++-------- .../Dockerfile | 17 +++--- .../Dockerfile | 57 +++++++++++-------- 3 files changed, 74 insertions(+), 57 deletions(-) diff --git a/nextflow-base-images/gen3-cuda-11.8-ubuntu22.04-openssl/Dockerfile b/nextflow-base-images/gen3-cuda-11.8-ubuntu22.04-openssl/Dockerfile index 28d14dad..85778e5c 100644 --- a/nextflow-base-images/gen3-cuda-11.8-ubuntu22.04-openssl/Dockerfile +++ b/nextflow-base-images/gen3-cuda-11.8-ubuntu22.04-openssl/Dockerfile @@ -1,36 +1,45 @@ # Use the specified base image -FROM nvcr.io/nvidia/cuda:11.8.0-base-ubuntu22.04 - -RUN apt-get purge -y --auto-remove openssl && apt-get autoremove && apt-get autoclean +FROM nvcr.io/nvidia/cuda:11.8.0-base-ubuntu22.04 AS openssl-builder -RUN apt-get update && apt-get -y upgrade && apt install -y wget +# Install build deps only +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + wget \ + ca-certificates && \ + rm -rf /var/lib/apt/lists/* -RUN apt install -y build-essential && \ - apt-get install -y python3 && \ - apt-get install -y python3-pip - -# install openssl 3.0.8 as it is required for FIPS compliance. WORKDIR /tmp RUN wget https://www.openssl.org/source/openssl-3.0.8.tar.gz && \ - tar -xzvf openssl-3.0.8.tar.gz && \ - rm openssl-3.0.8.tar.gz + tar -xzf openssl-3.0.8.tar.gz WORKDIR /tmp/openssl-3.0.8 -RUN ./Configure enable-fips && \ - make && \ - make install +RUN ./Configure enable-fips && make -j$(nproc) && make install_sw install_ssldirs -# Changing adding `/usr/local/lib` as a prefix to LD_LIBRARY_PATH will -# give precedence to OpenSSL 3.0.8 library files over the 3.0.2 -ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH +FROM nvcr.io/nvidia/cuda:11.8.0-base-ubuntu22.04 + +ENV DEBIAN_FRONTEND=noninteractive -# Make config changes ti ensure FIPS compliance -RUN sed -i 's$# .include fipsmodule.cnf$.include /usr/local/ssl/fipsmodule.cnf$g' /usr/local/ssl/openssl.cnf -RUN sed -i 's$providers = provider_sect$providers = provider_sect\nalg_section = algorithm_sect$g' /usr/local/ssl/openssl.cnf -RUN sed -i 's$# fips = fips_sect$fips = fips_sect$g' /usr/local/ssl/openssl.cnf -RUN sed -i -e 's$# activate = 1$activate = 1 \n\n[algorithm_sect]\ndefault_properties = fips=yes$g' /usr/local/ssl/openssl.cnf +# Install runtime deps only +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + python3 \ + python3-pip \ + libglib2.0-0 && \ + rm -rf /var/lib/apt/lists/* +# Copy OpenSSL runtime artifacts only +COPY --from=openssl-builder /usr/local/ssl /usr/local/ssl +COPY --from=openssl-builder /usr/local/lib /usr/local/lib +COPY --from=openssl-builder /usr/local/lib64 /usr/local/lib64 +COPY --from=openssl-builder /usr/local/bin/openssl /usr/local/bin/openssl + +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH +ENV OPENSSL_CONF=/usr/local/ssl/openssl.cnf -# Clean up the temporary directory +# FIPS config +RUN sed -i 's$# .include fipsmodule.cnf$.include /usr/local/ssl/fipsmodule.cnf$g' /usr/local/ssl/openssl.cnf && \ + sed -i 's$providers = provider_sect$providers = provider_sect\nalg_section = algorithm_sect$g' /usr/local/ssl/openssl.cnf && \ + sed -i 's$# fips = fips_sect$fips = fips_sect$g' /usr/local/ssl/openssl.cnf && \ + sed -i -e 's$# activate = 1$activate = 1 \n\n[algorithm_sect]\ndefault_properties = fips=yes$g' /usr/local/ssl/openssl.cnf WORKDIR / -RUN rm -rf /tmp/openssl-3.0.8 diff --git a/nextflow-base-images/gen3-cuda-12.3-torch2.2-ubuntu22.04-openssl/Dockerfile b/nextflow-base-images/gen3-cuda-12.3-torch2.2-ubuntu22.04-openssl/Dockerfile index 5ff21418..9b3467f3 100644 --- a/nextflow-base-images/gen3-cuda-12.3-torch2.2-ubuntu22.04-openssl/Dockerfile +++ b/nextflow-base-images/gen3-cuda-12.3-torch2.2-ubuntu22.04-openssl/Dockerfile @@ -3,20 +3,19 @@ FROM public.ecr.aws/u5x5h6w3/nextflow-approved/public:gen3-cuda-12.3-ubuntu22.04 # Set environment variables ENV DEBIAN_FRONTEND=noninteractive -# Install system dependencies +# Install system dependencies + clean up RUN apt-get update && \ - apt-get install -y \ + apt-get install -y --no-install-recommends \ git \ python3-pip \ python3-dev \ python3-opencv \ - libglib2.0-0 - -# Upgrade pip -RUN python3 -m pip install --upgrade pip - -# Install PyTorch and torchvision -RUN pip3 install torch -f https://download.pytorch.org/whl/cu123/torch_stable.html + libglib2.0-0 && \ + rm -rf /var/lib/apt/lists/* +# Upgrade pip and install torch (NO CACHE) +RUN python3 -m pip install --upgrade pip && \ + pip3 install --no-cache-dir torch \ + -f https://download.pytorch.org/whl/cu123/torch_stable.html # Set the working directory WORKDIR /app diff --git a/nextflow-base-images/gen3-cuda-12.3-ubuntu22.04-openssl/Dockerfile b/nextflow-base-images/gen3-cuda-12.3-ubuntu22.04-openssl/Dockerfile index 65e8347f..b66df72a 100644 --- a/nextflow-base-images/gen3-cuda-12.3-ubuntu22.04-openssl/Dockerfile +++ b/nextflow-base-images/gen3-cuda-12.3-ubuntu22.04-openssl/Dockerfile @@ -1,36 +1,45 @@ # Use the specified base image -FROM nvcr.io/nvidia/cuda:12.3.1-base-ubuntu22.04 - -RUN apt-get purge -y --auto-remove openssl && apt-get autoremove && apt-get autoclean +FROM nvcr.io/nvidia/cuda:12.3.1-base-ubuntu22.04 AS openssl-builder -RUN apt-get update && apt-get -y upgrade && apt install -y wget +# Install build deps only +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + wget \ + ca-certificates && \ + rm -rf /var/lib/apt/lists/* -RUN apt install -y build-essential && \ - apt-get install -y python3 && \ - apt-get install -y python3-pip - -# install openssl 3.0.8 as it is required for FIPS compliance. WORKDIR /tmp RUN wget https://www.openssl.org/source/openssl-3.0.8.tar.gz && \ - tar -xzvf openssl-3.0.8.tar.gz && \ - rm openssl-3.0.8.tar.gz + tar -xzf openssl-3.0.8.tar.gz WORKDIR /tmp/openssl-3.0.8 -RUN ./Configure enable-fips && \ - make && \ - make install +RUN ./Configure enable-fips && make -j$(nproc) && make install_sw install_ssldirs -# Changing adding `/usr/local/lib` as a prefix to LD_LIBRARY_PATH will -# give precedence to OpenSSL 3.0.8 library files over the 3.0.2 -ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH +FROM nvcr.io/nvidia/cuda:12.3.1-base-ubuntu22.04 + +ENV DEBIAN_FRONTEND=noninteractive -# Make config changes ti ensure FIPS compliance -RUN sed -i 's$# .include fipsmodule.cnf$.include /usr/local/ssl/fipsmodule.cnf$g' /usr/local/ssl/openssl.cnf -RUN sed -i 's$providers = provider_sect$providers = provider_sect\nalg_section = algorithm_sect$g' /usr/local/ssl/openssl.cnf -RUN sed -i 's$# fips = fips_sect$fips = fips_sect$g' /usr/local/ssl/openssl.cnf -RUN sed -i -e 's$# activate = 1$activate = 1 \n\n[algorithm_sect]\ndefault_properties = fips=yes$g' /usr/local/ssl/openssl.cnf +# Install runtime deps only +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + python3 \ + python3-pip \ + libglib2.0-0 && \ + rm -rf /var/lib/apt/lists/* +# Copy OpenSSL runtime artifacts only +COPY --from=openssl-builder /usr/local/ssl /usr/local/ssl +COPY --from=openssl-builder /usr/local/lib /usr/local/lib +COPY --from=openssl-builder /usr/local/lib64 /usr/local/lib64 +COPY --from=openssl-builder /usr/local/bin/openssl /usr/local/bin/openssl + +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH +ENV OPENSSL_CONF=/usr/local/ssl/openssl.cnf -# Clean up the temporary directory +# FIPS config +RUN sed -i 's$# .include fipsmodule.cnf$.include /usr/local/ssl/fipsmodule.cnf$g' /usr/local/ssl/openssl.cnf && \ + sed -i 's$providers = provider_sect$providers = provider_sect\nalg_section = algorithm_sect$g' /usr/local/ssl/openssl.cnf && \ + sed -i 's$# fips = fips_sect$fips = fips_sect$g' /usr/local/ssl/openssl.cnf && \ + sed -i -e 's$# activate = 1$activate = 1 \n\n[algorithm_sect]\ndefault_properties = fips=yes$g' /usr/local/ssl/openssl.cnf WORKDIR / -RUN rm -rf /tmp/openssl-3.0.8 From 43b75594c9ddbce2edaa29155fb9b494f8c09c98 Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Date: Fri, 2 Jan 2026 11:44:22 -0600 Subject: [PATCH 3/9] Test with current branch's Dockerfiles --- .github/workflows/build_and_push_nf_base_images.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_push_nf_base_images.yml b/.github/workflows/build_and_push_nf_base_images.yml index 40059f45..08b20ded 100644 --- a/.github/workflows/build_and_push_nf_base_images.yml +++ b/.github/workflows/build_and_push_nf_base_images.yml @@ -18,6 +18,7 @@ jobs: uses: actions/checkout@v3 with: path: containers + ref: 'chore/improve_nf_base_images_gh_Action' - name: Checkout other private repository uses: actions/checkout@v3 From cfcbc95aae1b9af292753b059be5948eaeb646bc Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Date: Fri, 2 Jan 2026 16:14:05 -0600 Subject: [PATCH 4/9] Remove branch name in base image checkout --- .github/workflows/build_and_push_nf_base_images.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_and_push_nf_base_images.yml b/.github/workflows/build_and_push_nf_base_images.yml index 08b20ded..40059f45 100644 --- a/.github/workflows/build_and_push_nf_base_images.yml +++ b/.github/workflows/build_and_push_nf_base_images.yml @@ -18,7 +18,6 @@ jobs: uses: actions/checkout@v3 with: path: containers - ref: 'chore/improve_nf_base_images_gh_Action' - name: Checkout other private repository uses: actions/checkout@v3 From 6647e4ba5041473ebd16f4d39d1c5a5f4f5c7c81 Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Date: Fri, 2 Jan 2026 16:33:14 -0600 Subject: [PATCH 5/9] Build for amd64 --- .github/workflows/build_and_push_nf_base_images.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_push_nf_base_images.yml b/.github/workflows/build_and_push_nf_base_images.yml index 40059f45..220b825c 100644 --- a/.github/workflows/build_and_push_nf_base_images.yml +++ b/.github/workflows/build_and_push_nf_base_images.yml @@ -18,6 +18,7 @@ jobs: uses: actions/checkout@v3 with: path: containers + ref: 'chore/improve_nf_base_images_gh_Action' - name: Checkout other private repository uses: actions/checkout@v3 @@ -46,14 +47,14 @@ jobs: echo "Building an image present in $dir" image_name=nextflow-approved/public tag_name=$(basename "$dir") - docker buildx build --push -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir + docker buildx build --push --platform linux/amd64 -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir echo "Built an image with name --> $image_name:$tag_name" for dir in containers/nextflow-base-images/*/;do echo "Building an image present in $dir" image_name=nextflow-approved/public tag_name=$(basename "$dir") - docker buildx build --push -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir + docker buildx build --push --platform linux/amd64 -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir echo "Built an image with name --> $image_name:$tag_name" done From 3894192ebfcf46dc78cc771d19be2a4b1731d571 Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Date: Fri, 2 Jan 2026 17:22:34 -0600 Subject: [PATCH 6/9] Update get_layers to fetch manifest layers when image is built using docker buildx --- .../workflows/get_layer_info_for_nf_imgs.yml | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/get_layer_info_for_nf_imgs.yml b/.github/workflows/get_layer_info_for_nf_imgs.yml index b6072c1e..eeeaf51d 100644 --- a/.github/workflows/get_layer_info_for_nf_imgs.yml +++ b/.github/workflows/get_layer_info_for_nf_imgs.yml @@ -45,10 +45,40 @@ jobs: echo "Line $LINENO: Rate limit exceeded, waiting for $wait_time seconds before retrying..." >&2 sleep $wait_time wait_time=$((wait_time * 2)) # Exponential backoff - else + continue + fi + + mediaType=$(jq -r '.mediaType // empty' <<<"$response") + + if [ "$mediaType" == "application/vnd.oci.image.index.v1+json" ]; then + # Pick the amd64/linux image manifest (ignore attestations) + digest=$(jq -r ' + .manifests[] + | select(.platform.architecture=="amd64" and .platform.os=="linux") + | .digest + ' <<<"$response") + + if [ -z "$digest" ]; then + echo "Line $LINENO: No amd64/linux manifest found in OCI index" >&2 + return 1 + fi + + # Replace tag with digest to form concrete manifest URL + resolved_url="$(sed "s|/manifests/.*|/manifests/$digest|" <<<"$url")" + + # Fetch and echo the resolved image manifest + response=$(curl -s \ + -H "Authorization: Bearer $TOKEN" \ + -H "Accept: application/vnd.oci.image.manifest.v1+json" \ + "$resolved_url") + echo "$response" return fi + + echo "$response" + return + done echo "Line $LINENO: Failed to fetch manifest after $retries attempts due to rate limiting." >&2 From c40a9b30d4f9fa16d3f9623721871a2df1d9493b Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Narumanchi Date: Fri, 2 Jan 2026 17:36:02 -0600 Subject: [PATCH 7/9] Improve manifest fetching with response logging Updated the fetching of the manifest to include logging of the response before extracting the last layer's digest. --- .github/workflows/get_layer_info_for_nf_imgs.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/get_layer_info_for_nf_imgs.yml b/.github/workflows/get_layer_info_for_nf_imgs.yml index eeeaf51d..019007c4 100644 --- a/.github/workflows/get_layer_info_for_nf_imgs.yml +++ b/.github/workflows/get_layer_info_for_nf_imgs.yml @@ -102,7 +102,9 @@ jobs: tag_name=$(echo "${image_url}" | sed 's|\(.*\):\(.*\)|\2|') echo "$LINENO: manifest_url = $manifest_url, tag_name = $tag_name" # Fetch the manifest with retry using exponential backoff - response=$(fetch_manifest "$manifest_url" | jq "[.layers[].digest]|.[-1]") + response=$(fetch_manifest "$manifest_url") + echo "Response from manifest_url: $response" + response=$(echo "${response}" | jq "[.layers[].digest]|.[-1]") if [ $? -ne 0 ] || [ -z "$response" ]; then echo "Line $LINENO: Failed to retrieve valid response from manifest_url -- $manifest_url" >&2 From 081ef25576a991cef63d4805385976c2677fd959 Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Date: Fri, 2 Jan 2026 17:45:51 -0600 Subject: [PATCH 8/9] change containers repo branch to get Dockerfiles from master --- .github/workflows/build_and_push_nf_base_images.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_and_push_nf_base_images.yml b/.github/workflows/build_and_push_nf_base_images.yml index 220b825c..8a1fcbdc 100644 --- a/.github/workflows/build_and_push_nf_base_images.yml +++ b/.github/workflows/build_and_push_nf_base_images.yml @@ -18,7 +18,6 @@ jobs: uses: actions/checkout@v3 with: path: containers - ref: 'chore/improve_nf_base_images_gh_Action' - name: Checkout other private repository uses: actions/checkout@v3 From e488a9d630728958946a9f46654a75def8457d99 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Tue, 6 Jan 2026 13:36:23 -0600 Subject: [PATCH 9/9] Consistent line logs and indentation fix --- .../workflows/get_layer_info_for_nf_imgs.yml | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/get_layer_info_for_nf_imgs.yml b/.github/workflows/get_layer_info_for_nf_imgs.yml index 019007c4..ddb15c02 100644 --- a/.github/workflows/get_layer_info_for_nf_imgs.yml +++ b/.github/workflows/get_layer_info_for_nf_imgs.yml @@ -29,7 +29,7 @@ jobs: if [ -z "$layer_json" ]; then layer_json="{}" fi - echo "Line: $LINENO layer_json=$layer_json" + echo "Line $LINENO: layer_json=$layer_json" # Function to fetch the manifest with retry logic for rate limiting fetch_manifest() { local url=$1 @@ -85,25 +85,26 @@ jobs: return 1 } + echo "Line $LINENO: started reading lines from script" while IFS= read -r image_url; do # Ignore lines that start with # - if [[ "$image_url" == \#* ]]; then - echo "Line_$LINENO: Skipping this line -- $image_url" - continue - fi + if [[ "$image_url" == \#* ]]; then + echo "Line $LINENO: Skipping this line -- $image_url" + continue + fi # Strip the first * if a line starts with * - if [[ "$image_url" == \** ]]; then - echo "Line_$LINENO: Stripping * from this line -- $image_url" - image_url=$(echo "${image_url:1}"| sed 's/^[ \t]*//;s/[ \t]*$//') - fi - echo "Line $LINENO: started reading lines from script" + if [[ "$image_url" == \** ]]; then + echo "Line $LINENO: Stripping * from this line -- $image_url" + image_url=$(echo "${image_url:1}"| sed 's/^[ \t]*//;s/[ \t]*$//') + fi + manifest_url=$(echo "${image_url}" | sed 's|public\.ecr\.aws/\(.*\):\(.*\)|https://public.ecr.aws/v2/\1/manifests/\2|') tag_name=$(echo "${image_url}" | sed 's|\(.*\):\(.*\)|\2|') - echo "$LINENO: manifest_url = $manifest_url, tag_name = $tag_name" + echo "Line $LINENO: manifest_url = $manifest_url - tag_name = $tag_name" # Fetch the manifest with retry using exponential backoff response=$(fetch_manifest "$manifest_url") - echo "Response from manifest_url: $response" + echo "Response from manifest_url: $response" response=$(echo "${response}" | jq "[.layers[].digest]|.[-1]") if [ $? -ne 0 ] || [ -z "$response" ]; then @@ -126,7 +127,7 @@ jobs: done <<< "$url_list" if [ -n "$layer_json" ]; then - echo "$layer_json" > $output_file_name + echo "$layer_json" > $output_file_name fi - name: Commit and push changes uses: stefanzweifel/git-auto-commit-action@v5