All-in-one minimal multi-stage Dockerfiles, and GitHub action to buil…

.dockerignore

@@ -0,0 +1,3 @@
+# When hacking the build process, we don't want to totally rebuild if we can help it
+.github
+test-results

.github/workflows/build-container-image.yaml

@@ -0,0 +1,165 @@
+name: Build container image
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [master]
+    tags:
+        - 'v*'
+  release:
+    types: [published]
+  pull_request:
+    branches: [master]
+    types: [ready_for_review, opened, synchronize, reopened]
+    paths:
+      - .github/workflows/linux_build.yaml
+      - .github/workflows/build-container-image.yaml
+      - ./App/**
+      - ./OREAnalytics/**
+      - '!./OREAnalytics/doc/**'
+      - ./OREData/**
+      - '!./OREData/doc/**'
+      - ./ORETest/**
+      - ./QuantExt/**
+      - '!./QuantExt/doc/**'
+      - CMakeLists.txt
+      - Docker/Dockerfile.aio.*
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository_owner }}/ore-engine
+
+jobs:
+  build-and-release:
+    runs-on: ubuntu-24.04
+
+    strategy:
+      matrix:
+        platform: 
+          - linux/amd64
+        flavour: 
+          - name: debian11
+          - name: centos-stream9
+        # fully-defined example:
+        # - name: centos-stream9
+        #   dockerfile_suffix: centos-stream9
+        #   tag_prefix: centos-stream9  
+        #   build_args: |
+        #     builder_base_image=quay.io/centos/centos:stream9
+        #     release_base_image=quay.io/centos/centos:stream9-minimal
+        #     code_ready_builder_repo=crb
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Log into registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # use both GHA and registry caching to make this faster
+      # https://blacksmith.sh/blog/cache-is-king-a-guide-for-docker-layer-caching-in-github-actions
+
+      # Build the first stage, which we can use as the testing facility    
+      - name: Extract metadata (tags, labels) for the builder
+        id: builder-meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-builder
+          flavor: |
+            latest=false
+            prefix=${{ matrix.flavour.tag_prefix || matrix.flavour.name }}-,onlatest=true
+          tags: |
+            type=sha,prefix=${{ matrix.flavour.tag_prefix || matrix.flavour.name }}-sha-
+            type=ref,event=branch
+            type=ref,event=tag
+      - name: Map builder tags to cache specs
+        uses: actions/github-script@v7
+        id: builder-tag-cache-mapper
+        env:
+          TAGS: ${{ steps.builder-meta.outputs.tags }}
+        with:
+          script: |
+            return {
+              cache_from: process.env.TAGS.split("\n").map(t=>'type=registry,ref='+ t).join("\n"),
+              cache_to: process.env.TAGS.split("\n").map(t=>'type=registry,ref='+ t + ',mode=max').join("\n")
+            }
+
+      - name: Build the "builder" container
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          load: true
+          file: ./Docker/Dockerfile.aio.${{ matrix.flavour.dockerfile_suffix || matrix.flavour.name }}
+          build-args: ${{ matrix.flavour.build_args }}
+          platforms: ${{ matrix.platform }}
+          tags: ore-builder
+          target: builder-tested
+          cache-from: |
+            type=gha,scope=${{ matrix.flavour.name }}
+            type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-builder:${{ matrix.flavour.tag_prefix || matrix.flavour.name }}-latest
+            ${{ fromJSON(steps.builder-tag-cache-mapper.outputs.result).cache_from }}
+          cache-to: |
+            type=gha,mode=max,scope=${{ matrix.flavour.name }}
+            ${{ (github.event_name != 'pull_request') && fromJSON(steps.builder-tag-cache-mapper.outputs.result).cache_to || '' }}
+
+      # Register the test results
+      - name: Fetch test results from builder image
+        uses: shrink/actions-docker-extract@v3
+        with:
+          image: ore-builder
+          path: /ore/test-results/.
+          destination: test-results
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          check_name: Test results for ${{ matrix.flavour.name }}
+          files: test-results/**/*.xml
+
+      # Build the rest of the stages - the first stage should be cached
+      - name: Extract metadata (tags, labels) for Docker release
+        id: release-meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            latest=auto
+            prefix=${{ matrix.flavour.tag_prefix || matrix.flavour.name }}-,onlatest=true
+          tags: |
+            type=sha,prefix=${{ matrix.flavour.tag_prefix || matrix.flavour.name }}-sha-
+            type=ref,event=branch
+            type=ref,event=tag
+      - name: Map release tags to cache specs
+        uses: actions/github-script@v7
+        id: release-tag-cache-mapper
+        env:
+          TAGS: ${{ steps.builder-meta.outputs.tags }}
+        with:
+          script: |
+            return {
+              cache_from: process.env.TAGS.split("\n").map(t=>'type=registry,ref='+ t).join("\n"),
+              cache_to: process.env.TAGS.split("\n").map(t=>'type=registry,ref='+ t + ',mode=max').join("\n")
+            }
+
+      - name: Build the "release" container
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Docker/Dockerfile.aio.${{ matrix.flavour.dockerfile_suffix || matrix.flavour.name }}
+          build-args: ${{ matrix.flavour.build_args }}
+          platforms: ${{ matrix.platform }}
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.release-meta.outputs.tags }}
+          labels: ${{ steps.release-meta.outputs.labels }}
+          cache-from: |
+            type=gha,scope=${{ matrix.flavour.name }}
+            type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ matrix.flavour.tag_prefix || matrix.flavour.name }}-latest
+            ${{ fromJSON(steps.release-tag-cache-mapper.outputs.result).cache_from }}
+          cache-to: ${{ (github.event_name != 'pull_request') && fromJSON(steps.release-tag-cache-mapper.outputs.result).cache_to || '' }}

Docker/Dockerfile.aio.centos-stream9

@@ -0,0 +1,119 @@
+ARG builder_base_image=quay.io/centos/centos:stream9
+ARG release_base_image=quay.io/centos/centos:stream9-minimal
+# set `variant` to `ci` to omit the test run - only useful for CI where we want to run (and extract) results separately
+ARG variant=tested
+
+#
+# builder image
+#
+FROM ${builder_base_image} AS builder-ci
+
+# the name of the Code Ready Builder repository (contains eigen3)
+ARG code_ready_builder_repo=crb
+
+# Set to override number of build jobs. Will default to number of cores available.
+ARG build_jobs=
+ARG cmake_build_type=Release
+
+COPY CMakeLists.txt /ore/CMakeLists.txt
+COPY QuantLib /ore/QuantLib
+COPY QuantExt /ore/QuantExt
+COPY OREData /ore/OREData
+COPY OREAnalytics /ore/OREAnalytics
+COPY App /ore/App
+COPY ThirdPartyLibs /ore/ThirdPartyLibs
+COPY ORETest /ore/ORETest
+COPY cmake /ore/cmake
+
+# Install dependencies
+RUN yum -q -y clean expire-cache \
+  && yum -q -y update \
+  && yum -q -y --enablerepo=${code_ready_builder_repo} install \
+    boost-devel \
+    bzip2-devel \
+    cmake \
+    # Example_43 requires this https://github.com/OpenSourceRisk/Engine/issues/142
+    eigen3-devel \
+    gcc \
+    gcc-c++ \
+    glibc-devel \
+    ninja-build \
+    ocl-icd-devel \
+    opencl-headers \
+    zlib-devel
+
+# set up the build
+RUN cd / \
+  && mkdir -p /ore/build && cd /ore/build \
+  && cmake /ore -GNinja -DCMAKE_BUILD_TYPE=${cmake_build_type} -DORE_BUILD_DOC=OFF -DORE_USE_ZLIB=ON -DQL_BUILD_EXAMPLES=false -DQL_BUILD_TEST_SUITE=false -DQL_BUILD_BENCHMARK=false -DQL_ENABLE_SESSIONS=ON -DORE_ENABLE_OPENCL=ON
+
+# build
+RUN cd /ore/build \
+  && cmake --build . -- -j${build_jobs:-$(nproc)} install
+
+RUN ldconfig
+
+# test facility
+# Set up the Python 3.9 venv so nose works.  Migration from nose seems needed
+RUN yum -q -y install python3.9 python3-pip 
+ENV VIRTUAL_ENV=/opt/python3.9-venv
+RUN python3.9 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN pip3 install \
+    datacompy \
+    jsondiff \
+    lxml \
+    matplotlib \
+    pandas \
+    nose \
+    nose_xunitmp \
+    xmldiff
+
+COPY Examples /ore/Examples
+COPY Tools /ore/Tools
+
+# 
+# testing
+#
+FROM builder-ci AS builder-tested
+ENV NOSE_PROCESSES=${build_jobs:-$(nproc)} NOSE_PROCESS_TIMEOUT=600 NOSE_WITH_XUNITMP=true NOSE_XUNITMP_FILE=/ore/test-results/examples.xml
+# This always succeeds - we test for the "failed" status in the downstream builder - allowing us to extract detailed results from this layer in CI
+RUN mkdir -p /ore/test-results && cd /ore/build \
+  && ( \
+    ctest -j${build_jobs:-$(nproc)} --test-dir . --output-junit /ore/test-results/ctest.xml --timeout 5400; \
+    FLAG=$?; \
+    if [ "$FLAG" -eq "0" ]; then echo "Test SUCCESS"; else echo "Test FAILURE"; fi; \
+    if [ "$FLAG" -ne "0" ]; then echo "FAILURE" > /ore/test-results/tests-failed; fi \
+  )
+
+#
+# final "builder"
+# 
+FROM builder-${variant} AS builder
+# assert that we must not have the "tests failed" indicator file from the variant builder
+RUN if [ -f /ore/test-results/tests-failed ]; then echo "Tests failed, cannot continue"; exit 1; fi
+
+#
+# release image
+#
+FROM ${release_base_image} AS release
+
+LABEL org.opencontainers.image.authors="Quaternion Risk Management"
+LABEL org.opencontainers.image.description="Open Source Risk Engine"
+
+# libs to run
+RUN microdnf install -y boost ocl-icd --nodocs && \
+    microdnf clean all -y
+
+# fetch built files from orebuild - we use a bind mount to allow us to copy library symlinks
+RUN mkdir -p /ore/bin /ore/lib
+RUN --mount=type=bind,from=builder,source=/usr/local,target=/orebuild cp -a /orebuild/bin/ore /ore/bin && cp -a /orebuild/lib/lib*.so* /ore/lib
+
+# tell the linker where the libraries are, as the relative directories are no longer valid
+ENV LD_LIBRARY_PATH=/ore/lib
+# ensure ore is on the PATH so that if a interactive terminal is used, it's easily findable
+ENV PATH=/ore/bin:$PATH
+# Adjust the locale so 1,000 comma separators are suppressed
+ENV LC_NUMERIC=C
+
+ENTRYPOINT ["/ore/bin/ore"]