From a7da2b4dedca67c1f20f78b8118582369fe4bf37 Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Sun, 22 Jan 2023 21:57:47 +0100 Subject: [PATCH 01/17] Try using Depot --- .github/workflows/tests.yml | 34 ++------------------ cpu-arm.Makefile | 64 ++++++++++++++++++++++++++++++++++--- cpu-x86.Makefile | 58 ++++++++++++++++++++++++++++++--- depot.json | 3 ++ 4 files changed, 120 insertions(+), 39 deletions(-) create mode 100644 depot.json diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 999bf770..93de790b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,44 +24,16 @@ jobs: - uses: actions/checkout@v3 - # See https://stackoverflow.com/questions/70312490/github-actions-runner-environment-doesnt-build-for-arm-images - - name: Set up QEMU to build ARM images - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker buildx to use BuildKit features - uses: docker/setup-buildx-action@v2 - with: - # Sets up `docker build` command as an alias to `docker buildx` - install: true + - uses: depot/setup-action@v1 - name: Build Docker images - uses: docker/bake-action@v2.3.0 + run: make -f cpu-${{ matrix.cpu }}.Makefile docker-images env: + DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }} PHP_VERSION: ${{ matrix.php_version }} CPU: ${{ matrix.cpu }} CPU_PREFIX: ${{ (matrix.cpu == 'arm') && 'arm-' || '' }} IMAGE_VERSION_SUFFIX: ${{ (matrix.cpu == 'arm') && 'arm64' || 'x86_64' }} - with: - # This is needed to make the built images available in later steps - # https://docs.docker.com/engine/reference/commandline/buildx_build/#load - load: true - # Cache Docker layers in GitHub Actions cache, scoped per image - # https://github.com/docker/bake-action/issues/87#issuecomment-1184659151 - # We unfortunately don't use `mode=max` (which caches ALL layers instead of just tags) - # nor do we cache all images because it creates a huge number of cache requests - # and we get GitHub Actions cache timeouts: - # https://github.com/moby/buildkit/issues/2804 - set: | - base-devel.cache-from=type=gha,scope=base-devel-${{ matrix.cpu }} - base-devel.cache-to=type=gha,scope=base-devel-${{ matrix.cpu }} - build-php.cache-from=type=gha,scope=build-${{ matrix.cpu }}-php-${{ matrix.php_version }} - build-php.cache-to=type=gha,scope=build-${{ matrix.cpu }}-php-${{ matrix.php_version }} - fpm-internal-src.cache-from=type=gha,scope=fpm-internal-src - fpm-internal-src.cache-to=type=gha,scope=fpm-internal-src - console.cache-from=type=gha,scope=${{ matrix.cpu }}-console-${{ matrix.php_version }} - console.cache-to=type=gha,scope=${{ matrix.cpu }}-console-${{ matrix.php_version }} - php-fpm-dev.cache-from=type=gha,scope=${{ matrix.cpu }}-php-fpm-dev-${{ matrix.php_version }} - php-fpm-dev.cache-to=type=gha,scope=${{ matrix.cpu }}-php-fpm-dev-${{ matrix.php_version }} - name: Test that layers can be exported run: | diff --git a/cpu-arm.Makefile b/cpu-arm.Makefile index bbbb5f8f..68085eda 100644 --- a/cpu-arm.Makefile +++ b/cpu-arm.Makefile @@ -12,10 +12,66 @@ default: docker-images layers # Build Docker images *locally* -docker-images: - PHP_VERSION=80 docker buildx bake --load - PHP_VERSION=81 docker buildx bake --load - PHP_VERSION=82 docker buildx bake --load +docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 +docker-image-base-devel: + depot build \ + --platform=linux/arm64 \ + --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ + --load \ + --tag=bref/base-devel-${CPU} \ + base-devel +docker-image-fpm-internal-src: + depot build \ + --load \ + --tag=bref/fpm-internal-src \ + layers/fpm +docker-images-php-%: docker-image-base-devel docker-image-fpm-internal-src + # build + depot build \ + --platform=linux/arm64 \ + --build-arg=CPU=${CPU} \ + --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ + --load \ + --tag=bref/${CPU_PREFIX}build-php-$* \ + --file=php-$*/Dockerfile \ + --target=build-environment \ + . + # php + depot build \ + --platform=linux/arm64 \ + --build-arg=CPU=${CPU} \ + --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ + --load \ + --tag=bref/${CPU_PREFIX}php-$* \ + --file=php-$*/Dockerfile \ + --target=function \ + . + # php-fpm + depot build \ + --platform=linux/arm64 \ + --build-arg=CPU=${CPU} \ + --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ + --load \ + --tag=bref/${CPU_PREFIX}php-$*-fpm \ + --file=php-$*/Dockerfile \ + --target=fpm \ + . + # console + depot build \ + --platform=linux/arm64 \ + --build-arg=PHP_VERSION=$* \ + --build-arg=CPU_PREFIX=${CPU_PREFIX} \ + --load \ + --tag=bref/${CPU_PREFIX}php-$*-console \ + layers/console + # php-fpm-dev + depot build \ + --platform=linux/arm64 \ + --build-arg=PHP_VERSION=$* \ + --build-arg=CPU_PREFIX=${CPU_PREFIX} \ + --load \ + --tag=bref/${CPU_PREFIX}php-$*-fpm-dev \ + layers/fpm-dev # Build Lambda layers (zip files) *locally* diff --git a/cpu-x86.Makefile b/cpu-x86.Makefile index b904b3bf..761b54ed 100644 --- a/cpu-x86.Makefile +++ b/cpu-x86.Makefile @@ -12,10 +12,60 @@ default: docker-images layers # Build Docker images *locally* -docker-images: - PHP_VERSION=80 docker buildx bake --load - PHP_VERSION=81 docker buildx bake --load - PHP_VERSION=82 docker buildx bake --load +docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 +docker-image-base-devel: + depot build \ + --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ + --load \ + --tag=bref/base-devel-${CPU} \ + base-devel +docker-image-fpm-internal-src: + depot build \ + --load \ + --tag=bref/fpm-internal-src \ + layers/fpm +docker-images-php-%: docker-image-base-devel docker-image-fpm-internal-src + # build + depot build \ + --build-arg=CPU=${CPU} \ + --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ + --load \ + --tag=bref/${CPU_PREFIX}build-php-$* \ + --file=php-$*/Dockerfile \ + --target=build-environment \ + . + # php + depot build \ + --build-arg=CPU=${CPU} \ + --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ + --load \ + --tag=bref/${CPU_PREFIX}php-$* \ + --file=php-$*/Dockerfile \ + --target=function \ + . + # php-fpm + depot build \ + --build-arg=CPU=${CPU} \ + --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ + --load \ + --tag=bref/${CPU_PREFIX}php-$*-fpm \ + --file=php-$*/Dockerfile \ + --target=fpm \ + . + # console + depot build \ + --build-arg=PHP_VERSION=$* \ + --build-arg=CPU_PREFIX=${CPU_PREFIX} \ + --load \ + --tag=bref/${CPU_PREFIX}php-$*-console \ + layers/console + # php-fpm-dev + depot build \ + --build-arg=PHP_VERSION=$* \ + --build-arg=CPU_PREFIX=${CPU_PREFIX} \ + --load \ + --tag=bref/${CPU_PREFIX}php-$*-fpm-dev \ + layers/fpm-dev # Build Lambda layers (zip files) *locally* diff --git a/depot.json b/depot.json new file mode 100644 index 00000000..40eeea09 --- /dev/null +++ b/depot.json @@ -0,0 +1,3 @@ +{ + "id": "t048vfb17n" +} From 33609e663564f871143afecdbfcb8caad960ed51 Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Sun, 22 Jan 2023 22:32:14 +0100 Subject: [PATCH 02/17] Merge the base-devel image into PHP Dockerfiles This is to avoid having the dangling base-devel-xx Docker images that are used only to cache internal steps (common throughout PHP versions). Since Depot caches all intermediary layers very aggressively, that intermediary image that serves as a cache becomes useless. --- base-devel/Dockerfile | 332 ------------------------------------------ cpu-arm.Makefile | 9 +- cpu-x86.Makefile | 13 +- docker-bake.hcl | 16 -- php-80/Dockerfile | 332 +++++++++++++++++++++++++++++++++++++++++- php-81/Dockerfile | 332 +++++++++++++++++++++++++++++++++++++++++- php-82/Dockerfile | 332 +++++++++++++++++++++++++++++++++++++++++- 7 files changed, 991 insertions(+), 375 deletions(-) delete mode 100644 base-devel/Dockerfile diff --git a/base-devel/Dockerfile b/base-devel/Dockerfile deleted file mode 100644 index fa8a0447..00000000 --- a/base-devel/Dockerfile +++ /dev/null @@ -1,332 +0,0 @@ -# The container we build here contains everything needed to compile PHP. -# We build in here everything that is stable (e.g. system tools) so that we don't -# recompile them every time we change PHP. - - -# Can be "x86_64" or "arm64" -ARG IMAGE_VERSION_SUFFIX - - -# Lambda uses a custom AMI named Amazon Linux 2 -# https://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html -# AWS provides a Docker image that we use here: -# https://github.com/amazonlinux/container-images/tree/amzn2 -FROM public.ecr.aws/lambda/provided:al2-${IMAGE_VERSION_SUFFIX} - - -# Temp directory in which all compilation happens -WORKDIR /tmp - - -RUN set -xe \ - # Download yum repository data to cache - && yum makecache \ - # Default Development Tools - && yum groupinstall -y "Development Tools" --setopt=group_package_types=mandatory,default - - -# The default version of cmake is 2.8.12. We need cmake to build a few of -# our libraries, and at least one library requires a version of cmake greater than that. -# Needed to build: -# - libzip: minimum required CMAKE version 3.0. -RUN LD_LIBRARY_PATH= yum install -y cmake3 -# Override the default `cmake` -RUN ln -s /usr/bin/cmake3 /usr/bin/cmake - -# Use the bash shell, instead of /bin/sh -# Why? We need to document this. -SHELL ["/bin/bash", "-c"] - -# We need a base path for all the sourcecode we will build from. -ENV BUILD_DIR="/tmp/build" - -# Target installation path for all the packages we will compile -ENV INSTALL_DIR="/tmp/bref" - -# We need some default compiler variables setup -ENV PKG_CONFIG_PATH="${INSTALL_DIR}/lib64/pkgconfig:${INSTALL_DIR}/lib/pkgconfig" \ - PKG_CONFIG="/usr/bin/pkg-config" \ - PATH="${INSTALL_DIR}/bin:${PATH}" - -ENV LD_LIBRARY_PATH="${INSTALL_DIR}/lib64:${INSTALL_DIR}/lib" - -# Enable parallelism by default for make and cmake (like make -j) -# See https://stackoverflow.com/a/50883540/245552 -ENV CMAKE_BUILD_PARALLEL_LEVEL=4 -ENV MAKEFLAGS='-j4' - -# Ensure we have all the directories we require in the container. -RUN mkdir -p ${BUILD_DIR} \ - ${INSTALL_DIR}/bin \ - ${INSTALL_DIR}/doc \ - ${INSTALL_DIR}/etc/php \ - ${INSTALL_DIR}/etc/php/conf.d \ - ${INSTALL_DIR}/include \ - ${INSTALL_DIR}/lib \ - ${INSTALL_DIR}/lib64 \ - ${INSTALL_DIR}/libexec \ - ${INSTALL_DIR}/sbin \ - ${INSTALL_DIR}/share - - -############################################################################### -# OPENSSL -# https://github.com/openssl/openssl/releases -# Needs: -# - zlib -# Needed by: -# - curl -# - php -ENV VERSION_OPENSSL=1.1.1s -ENV OPENSSL_BUILD_DIR=${BUILD_DIR}/openssl -ENV CA_BUNDLE_SOURCE="https://curl.se/ca/cacert.pem" -ENV CA_BUNDLE="${INSTALL_DIR}/ssl/cert.pem" -RUN set -xe; \ - mkdir -p ${OPENSSL_BUILD_DIR}; \ - curl -Ls https://github.com/openssl/openssl/archive/OpenSSL_${VERSION_OPENSSL//./_}.tar.gz \ - | tar xzC ${OPENSSL_BUILD_DIR} --strip-components=1 -WORKDIR ${OPENSSL_BUILD_DIR}/ -RUN CFLAGS="" \ - CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ - LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ - ./config \ - --prefix=${INSTALL_DIR} \ - --openssldir=${INSTALL_DIR}/ssl \ - --release \ - no-tests \ - shared \ - zlib -# Explicitly compile make without parallelism because it fails if we use -jX (no error message) -# I'm not 100% sure why, and I already lost 4 hours on this, but I found this: -# https://github.com/openssl/openssl/issues/9931 -# https://stackoverflow.com/questions/28639207/why-cant-i-compile-openssl-with-multiple-threads-make-j3 -# Run `make install_sw install_ssldirs` instead of `make install` to skip installing man pages https://github.com/openssl/openssl/issues/8170 -RUN make -j1 install_sw install_ssldirs -RUN curl -Lk -o ${CA_BUNDLE} ${CA_BUNDLE_SOURCE} - - -############################################################################### -# LIBSSH2 -# https://github.com/libssh2/libssh2/releases -# Needs: -# - zlib -# - OpenSSL -# Needed by: -# - curl -ENV VERSION_LIBSSH2=1.10.0 -ENV LIBSSH2_BUILD_DIR=${BUILD_DIR}/libssh2 -RUN set -xe; \ - mkdir -p ${LIBSSH2_BUILD_DIR}/bin; \ - curl -Ls https://github.com/libssh2/libssh2/releases/download/libssh2-${VERSION_LIBSSH2}/libssh2-${VERSION_LIBSSH2}.tar.gz \ - | tar xzC ${LIBSSH2_BUILD_DIR} --strip-components=1 -WORKDIR ${LIBSSH2_BUILD_DIR}/bin/ -RUN CFLAGS="" \ - CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ - LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ - cmake .. \ - # Build as a shared library (.so) instead of a static one - -DBUILD_SHARED_LIBS=ON \ - # Build with OpenSSL support - -DCRYPTO_BACKEND=OpenSSL \ - # Build with zlib support - -DENABLE_ZLIB_COMPRESSION=ON \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ - -DCMAKE_BUILD_TYPE=RELEASE -RUN cmake --build . --target install - - -############################################################################### -# LIBNGHTTP2 -# This adds support for HTTP 2 requests in curl. -# See https://github.com/brefphp/bref/issues/727 and https://github.com/brefphp/bref/pull/740 -# https://github.com/nghttp2/nghttp2/releases -# Needs: -# - zlib -# - OpenSSL -# Needed by: -# - curl -ENV VERSION_NGHTTP2=1.51.0 -ENV NGHTTP2_BUILD_DIR=${BUILD_DIR}/nghttp2 -RUN set -xe; \ - mkdir -p ${NGHTTP2_BUILD_DIR}; \ - curl -Ls https://github.com/nghttp2/nghttp2/releases/download/v${VERSION_NGHTTP2}/nghttp2-${VERSION_NGHTTP2}.tar.gz \ - | tar xzC ${NGHTTP2_BUILD_DIR} --strip-components=1 -WORKDIR ${NGHTTP2_BUILD_DIR}/ -RUN CFLAGS="" \ - CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ - LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ - ./configure \ - --enable-lib-only \ - --prefix=${INSTALL_DIR} -RUN make install - - -############################################################################### -# CURL -# # https://github.com/curl/curl/releases -# # Needs: -# # - zlib -# # - OpenSSL -# # - libssh2 -# # Needed by: -# # - php -ENV VERSION_CURL=7.85.0 -ENV CURL_BUILD_DIR=${BUILD_DIR}/curl -RUN set -xe; \ - mkdir -p ${CURL_BUILD_DIR}/bin; \ - curl -Ls https://github.com/curl/curl/archive/curl-${VERSION_CURL//./_}.tar.gz \ - | tar xzC ${CURL_BUILD_DIR} --strip-components=1 -WORKDIR ${CURL_BUILD_DIR}/ -RUN ./buildconf \ - && CFLAGS="" \ - CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ - LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ - ./configure \ - --prefix=${INSTALL_DIR} \ - --with-ca-bundle=${CA_BUNDLE} \ - --enable-shared \ - --disable-static \ - --enable-optimize \ - --disable-warnings \ - --disable-dependency-tracking \ - --with-zlib \ - --enable-http \ - --enable-ftp \ - --enable-file \ - --enable-proxy \ - --enable-tftp \ - --enable-ipv6 \ - --enable-openssl-auto-load-config \ - --enable-cookies \ - --with-gnu-ld \ - --with-ssl \ - --with-libssh2 \ - --with-nghttp2 -RUN make install - - -############################################################################### -# LIBXML2 -# https://github.com/GNOME/libxml2/releases -# Uses: -# - zlib -# Needed by: -# - php -ENV VERSION_XML2=2.10.3 -ENV XML2_BUILD_DIR=${BUILD_DIR}/xml2 -RUN set -xe; \ - mkdir -p ${XML2_BUILD_DIR}; \ - curl -Ls https://download.gnome.org/sources/libxml2/${VERSION_XML2%.*}/libxml2-${VERSION_XML2}.tar.xz \ - | tar xJC ${XML2_BUILD_DIR} --strip-components=1 -WORKDIR ${XML2_BUILD_DIR}/ -RUN CFLAGS="" \ - CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ - LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ - ./configure \ - --prefix=${INSTALL_DIR} \ - --with-sysroot=${INSTALL_DIR} \ - --enable-shared \ - --disable-static \ - --with-html \ - --with-history \ - --enable-ipv6=no \ - --with-icu \ - --with-zlib \ - --without-python -RUN make install \ - && cp xml2-config ${INSTALL_DIR}/bin/xml2-config - - -############################################################################### -# LIBZIP -# https://github.com/nih-at/libzip/releases -# Needed by: -# - php -ENV VERSION_ZIP=1.9.2 -ENV ZIP_BUILD_DIR=${BUILD_DIR}/zip -RUN set -xe; \ - mkdir -p ${ZIP_BUILD_DIR}/bin/; \ - curl -Ls https://github.com/nih-at/libzip/releases/download/v${VERSION_ZIP}/libzip-${VERSION_ZIP}.tar.gz \ - | tar xzC ${ZIP_BUILD_DIR} --strip-components=1 -WORKDIR ${ZIP_BUILD_DIR}/bin/ -RUN CFLAGS="" \ - CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ - LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ - cmake .. \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ - -DCMAKE_BUILD_TYPE=RELEASE -RUN cmake --build . --target install - - -############################################################################### -# LIBSODIUM -# https://github.com/jedisct1/libsodium/releases -# Needed by: -# - php -ENV VERSION_LIBSODIUM=1.0.18 -ENV LIBSODIUM_BUILD_DIR=${BUILD_DIR}/libsodium -RUN set -xe; \ - mkdir -p ${LIBSODIUM_BUILD_DIR}; \ - curl -Ls https://github.com/jedisct1/libsodium/archive/${VERSION_LIBSODIUM}.tar.gz \ - | tar xzC ${LIBSODIUM_BUILD_DIR} --strip-components=1 -WORKDIR ${LIBSODIUM_BUILD_DIR}/ -RUN CFLAGS="" \ - CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ - LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ - ./autogen.sh \ -&& ./configure --prefix=${INSTALL_DIR} -RUN make install - - -############################################################################### -# Postgres -# https://github.com/postgres/postgres/releases -# Needs: -# - OpenSSL -# Needed by: -# - php -ENV VERSION_POSTGRES=15.1 -ENV POSTGRES_BUILD_DIR=${BUILD_DIR}/postgres -RUN set -xe; \ - mkdir -p ${POSTGRES_BUILD_DIR}/bin; \ - curl -Ls https://github.com/postgres/postgres/archive/REL_${VERSION_POSTGRES//./_}.tar.gz \ - | tar xzC ${POSTGRES_BUILD_DIR} --strip-components=1 -WORKDIR ${POSTGRES_BUILD_DIR}/ -RUN CFLAGS="" \ - CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ - LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ - ./configure --prefix=${INSTALL_DIR} --with-openssl --without-readline -RUN cd ${POSTGRES_BUILD_DIR}/src/interfaces/libpq && make && make install -RUN cd ${POSTGRES_BUILD_DIR}/src/bin/pg_config && make && make install -RUN cd ${POSTGRES_BUILD_DIR}/src/backend && make generated-headers -RUN cd ${POSTGRES_BUILD_DIR}/src/include && make install - - -############################################################################### -# Oniguruma -# This library is not packaged in PHP since PHP 7.4. -# See https://github.com/php/php-src/blob/43dc7da8e3719d3e89bd8ec15ebb13f997bbbaa9/UPGRADING#L578-L581 -# We do not install the system version because I didn't manage to make it work... -# Ideally we shouldn't compile it ourselves. -# https://github.com/kkos/oniguruma/releases -# Needed by: -# - php mbstring -ENV VERSION_ONIG=6.9.8 -ENV ONIG_BUILD_DIR=${BUILD_DIR}/oniguruma -RUN set -xe; \ - mkdir -p ${ONIG_BUILD_DIR}; \ - curl -Ls https://github.com/kkos/oniguruma/releases/download/v${VERSION_ONIG}/onig-${VERSION_ONIG}.tar.gz \ - | tar xzC ${ONIG_BUILD_DIR} --strip-components=1 -WORKDIR ${ONIG_BUILD_DIR} -RUN ./configure --prefix=${INSTALL_DIR} -RUN make && make install - - -############################################################################### -# Install some dev files for using old libraries already on the system -# readline-devel : needed for the readline extension -# gettext-devel : needed for the --with-gettext flag -# libicu-devel : needed for intl -# libxslt-devel : needed for the XSL extension -# sqlite-devel : Since PHP 7.4 this must be installed (https://github.com/php/php-src/blob/99b8e67615159fc600a615e1e97f2d1cf18f14cb/UPGRADING#L616-L619) -RUN LD_LIBRARY_PATH= yum install -y readline-devel gettext-devel libicu-devel libxslt-devel sqlite-devel diff --git a/cpu-arm.Makefile b/cpu-arm.Makefile index 68085eda..8cd4041b 100644 --- a/cpu-arm.Makefile +++ b/cpu-arm.Makefile @@ -13,19 +13,12 @@ default: docker-images layers # Build Docker images *locally* docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 -docker-image-base-devel: - depot build \ - --platform=linux/arm64 \ - --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ - --load \ - --tag=bref/base-devel-${CPU} \ - base-devel docker-image-fpm-internal-src: depot build \ --load \ --tag=bref/fpm-internal-src \ layers/fpm -docker-images-php-%: docker-image-base-devel docker-image-fpm-internal-src +docker-images-php-%: docker-image-fpm-internal-src # build depot build \ --platform=linux/arm64 \ diff --git a/cpu-x86.Makefile b/cpu-x86.Makefile index 761b54ed..cb58bf20 100644 --- a/cpu-x86.Makefile +++ b/cpu-x86.Makefile @@ -13,20 +13,15 @@ default: docker-images layers # Build Docker images *locally* docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 -docker-image-base-devel: - depot build \ - --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ - --load \ - --tag=bref/base-devel-${CPU} \ - base-devel docker-image-fpm-internal-src: depot build \ --load \ --tag=bref/fpm-internal-src \ layers/fpm -docker-images-php-%: docker-image-base-devel docker-image-fpm-internal-src +docker-images-php-%: docker-image-fpm-internal-src # build depot build \ + --platform=linux/amd64 \ --build-arg=CPU=${CPU} \ --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ --load \ @@ -36,6 +31,7 @@ docker-images-php-%: docker-image-base-devel docker-image-fpm-internal-src . # php depot build \ + --platform=linux/amd64 \ --build-arg=CPU=${CPU} \ --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ --load \ @@ -45,6 +41,7 @@ docker-images-php-%: docker-image-base-devel docker-image-fpm-internal-src . # php-fpm depot build \ + --platform=linux/amd64 \ --build-arg=CPU=${CPU} \ --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ --load \ @@ -54,6 +51,7 @@ docker-images-php-%: docker-image-base-devel docker-image-fpm-internal-src . # console depot build \ + --platform=linux/amd64 \ --build-arg=PHP_VERSION=$* \ --build-arg=CPU_PREFIX=${CPU_PREFIX} \ --load \ @@ -61,6 +59,7 @@ docker-images-php-%: docker-image-base-devel docker-image-fpm-internal-src layers/console # php-fpm-dev depot build \ + --platform=linux/amd64 \ --build-arg=PHP_VERSION=$* \ --build-arg=CPU_PREFIX=${CPU_PREFIX} \ --load \ diff --git a/docker-bake.hcl b/docker-bake.hcl index 8f20fcac..da088c07 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -15,14 +15,6 @@ variable "IMAGE_VERSION_SUFFIX" { default = "x86_64" } -target "base-devel" { - context = "base-devel" - tags = ["bref/base-devel-${CPU}"] - args = { - "IMAGE_VERSION_SUFFIX" = "${IMAGE_VERSION_SUFFIX}" - } -} - target "build-php" { dockerfile = "php-${PHP_VERSION}/Dockerfile" target = "build-environment" @@ -31,10 +23,6 @@ target "build-php" { "CPU" = "${CPU}" "IMAGE_VERSION_SUFFIX" = "${IMAGE_VERSION_SUFFIX}" } - contexts = { - // Dependency to the base image - "bref/base-devel-${CPU}" = "target:base-devel" - } } target "php" { @@ -46,7 +34,6 @@ target "php" { "IMAGE_VERSION_SUFFIX" = "${IMAGE_VERSION_SUFFIX}" } contexts = { - "bref/base-devel-${CPU}" = "target:base-devel" "bref/${CPU_PREFIX}build-php-${PHP_VERSION}" = "target:build-php" } } @@ -65,7 +52,6 @@ target "php-fpm" { "IMAGE_VERSION_SUFFIX" = "${IMAGE_VERSION_SUFFIX}" } contexts = { - "bref/base-devel-${CPU}" = "target:base-devel" "bref/${CPU_PREFIX}build-php-${PHP_VERSION}" = "target:build-php" "bref/${CPU_PREFIX}php-${PHP_VERSION}" = "target:php" "bref/fpm-internal-src" = "target:fpm-internal-src" @@ -81,7 +67,6 @@ target "console" { CPU_PREFIX = "${CPU_PREFIX}" } contexts = { - "bref/base-devel-${CPU}" = "target:base-devel" "bref/${CPU_PREFIX}build-php-${PHP_VERSION}" = "target:build-php" "bref/${CPU_PREFIX}php-${PHP_VERSION}" = "target:php" } @@ -95,7 +80,6 @@ target "php-fpm-dev" { CPU_PREFIX = "${CPU_PREFIX}" } contexts = { - "bref/base-devel-${CPU}" = "target:base-devel" "bref/${CPU_PREFIX}build-php-${PHP_VERSION}" = "target:build-php" "bref/${CPU_PREFIX}php-${PHP_VERSION}" = "target:php" "bref/${CPU_PREFIX}php-${PHP_VERSION}-fpm" = "target:php-fpm" diff --git a/php-80/Dockerfile b/php-80/Dockerfile index 2c9746bb..ab39fec5 100644 --- a/php-80/Dockerfile +++ b/php-80/Dockerfile @@ -5,12 +5,333 @@ ARG CPU # Can be "x86_64" or "arm64" ARG IMAGE_VERSION_SUFFIX -FROM bref/base-devel-${CPU} as build-environment +ARG VERSION_PHP=8.0.25 -ENV VERSION_PHP=8.0.25 -RUN mkdir -p /tmp/php -WORKDIR /tmp/php +# Lambda uses a custom AMI named Amazon Linux 2 +# https://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html +# AWS provides a Docker image that we use here: +# https://github.com/amazonlinux/container-images/tree/amzn2 +FROM public.ecr.aws/lambda/provided:al2-${IMAGE_VERSION_SUFFIX} as build-environment + + +# Temp directory in which all compilation happens +WORKDIR /tmp + + +RUN set -xe \ + # Download yum repository data to cache + && yum makecache \ + # Default Development Tools + && yum groupinstall -y "Development Tools" --setopt=group_package_types=mandatory,default + + +# The default version of cmake is 2.8.12. We need cmake to build a few of +# our libraries, and at least one library requires a version of cmake greater than that. +# Needed to build: +# - libzip: minimum required CMAKE version 3.0. +RUN LD_LIBRARY_PATH= yum install -y cmake3 +# Override the default `cmake` +RUN ln -s /usr/bin/cmake3 /usr/bin/cmake + +# Use the bash shell, instead of /bin/sh +# Why? We need to document this. +SHELL ["/bin/bash", "-c"] + +# We need a base path for all the sourcecode we will build from. +ENV BUILD_DIR="/tmp/build" + +# Target installation path for all the packages we will compile +ENV INSTALL_DIR="/tmp/bref" + +# We need some default compiler variables setup +ENV PKG_CONFIG_PATH="${INSTALL_DIR}/lib64/pkgconfig:${INSTALL_DIR}/lib/pkgconfig" \ + PKG_CONFIG="/usr/bin/pkg-config" \ + PATH="${INSTALL_DIR}/bin:${PATH}" + +ENV LD_LIBRARY_PATH="${INSTALL_DIR}/lib64:${INSTALL_DIR}/lib" + +# Enable parallelism by default for make and cmake (like make -j) +# See https://stackoverflow.com/a/50883540/245552 +ENV CMAKE_BUILD_PARALLEL_LEVEL=4 +ENV MAKEFLAGS='-j4' + +# Ensure we have all the directories we require in the container. +RUN mkdir -p ${BUILD_DIR} \ + ${INSTALL_DIR}/bin \ + ${INSTALL_DIR}/doc \ + ${INSTALL_DIR}/etc/php \ + ${INSTALL_DIR}/etc/php/conf.d \ + ${INSTALL_DIR}/include \ + ${INSTALL_DIR}/lib \ + ${INSTALL_DIR}/lib64 \ + ${INSTALL_DIR}/libexec \ + ${INSTALL_DIR}/sbin \ + ${INSTALL_DIR}/share + + +############################################################################### +# OPENSSL +# https://github.com/openssl/openssl/releases +# Needs: +# - zlib +# Needed by: +# - curl +# - php +ENV VERSION_OPENSSL=1.1.1s +ENV OPENSSL_BUILD_DIR=${BUILD_DIR}/openssl +ENV CA_BUNDLE_SOURCE="https://curl.se/ca/cacert.pem" +ENV CA_BUNDLE="${INSTALL_DIR}/ssl/cert.pem" +RUN set -xe; \ + mkdir -p ${OPENSSL_BUILD_DIR}; \ + curl -Ls https://github.com/openssl/openssl/archive/OpenSSL_${VERSION_OPENSSL//./_}.tar.gz \ + | tar xzC ${OPENSSL_BUILD_DIR} --strip-components=1 +WORKDIR ${OPENSSL_BUILD_DIR}/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./config \ + --prefix=${INSTALL_DIR} \ + --openssldir=${INSTALL_DIR}/ssl \ + --release \ + no-tests \ + shared \ + zlib +# Explicitly compile make without parallelism because it fails if we use -jX (no error message) +# I'm not 100% sure why, and I already lost 4 hours on this, but I found this: +# https://github.com/openssl/openssl/issues/9931 +# https://stackoverflow.com/questions/28639207/why-cant-i-compile-openssl-with-multiple-threads-make-j3 +# Run `make install_sw install_ssldirs` instead of `make install` to skip installing man pages https://github.com/openssl/openssl/issues/8170 +RUN make -j1 install_sw install_ssldirs +RUN curl -Lk -o ${CA_BUNDLE} ${CA_BUNDLE_SOURCE} + + +############################################################################### +# LIBSSH2 +# https://github.com/libssh2/libssh2/releases +# Needs: +# - zlib +# - OpenSSL +# Needed by: +# - curl +ENV VERSION_LIBSSH2=1.10.0 +ENV LIBSSH2_BUILD_DIR=${BUILD_DIR}/libssh2 +RUN set -xe; \ + mkdir -p ${LIBSSH2_BUILD_DIR}/bin; \ + curl -Ls https://github.com/libssh2/libssh2/releases/download/libssh2-${VERSION_LIBSSH2}/libssh2-${VERSION_LIBSSH2}.tar.gz \ + | tar xzC ${LIBSSH2_BUILD_DIR} --strip-components=1 +WORKDIR ${LIBSSH2_BUILD_DIR}/bin/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + cmake .. \ + # Build as a shared library (.so) instead of a static one + -DBUILD_SHARED_LIBS=ON \ + # Build with OpenSSL support + -DCRYPTO_BACKEND=OpenSSL \ + # Build with zlib support + -DENABLE_ZLIB_COMPRESSION=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE=RELEASE +RUN cmake --build . --target install + + +############################################################################### +# LIBNGHTTP2 +# This adds support for HTTP 2 requests in curl. +# See https://github.com/brefphp/bref/issues/727 and https://github.com/brefphp/bref/pull/740 +# https://github.com/nghttp2/nghttp2/releases +# Needs: +# - zlib +# - OpenSSL +# Needed by: +# - curl +ENV VERSION_NGHTTP2=1.51.0 +ENV NGHTTP2_BUILD_DIR=${BUILD_DIR}/nghttp2 +RUN set -xe; \ + mkdir -p ${NGHTTP2_BUILD_DIR}; \ + curl -Ls https://github.com/nghttp2/nghttp2/releases/download/v${VERSION_NGHTTP2}/nghttp2-${VERSION_NGHTTP2}.tar.gz \ + | tar xzC ${NGHTTP2_BUILD_DIR} --strip-components=1 +WORKDIR ${NGHTTP2_BUILD_DIR}/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./configure \ + --enable-lib-only \ + --prefix=${INSTALL_DIR} +RUN make install + + +############################################################################### +# CURL +# # https://github.com/curl/curl/releases +# # Needs: +# # - zlib +# # - OpenSSL +# # - libssh2 +# # Needed by: +# # - php +ENV VERSION_CURL=7.85.0 +ENV CURL_BUILD_DIR=${BUILD_DIR}/curl +RUN set -xe; \ + mkdir -p ${CURL_BUILD_DIR}/bin; \ + curl -Ls https://github.com/curl/curl/archive/curl-${VERSION_CURL//./_}.tar.gz \ + | tar xzC ${CURL_BUILD_DIR} --strip-components=1 +WORKDIR ${CURL_BUILD_DIR}/ +RUN ./buildconf \ + && CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./configure \ + --prefix=${INSTALL_DIR} \ + --with-ca-bundle=${CA_BUNDLE} \ + --enable-shared \ + --disable-static \ + --enable-optimize \ + --disable-warnings \ + --disable-dependency-tracking \ + --with-zlib \ + --enable-http \ + --enable-ftp \ + --enable-file \ + --enable-proxy \ + --enable-tftp \ + --enable-ipv6 \ + --enable-openssl-auto-load-config \ + --enable-cookies \ + --with-gnu-ld \ + --with-ssl \ + --with-libssh2 \ + --with-nghttp2 +RUN make install + + +############################################################################### +# LIBXML2 +# https://github.com/GNOME/libxml2/releases +# Uses: +# - zlib +# Needed by: +# - php +ENV VERSION_XML2=2.10.3 +ENV XML2_BUILD_DIR=${BUILD_DIR}/xml2 +RUN set -xe; \ + mkdir -p ${XML2_BUILD_DIR}; \ + curl -Ls https://download.gnome.org/sources/libxml2/${VERSION_XML2%.*}/libxml2-${VERSION_XML2}.tar.xz \ + | tar xJC ${XML2_BUILD_DIR} --strip-components=1 +WORKDIR ${XML2_BUILD_DIR}/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./configure \ + --prefix=${INSTALL_DIR} \ + --with-sysroot=${INSTALL_DIR} \ + --enable-shared \ + --disable-static \ + --with-html \ + --with-history \ + --enable-ipv6=no \ + --with-icu \ + --with-zlib \ + --without-python +RUN make install \ + && cp xml2-config ${INSTALL_DIR}/bin/xml2-config + + +############################################################################### +# LIBZIP +# https://github.com/nih-at/libzip/releases +# Needed by: +# - php +ENV VERSION_ZIP=1.9.2 +ENV ZIP_BUILD_DIR=${BUILD_DIR}/zip +RUN set -xe; \ + mkdir -p ${ZIP_BUILD_DIR}/bin/; \ + curl -Ls https://github.com/nih-at/libzip/releases/download/v${VERSION_ZIP}/libzip-${VERSION_ZIP}.tar.gz \ + | tar xzC ${ZIP_BUILD_DIR} --strip-components=1 +WORKDIR ${ZIP_BUILD_DIR}/bin/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + cmake .. \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE=RELEASE +RUN cmake --build . --target install + + +############################################################################### +# LIBSODIUM +# https://github.com/jedisct1/libsodium/releases +# Needed by: +# - php +ENV VERSION_LIBSODIUM=1.0.18 +ENV LIBSODIUM_BUILD_DIR=${BUILD_DIR}/libsodium +RUN set -xe; \ + mkdir -p ${LIBSODIUM_BUILD_DIR}; \ + curl -Ls https://github.com/jedisct1/libsodium/archive/${VERSION_LIBSODIUM}.tar.gz \ + | tar xzC ${LIBSODIUM_BUILD_DIR} --strip-components=1 +WORKDIR ${LIBSODIUM_BUILD_DIR}/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./autogen.sh \ +&& ./configure --prefix=${INSTALL_DIR} +RUN make install + + +############################################################################### +# Postgres +# https://github.com/postgres/postgres/releases +# Needs: +# - OpenSSL +# Needed by: +# - php +ENV VERSION_POSTGRES=15.1 +ENV POSTGRES_BUILD_DIR=${BUILD_DIR}/postgres +RUN set -xe; \ + mkdir -p ${POSTGRES_BUILD_DIR}/bin; \ + curl -Ls https://github.com/postgres/postgres/archive/REL_${VERSION_POSTGRES//./_}.tar.gz \ + | tar xzC ${POSTGRES_BUILD_DIR} --strip-components=1 +WORKDIR ${POSTGRES_BUILD_DIR}/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./configure --prefix=${INSTALL_DIR} --with-openssl --without-readline +RUN cd ${POSTGRES_BUILD_DIR}/src/interfaces/libpq && make && make install +RUN cd ${POSTGRES_BUILD_DIR}/src/bin/pg_config && make && make install +RUN cd ${POSTGRES_BUILD_DIR}/src/backend && make generated-headers +RUN cd ${POSTGRES_BUILD_DIR}/src/include && make install + + +############################################################################### +# Oniguruma +# This library is not packaged in PHP since PHP 7.4. +# See https://github.com/php/php-src/blob/43dc7da8e3719d3e89bd8ec15ebb13f997bbbaa9/UPGRADING#L578-L581 +# We do not install the system version because I didn't manage to make it work... +# Ideally we shouldn't compile it ourselves. +# https://github.com/kkos/oniguruma/releases +# Needed by: +# - php mbstring +ENV VERSION_ONIG=6.9.8 +ENV ONIG_BUILD_DIR=${BUILD_DIR}/oniguruma +RUN set -xe; \ + mkdir -p ${ONIG_BUILD_DIR}; \ + curl -Ls https://github.com/kkos/oniguruma/releases/download/v${VERSION_ONIG}/onig-${VERSION_ONIG}.tar.gz \ + | tar xzC ${ONIG_BUILD_DIR} --strip-components=1 +WORKDIR ${ONIG_BUILD_DIR} +RUN ./configure --prefix=${INSTALL_DIR} +RUN make && make install + + +############################################################################### +# Install some dev files for using old libraries already on the system +# readline-devel : needed for the readline extension +# gettext-devel : needed for the --with-gettext flag +# libicu-devel : needed for intl +# libxslt-devel : needed for the XSL extension +# sqlite-devel : Since PHP 7.4 this must be installed (https://github.com/php/php-src/blob/99b8e67615159fc600a615e1e97f2d1cf18f14cb/UPGRADING#L616-L619) +RUN LD_LIBRARY_PATH= yum install -y readline-devel gettext-devel libicu-devel libxslt-devel sqlite-devel + # PHP Build # https://github.com/php/php-src/releases @@ -20,11 +341,14 @@ WORKDIR /tmp/php # - openssl # - readline # - sodium +RUN mkdir -p /tmp/php +WORKDIR /tmp/php # Download and unpack the source code # --location will follow redirects # --silent will hide the progress, but also the errors: we restore error messages with --show-error # --fail makes sure that curl returns an error instead of fetching the 404 page +ARG VERSION_PHP RUN curl --location --silent --show-error --fail https://www.php.net/get/php-${VERSION_PHP}.tar.gz/from/this/mirror \ | tar xzC . --strip-components=1 diff --git a/php-81/Dockerfile b/php-81/Dockerfile index 1a82f244..f34274ad 100644 --- a/php-81/Dockerfile +++ b/php-81/Dockerfile @@ -5,12 +5,333 @@ ARG CPU # Can be "x86_64" or "arm64" ARG IMAGE_VERSION_SUFFIX -FROM bref/base-devel-${CPU} as build-environment +ARG VERSION_PHP=8.1.14 -ENV VERSION_PHP=8.1.14 -RUN mkdir -p /tmp/php -WORKDIR /tmp/php +# Lambda uses a custom AMI named Amazon Linux 2 +# https://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html +# AWS provides a Docker image that we use here: +# https://github.com/amazonlinux/container-images/tree/amzn2 +FROM public.ecr.aws/lambda/provided:al2-${IMAGE_VERSION_SUFFIX} as build-environment + + +# Temp directory in which all compilation happens +WORKDIR /tmp + + +RUN set -xe \ + # Download yum repository data to cache + && yum makecache \ + # Default Development Tools + && yum groupinstall -y "Development Tools" --setopt=group_package_types=mandatory,default + + +# The default version of cmake is 2.8.12. We need cmake to build a few of +# our libraries, and at least one library requires a version of cmake greater than that. +# Needed to build: +# - libzip: minimum required CMAKE version 3.0. +RUN LD_LIBRARY_PATH= yum install -y cmake3 +# Override the default `cmake` +RUN ln -s /usr/bin/cmake3 /usr/bin/cmake + +# Use the bash shell, instead of /bin/sh +# Why? We need to document this. +SHELL ["/bin/bash", "-c"] + +# We need a base path for all the sourcecode we will build from. +ENV BUILD_DIR="/tmp/build" + +# Target installation path for all the packages we will compile +ENV INSTALL_DIR="/tmp/bref" + +# We need some default compiler variables setup +ENV PKG_CONFIG_PATH="${INSTALL_DIR}/lib64/pkgconfig:${INSTALL_DIR}/lib/pkgconfig" \ + PKG_CONFIG="/usr/bin/pkg-config" \ + PATH="${INSTALL_DIR}/bin:${PATH}" + +ENV LD_LIBRARY_PATH="${INSTALL_DIR}/lib64:${INSTALL_DIR}/lib" + +# Enable parallelism by default for make and cmake (like make -j) +# See https://stackoverflow.com/a/50883540/245552 +ENV CMAKE_BUILD_PARALLEL_LEVEL=4 +ENV MAKEFLAGS='-j4' + +# Ensure we have all the directories we require in the container. +RUN mkdir -p ${BUILD_DIR} \ + ${INSTALL_DIR}/bin \ + ${INSTALL_DIR}/doc \ + ${INSTALL_DIR}/etc/php \ + ${INSTALL_DIR}/etc/php/conf.d \ + ${INSTALL_DIR}/include \ + ${INSTALL_DIR}/lib \ + ${INSTALL_DIR}/lib64 \ + ${INSTALL_DIR}/libexec \ + ${INSTALL_DIR}/sbin \ + ${INSTALL_DIR}/share + + +############################################################################### +# OPENSSL +# https://github.com/openssl/openssl/releases +# Needs: +# - zlib +# Needed by: +# - curl +# - php +ENV VERSION_OPENSSL=1.1.1s +ENV OPENSSL_BUILD_DIR=${BUILD_DIR}/openssl +ENV CA_BUNDLE_SOURCE="https://curl.se/ca/cacert.pem" +ENV CA_BUNDLE="${INSTALL_DIR}/ssl/cert.pem" +RUN set -xe; \ + mkdir -p ${OPENSSL_BUILD_DIR}; \ + curl -Ls https://github.com/openssl/openssl/archive/OpenSSL_${VERSION_OPENSSL//./_}.tar.gz \ + | tar xzC ${OPENSSL_BUILD_DIR} --strip-components=1 +WORKDIR ${OPENSSL_BUILD_DIR}/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./config \ + --prefix=${INSTALL_DIR} \ + --openssldir=${INSTALL_DIR}/ssl \ + --release \ + no-tests \ + shared \ + zlib +# Explicitly compile make without parallelism because it fails if we use -jX (no error message) +# I'm not 100% sure why, and I already lost 4 hours on this, but I found this: +# https://github.com/openssl/openssl/issues/9931 +# https://stackoverflow.com/questions/28639207/why-cant-i-compile-openssl-with-multiple-threads-make-j3 +# Run `make install_sw install_ssldirs` instead of `make install` to skip installing man pages https://github.com/openssl/openssl/issues/8170 +RUN make -j1 install_sw install_ssldirs +RUN curl -Lk -o ${CA_BUNDLE} ${CA_BUNDLE_SOURCE} + + +############################################################################### +# LIBSSH2 +# https://github.com/libssh2/libssh2/releases +# Needs: +# - zlib +# - OpenSSL +# Needed by: +# - curl +ENV VERSION_LIBSSH2=1.10.0 +ENV LIBSSH2_BUILD_DIR=${BUILD_DIR}/libssh2 +RUN set -xe; \ + mkdir -p ${LIBSSH2_BUILD_DIR}/bin; \ + curl -Ls https://github.com/libssh2/libssh2/releases/download/libssh2-${VERSION_LIBSSH2}/libssh2-${VERSION_LIBSSH2}.tar.gz \ + | tar xzC ${LIBSSH2_BUILD_DIR} --strip-components=1 +WORKDIR ${LIBSSH2_BUILD_DIR}/bin/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + cmake .. \ + # Build as a shared library (.so) instead of a static one + -DBUILD_SHARED_LIBS=ON \ + # Build with OpenSSL support + -DCRYPTO_BACKEND=OpenSSL \ + # Build with zlib support + -DENABLE_ZLIB_COMPRESSION=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE=RELEASE +RUN cmake --build . --target install + + +############################################################################### +# LIBNGHTTP2 +# This adds support for HTTP 2 requests in curl. +# See https://github.com/brefphp/bref/issues/727 and https://github.com/brefphp/bref/pull/740 +# https://github.com/nghttp2/nghttp2/releases +# Needs: +# - zlib +# - OpenSSL +# Needed by: +# - curl +ENV VERSION_NGHTTP2=1.51.0 +ENV NGHTTP2_BUILD_DIR=${BUILD_DIR}/nghttp2 +RUN set -xe; \ + mkdir -p ${NGHTTP2_BUILD_DIR}; \ + curl -Ls https://github.com/nghttp2/nghttp2/releases/download/v${VERSION_NGHTTP2}/nghttp2-${VERSION_NGHTTP2}.tar.gz \ + | tar xzC ${NGHTTP2_BUILD_DIR} --strip-components=1 +WORKDIR ${NGHTTP2_BUILD_DIR}/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./configure \ + --enable-lib-only \ + --prefix=${INSTALL_DIR} +RUN make install + + +############################################################################### +# CURL +# # https://github.com/curl/curl/releases +# # Needs: +# # - zlib +# # - OpenSSL +# # - libssh2 +# # Needed by: +# # - php +ENV VERSION_CURL=7.85.0 +ENV CURL_BUILD_DIR=${BUILD_DIR}/curl +RUN set -xe; \ + mkdir -p ${CURL_BUILD_DIR}/bin; \ + curl -Ls https://github.com/curl/curl/archive/curl-${VERSION_CURL//./_}.tar.gz \ + | tar xzC ${CURL_BUILD_DIR} --strip-components=1 +WORKDIR ${CURL_BUILD_DIR}/ +RUN ./buildconf \ + && CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./configure \ + --prefix=${INSTALL_DIR} \ + --with-ca-bundle=${CA_BUNDLE} \ + --enable-shared \ + --disable-static \ + --enable-optimize \ + --disable-warnings \ + --disable-dependency-tracking \ + --with-zlib \ + --enable-http \ + --enable-ftp \ + --enable-file \ + --enable-proxy \ + --enable-tftp \ + --enable-ipv6 \ + --enable-openssl-auto-load-config \ + --enable-cookies \ + --with-gnu-ld \ + --with-ssl \ + --with-libssh2 \ + --with-nghttp2 +RUN make install + + +############################################################################### +# LIBXML2 +# https://github.com/GNOME/libxml2/releases +# Uses: +# - zlib +# Needed by: +# - php +ENV VERSION_XML2=2.10.3 +ENV XML2_BUILD_DIR=${BUILD_DIR}/xml2 +RUN set -xe; \ + mkdir -p ${XML2_BUILD_DIR}; \ + curl -Ls https://download.gnome.org/sources/libxml2/${VERSION_XML2%.*}/libxml2-${VERSION_XML2}.tar.xz \ + | tar xJC ${XML2_BUILD_DIR} --strip-components=1 +WORKDIR ${XML2_BUILD_DIR}/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./configure \ + --prefix=${INSTALL_DIR} \ + --with-sysroot=${INSTALL_DIR} \ + --enable-shared \ + --disable-static \ + --with-html \ + --with-history \ + --enable-ipv6=no \ + --with-icu \ + --with-zlib \ + --without-python +RUN make install \ + && cp xml2-config ${INSTALL_DIR}/bin/xml2-config + + +############################################################################### +# LIBZIP +# https://github.com/nih-at/libzip/releases +# Needed by: +# - php +ENV VERSION_ZIP=1.9.2 +ENV ZIP_BUILD_DIR=${BUILD_DIR}/zip +RUN set -xe; \ + mkdir -p ${ZIP_BUILD_DIR}/bin/; \ + curl -Ls https://github.com/nih-at/libzip/releases/download/v${VERSION_ZIP}/libzip-${VERSION_ZIP}.tar.gz \ + | tar xzC ${ZIP_BUILD_DIR} --strip-components=1 +WORKDIR ${ZIP_BUILD_DIR}/bin/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + cmake .. \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE=RELEASE +RUN cmake --build . --target install + + +############################################################################### +# LIBSODIUM +# https://github.com/jedisct1/libsodium/releases +# Needed by: +# - php +ENV VERSION_LIBSODIUM=1.0.18 +ENV LIBSODIUM_BUILD_DIR=${BUILD_DIR}/libsodium +RUN set -xe; \ + mkdir -p ${LIBSODIUM_BUILD_DIR}; \ + curl -Ls https://github.com/jedisct1/libsodium/archive/${VERSION_LIBSODIUM}.tar.gz \ + | tar xzC ${LIBSODIUM_BUILD_DIR} --strip-components=1 +WORKDIR ${LIBSODIUM_BUILD_DIR}/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./autogen.sh \ +&& ./configure --prefix=${INSTALL_DIR} +RUN make install + + +############################################################################### +# Postgres +# https://github.com/postgres/postgres/releases +# Needs: +# - OpenSSL +# Needed by: +# - php +ENV VERSION_POSTGRES=15.1 +ENV POSTGRES_BUILD_DIR=${BUILD_DIR}/postgres +RUN set -xe; \ + mkdir -p ${POSTGRES_BUILD_DIR}/bin; \ + curl -Ls https://github.com/postgres/postgres/archive/REL_${VERSION_POSTGRES//./_}.tar.gz \ + | tar xzC ${POSTGRES_BUILD_DIR} --strip-components=1 +WORKDIR ${POSTGRES_BUILD_DIR}/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./configure --prefix=${INSTALL_DIR} --with-openssl --without-readline +RUN cd ${POSTGRES_BUILD_DIR}/src/interfaces/libpq && make && make install +RUN cd ${POSTGRES_BUILD_DIR}/src/bin/pg_config && make && make install +RUN cd ${POSTGRES_BUILD_DIR}/src/backend && make generated-headers +RUN cd ${POSTGRES_BUILD_DIR}/src/include && make install + + +############################################################################### +# Oniguruma +# This library is not packaged in PHP since PHP 7.4. +# See https://github.com/php/php-src/blob/43dc7da8e3719d3e89bd8ec15ebb13f997bbbaa9/UPGRADING#L578-L581 +# We do not install the system version because I didn't manage to make it work... +# Ideally we shouldn't compile it ourselves. +# https://github.com/kkos/oniguruma/releases +# Needed by: +# - php mbstring +ENV VERSION_ONIG=6.9.8 +ENV ONIG_BUILD_DIR=${BUILD_DIR}/oniguruma +RUN set -xe; \ + mkdir -p ${ONIG_BUILD_DIR}; \ + curl -Ls https://github.com/kkos/oniguruma/releases/download/v${VERSION_ONIG}/onig-${VERSION_ONIG}.tar.gz \ + | tar xzC ${ONIG_BUILD_DIR} --strip-components=1 +WORKDIR ${ONIG_BUILD_DIR} +RUN ./configure --prefix=${INSTALL_DIR} +RUN make && make install + + +############################################################################### +# Install some dev files for using old libraries already on the system +# readline-devel : needed for the readline extension +# gettext-devel : needed for the --with-gettext flag +# libicu-devel : needed for intl +# libxslt-devel : needed for the XSL extension +# sqlite-devel : Since PHP 7.4 this must be installed (https://github.com/php/php-src/blob/99b8e67615159fc600a615e1e97f2d1cf18f14cb/UPGRADING#L616-L619) +RUN LD_LIBRARY_PATH= yum install -y readline-devel gettext-devel libicu-devel libxslt-devel sqlite-devel + # PHP Build # https://github.com/php/php-src/releases @@ -20,11 +341,14 @@ WORKDIR /tmp/php # - openssl # - readline # - sodium +RUN mkdir -p /tmp/php +WORKDIR /tmp/php # Download and unpack the source code # --location will follow redirects # --silent will hide the progress, but also the errors: we restore error messages with --show-error # --fail makes sure that curl returns an error instead of fetching the 404 page +ARG VERSION_PHP RUN curl --location --silent --show-error --fail https://www.php.net/get/php-${VERSION_PHP}.tar.gz/from/this/mirror \ | tar xzC . --strip-components=1 diff --git a/php-82/Dockerfile b/php-82/Dockerfile index 0c3b02d0..dfc16003 100644 --- a/php-82/Dockerfile +++ b/php-82/Dockerfile @@ -5,12 +5,333 @@ ARG CPU # Can be "x86_64" or "arm64" ARG IMAGE_VERSION_SUFFIX -FROM bref/base-devel-${CPU} as build-environment +ARG VERSION_PHP=8.2.0 -ENV VERSION_PHP=8.2.0 -RUN mkdir -p /tmp/php -WORKDIR /tmp/php +# Lambda uses a custom AMI named Amazon Linux 2 +# https://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html +# AWS provides a Docker image that we use here: +# https://github.com/amazonlinux/container-images/tree/amzn2 +FROM public.ecr.aws/lambda/provided:al2-${IMAGE_VERSION_SUFFIX} as build-environment + + +# Temp directory in which all compilation happens +WORKDIR /tmp + + +RUN set -xe \ + # Download yum repository data to cache + && yum makecache \ + # Default Development Tools + && yum groupinstall -y "Development Tools" --setopt=group_package_types=mandatory,default + + +# The default version of cmake is 2.8.12. We need cmake to build a few of +# our libraries, and at least one library requires a version of cmake greater than that. +# Needed to build: +# - libzip: minimum required CMAKE version 3.0. +RUN LD_LIBRARY_PATH= yum install -y cmake3 +# Override the default `cmake` +RUN ln -s /usr/bin/cmake3 /usr/bin/cmake + +# Use the bash shell, instead of /bin/sh +# Why? We need to document this. +SHELL ["/bin/bash", "-c"] + +# We need a base path for all the sourcecode we will build from. +ENV BUILD_DIR="/tmp/build" + +# Target installation path for all the packages we will compile +ENV INSTALL_DIR="/tmp/bref" + +# We need some default compiler variables setup +ENV PKG_CONFIG_PATH="${INSTALL_DIR}/lib64/pkgconfig:${INSTALL_DIR}/lib/pkgconfig" \ + PKG_CONFIG="/usr/bin/pkg-config" \ + PATH="${INSTALL_DIR}/bin:${PATH}" + +ENV LD_LIBRARY_PATH="${INSTALL_DIR}/lib64:${INSTALL_DIR}/lib" + +# Enable parallelism by default for make and cmake (like make -j) +# See https://stackoverflow.com/a/50883540/245552 +ENV CMAKE_BUILD_PARALLEL_LEVEL=4 +ENV MAKEFLAGS='-j4' + +# Ensure we have all the directories we require in the container. +RUN mkdir -p ${BUILD_DIR} \ + ${INSTALL_DIR}/bin \ + ${INSTALL_DIR}/doc \ + ${INSTALL_DIR}/etc/php \ + ${INSTALL_DIR}/etc/php/conf.d \ + ${INSTALL_DIR}/include \ + ${INSTALL_DIR}/lib \ + ${INSTALL_DIR}/lib64 \ + ${INSTALL_DIR}/libexec \ + ${INSTALL_DIR}/sbin \ + ${INSTALL_DIR}/share + + +############################################################################### +# OPENSSL +# https://github.com/openssl/openssl/releases +# Needs: +# - zlib +# Needed by: +# - curl +# - php +ENV VERSION_OPENSSL=1.1.1s +ENV OPENSSL_BUILD_DIR=${BUILD_DIR}/openssl +ENV CA_BUNDLE_SOURCE="https://curl.se/ca/cacert.pem" +ENV CA_BUNDLE="${INSTALL_DIR}/ssl/cert.pem" +RUN set -xe; \ + mkdir -p ${OPENSSL_BUILD_DIR}; \ + curl -Ls https://github.com/openssl/openssl/archive/OpenSSL_${VERSION_OPENSSL//./_}.tar.gz \ + | tar xzC ${OPENSSL_BUILD_DIR} --strip-components=1 +WORKDIR ${OPENSSL_BUILD_DIR}/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./config \ + --prefix=${INSTALL_DIR} \ + --openssldir=${INSTALL_DIR}/ssl \ + --release \ + no-tests \ + shared \ + zlib +# Explicitly compile make without parallelism because it fails if we use -jX (no error message) +# I'm not 100% sure why, and I already lost 4 hours on this, but I found this: +# https://github.com/openssl/openssl/issues/9931 +# https://stackoverflow.com/questions/28639207/why-cant-i-compile-openssl-with-multiple-threads-make-j3 +# Run `make install_sw install_ssldirs` instead of `make install` to skip installing man pages https://github.com/openssl/openssl/issues/8170 +RUN make -j1 install_sw install_ssldirs +RUN curl -Lk -o ${CA_BUNDLE} ${CA_BUNDLE_SOURCE} + + +############################################################################### +# LIBSSH2 +# https://github.com/libssh2/libssh2/releases +# Needs: +# - zlib +# - OpenSSL +# Needed by: +# - curl +ENV VERSION_LIBSSH2=1.10.0 +ENV LIBSSH2_BUILD_DIR=${BUILD_DIR}/libssh2 +RUN set -xe; \ + mkdir -p ${LIBSSH2_BUILD_DIR}/bin; \ + curl -Ls https://github.com/libssh2/libssh2/releases/download/libssh2-${VERSION_LIBSSH2}/libssh2-${VERSION_LIBSSH2}.tar.gz \ + | tar xzC ${LIBSSH2_BUILD_DIR} --strip-components=1 +WORKDIR ${LIBSSH2_BUILD_DIR}/bin/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + cmake .. \ + # Build as a shared library (.so) instead of a static one + -DBUILD_SHARED_LIBS=ON \ + # Build with OpenSSL support + -DCRYPTO_BACKEND=OpenSSL \ + # Build with zlib support + -DENABLE_ZLIB_COMPRESSION=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE=RELEASE +RUN cmake --build . --target install + + +############################################################################### +# LIBNGHTTP2 +# This adds support for HTTP 2 requests in curl. +# See https://github.com/brefphp/bref/issues/727 and https://github.com/brefphp/bref/pull/740 +# https://github.com/nghttp2/nghttp2/releases +# Needs: +# - zlib +# - OpenSSL +# Needed by: +# - curl +ENV VERSION_NGHTTP2=1.51.0 +ENV NGHTTP2_BUILD_DIR=${BUILD_DIR}/nghttp2 +RUN set -xe; \ + mkdir -p ${NGHTTP2_BUILD_DIR}; \ + curl -Ls https://github.com/nghttp2/nghttp2/releases/download/v${VERSION_NGHTTP2}/nghttp2-${VERSION_NGHTTP2}.tar.gz \ + | tar xzC ${NGHTTP2_BUILD_DIR} --strip-components=1 +WORKDIR ${NGHTTP2_BUILD_DIR}/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./configure \ + --enable-lib-only \ + --prefix=${INSTALL_DIR} +RUN make install + + +############################################################################### +# CURL +# # https://github.com/curl/curl/releases +# # Needs: +# # - zlib +# # - OpenSSL +# # - libssh2 +# # Needed by: +# # - php +ENV VERSION_CURL=7.85.0 +ENV CURL_BUILD_DIR=${BUILD_DIR}/curl +RUN set -xe; \ + mkdir -p ${CURL_BUILD_DIR}/bin; \ + curl -Ls https://github.com/curl/curl/archive/curl-${VERSION_CURL//./_}.tar.gz \ + | tar xzC ${CURL_BUILD_DIR} --strip-components=1 +WORKDIR ${CURL_BUILD_DIR}/ +RUN ./buildconf \ + && CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./configure \ + --prefix=${INSTALL_DIR} \ + --with-ca-bundle=${CA_BUNDLE} \ + --enable-shared \ + --disable-static \ + --enable-optimize \ + --disable-warnings \ + --disable-dependency-tracking \ + --with-zlib \ + --enable-http \ + --enable-ftp \ + --enable-file \ + --enable-proxy \ + --enable-tftp \ + --enable-ipv6 \ + --enable-openssl-auto-load-config \ + --enable-cookies \ + --with-gnu-ld \ + --with-ssl \ + --with-libssh2 \ + --with-nghttp2 +RUN make install + + +############################################################################### +# LIBXML2 +# https://github.com/GNOME/libxml2/releases +# Uses: +# - zlib +# Needed by: +# - php +ENV VERSION_XML2=2.10.3 +ENV XML2_BUILD_DIR=${BUILD_DIR}/xml2 +RUN set -xe; \ + mkdir -p ${XML2_BUILD_DIR}; \ + curl -Ls https://download.gnome.org/sources/libxml2/${VERSION_XML2%.*}/libxml2-${VERSION_XML2}.tar.xz \ + | tar xJC ${XML2_BUILD_DIR} --strip-components=1 +WORKDIR ${XML2_BUILD_DIR}/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./configure \ + --prefix=${INSTALL_DIR} \ + --with-sysroot=${INSTALL_DIR} \ + --enable-shared \ + --disable-static \ + --with-html \ + --with-history \ + --enable-ipv6=no \ + --with-icu \ + --with-zlib \ + --without-python +RUN make install \ + && cp xml2-config ${INSTALL_DIR}/bin/xml2-config + + +############################################################################### +# LIBZIP +# https://github.com/nih-at/libzip/releases +# Needed by: +# - php +ENV VERSION_ZIP=1.9.2 +ENV ZIP_BUILD_DIR=${BUILD_DIR}/zip +RUN set -xe; \ + mkdir -p ${ZIP_BUILD_DIR}/bin/; \ + curl -Ls https://github.com/nih-at/libzip/releases/download/v${VERSION_ZIP}/libzip-${VERSION_ZIP}.tar.gz \ + | tar xzC ${ZIP_BUILD_DIR} --strip-components=1 +WORKDIR ${ZIP_BUILD_DIR}/bin/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + cmake .. \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_BUILD_TYPE=RELEASE +RUN cmake --build . --target install + + +############################################################################### +# LIBSODIUM +# https://github.com/jedisct1/libsodium/releases +# Needed by: +# - php +ENV VERSION_LIBSODIUM=1.0.18 +ENV LIBSODIUM_BUILD_DIR=${BUILD_DIR}/libsodium +RUN set -xe; \ + mkdir -p ${LIBSODIUM_BUILD_DIR}; \ + curl -Ls https://github.com/jedisct1/libsodium/archive/${VERSION_LIBSODIUM}.tar.gz \ + | tar xzC ${LIBSODIUM_BUILD_DIR} --strip-components=1 +WORKDIR ${LIBSODIUM_BUILD_DIR}/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./autogen.sh \ +&& ./configure --prefix=${INSTALL_DIR} +RUN make install + + +############################################################################### +# Postgres +# https://github.com/postgres/postgres/releases +# Needs: +# - OpenSSL +# Needed by: +# - php +ENV VERSION_POSTGRES=15.1 +ENV POSTGRES_BUILD_DIR=${BUILD_DIR}/postgres +RUN set -xe; \ + mkdir -p ${POSTGRES_BUILD_DIR}/bin; \ + curl -Ls https://github.com/postgres/postgres/archive/REL_${VERSION_POSTGRES//./_}.tar.gz \ + | tar xzC ${POSTGRES_BUILD_DIR} --strip-components=1 +WORKDIR ${POSTGRES_BUILD_DIR}/ +RUN CFLAGS="" \ + CPPFLAGS="-I${INSTALL_DIR}/include -I/usr/include" \ + LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib" \ + ./configure --prefix=${INSTALL_DIR} --with-openssl --without-readline +RUN cd ${POSTGRES_BUILD_DIR}/src/interfaces/libpq && make && make install +RUN cd ${POSTGRES_BUILD_DIR}/src/bin/pg_config && make && make install +RUN cd ${POSTGRES_BUILD_DIR}/src/backend && make generated-headers +RUN cd ${POSTGRES_BUILD_DIR}/src/include && make install + + +############################################################################### +# Oniguruma +# This library is not packaged in PHP since PHP 7.4. +# See https://github.com/php/php-src/blob/43dc7da8e3719d3e89bd8ec15ebb13f997bbbaa9/UPGRADING#L578-L581 +# We do not install the system version because I didn't manage to make it work... +# Ideally we shouldn't compile it ourselves. +# https://github.com/kkos/oniguruma/releases +# Needed by: +# - php mbstring +ENV VERSION_ONIG=6.9.8 +ENV ONIG_BUILD_DIR=${BUILD_DIR}/oniguruma +RUN set -xe; \ + mkdir -p ${ONIG_BUILD_DIR}; \ + curl -Ls https://github.com/kkos/oniguruma/releases/download/v${VERSION_ONIG}/onig-${VERSION_ONIG}.tar.gz \ + | tar xzC ${ONIG_BUILD_DIR} --strip-components=1 +WORKDIR ${ONIG_BUILD_DIR} +RUN ./configure --prefix=${INSTALL_DIR} +RUN make && make install + + +############################################################################### +# Install some dev files for using old libraries already on the system +# readline-devel : needed for the readline extension +# gettext-devel : needed for the --with-gettext flag +# libicu-devel : needed for intl +# libxslt-devel : needed for the XSL extension +# sqlite-devel : Since PHP 7.4 this must be installed (https://github.com/php/php-src/blob/99b8e67615159fc600a615e1e97f2d1cf18f14cb/UPGRADING#L616-L619) +RUN LD_LIBRARY_PATH= yum install -y readline-devel gettext-devel libicu-devel libxslt-devel sqlite-devel + # PHP Build # https://github.com/php/php-src/releases @@ -20,11 +341,14 @@ WORKDIR /tmp/php # - openssl # - readline # - sodium +RUN mkdir -p /tmp/php +WORKDIR /tmp/php # Download and unpack the source code # --location will follow redirects # --silent will hide the progress, but also the errors: we restore error messages with --show-error # --fail makes sure that curl returns an error instead of fetching the 404 page +ARG VERSION_PHP RUN curl --location --silent --show-error --fail https://www.php.net/get/php-${VERSION_PHP}.tar.gz/from/this/mirror \ | tar xzC . --strip-components=1 From df928eec622ab1aacd6f5c7023458915dbb32054 Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Mon, 23 Jan 2023 16:29:56 +0100 Subject: [PATCH 03/17] Inline the `bref/fpm-internal-src` image into PHP Dockerfiles This will help to solve build dependency issues and image visibility with Depot.dev. The added complexity (duplicated code) is very minor, and Docker will cache the build steps anyway, so I find that acceptable. --- .github/workflows/release.yml | 14 ++++++++++++++ README.md | 2 -- cpu-arm.Makefile | 8 +------- cpu-x86.Makefile | 8 +------- docker-bake.hcl | 6 ------ layers/docker-compose.yml | 6 ------ layers/fpm/Dockerfile | 9 --------- layers/fpm/composer.json | 2 +- php-80/Dockerfile | 12 +++++++++++- php-81/Dockerfile | 12 +++++++++++- php-82/Dockerfile | 12 +++++++++++- 11 files changed, 50 insertions(+), 41 deletions(-) delete mode 100644 layers/docker-compose.yml delete mode 100644 layers/fpm/Dockerfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ed7cd77b..dc1d8d47 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,17 +20,31 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + + - uses: depot/setup-action@v1 + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: arn:aws:iam::534081306603:role/bref-layer-publisher-github-actions role-session-name: bref-layer-publisher-github-actions aws-region: us-east-1 + - name: Configure Docker Hub credentials uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build Docker images + run: make -f cpu-${{ matrix.cpu }}.Makefile docker-images + env: + DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }} + PHP_VERSION: ${{ matrix.php_version }} + CPU: ${{ matrix.cpu }} + CPU_PREFIX: ${{ (matrix.cpu == 'arm') && 'arm-' || '' }} + IMAGE_VERSION_SUFFIX: ${{ (matrix.cpu == 'arm') && 'arm64' || 'x86_64' }} + - run: make -f cpu-x86.Makefile layers - run: make -f cpu-x86.Makefile test - run: make -f cpu-x86.Makefile upload-layers diff --git a/README.md b/README.md index ac3cd164..06694675 100644 --- a/README.md +++ b/README.md @@ -174,8 +174,6 @@ The 2nd layer is the `isolation` layer where we'll start from the standard AWS-p copied here as well. The 3rd layer is the `function` layer where everything is packet together and the `bootstrap` file is loaded. -The `bref-internal-src` images (see layers/fpm) are used to load Bref -classes into the layer. The 4th layer is `zip-function`, where we get a small and fast Linux (Alpine) just to install and zip the entire `/opt` content. We use docker compose volumes to map `/tmp/bref-zip` from host to the container so that we can diff --git a/cpu-arm.Makefile b/cpu-arm.Makefile index 8cd4041b..de5a25f6 100644 --- a/cpu-arm.Makefile +++ b/cpu-arm.Makefile @@ -13,12 +13,7 @@ default: docker-images layers # Build Docker images *locally* docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 -docker-image-fpm-internal-src: - depot build \ - --load \ - --tag=bref/fpm-internal-src \ - layers/fpm -docker-images-php-%: docker-image-fpm-internal-src +docker-images-php-%: # build depot build \ --platform=linux/arm64 \ @@ -114,7 +109,6 @@ clean: # Remove zip files rm -f output/arm-*.zip # Clean Docker images to force rebuilding them - docker image rm --force bref/arm-fpm-internal-src docker image rm --force bref/arm-build-php-80 docker image rm --force bref/arm-build-php-81 docker image rm --force bref/arm-build-php-82 diff --git a/cpu-x86.Makefile b/cpu-x86.Makefile index cb58bf20..b5e5aea1 100644 --- a/cpu-x86.Makefile +++ b/cpu-x86.Makefile @@ -13,12 +13,7 @@ default: docker-images layers # Build Docker images *locally* docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 -docker-image-fpm-internal-src: - depot build \ - --load \ - --tag=bref/fpm-internal-src \ - layers/fpm -docker-images-php-%: docker-image-fpm-internal-src +docker-images-php-%: # build depot build \ --platform=linux/amd64 \ @@ -119,7 +114,6 @@ clean: # Remove zip files rm -f output/*.zip # Clean Docker images to force rebuilding them - docker image rm --force bref/fpm-internal-src docker image rm --force bref/build-php-80 docker image rm --force bref/build-php-81 docker image rm --force bref/build-php-82 diff --git a/docker-bake.hcl b/docker-bake.hcl index da088c07..a01196f1 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -38,11 +38,6 @@ target "php" { } } -target "fpm-internal-src" { - context = "layers/fpm" - tags = ["bref/fpm-internal-src"] -} - target "php-fpm" { dockerfile = "php-${PHP_VERSION}/Dockerfile" target = "fpm" @@ -54,7 +49,6 @@ target "php-fpm" { contexts = { "bref/${CPU_PREFIX}build-php-${PHP_VERSION}" = "target:build-php" "bref/${CPU_PREFIX}php-${PHP_VERSION}" = "target:php" - "bref/fpm-internal-src" = "target:fpm-internal-src" } } diff --git a/layers/docker-compose.yml b/layers/docker-compose.yml deleted file mode 100644 index 1a7c518c..00000000 --- a/layers/docker-compose.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: '3.8' - -services: - fpm-pkg: - image: bref/fpm-internal-src - build: ./fpm diff --git a/layers/fpm/Dockerfile b/layers/fpm/Dockerfile deleted file mode 100644 index 59218421..00000000 --- a/layers/fpm/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM alpine:3.14 - -RUN apk add composer - -RUN mkdir -p /opt/bref/php-fpm-runtime -WORKDIR /opt/bref/php-fpm-runtime - -COPY composer.json composer.json -RUN composer install --ignore-platform-req=ext-posix --ignore-platform-req=ext-simplexml diff --git a/layers/fpm/composer.json b/layers/fpm/composer.json index 8fb399d2..2b2e25f4 100644 --- a/layers/fpm/composer.json +++ b/layers/fpm/composer.json @@ -1,6 +1,6 @@ { "require": { - "bref/php-fpm-runtime": "^2" + "bref/php-fpm-runtime": "2.0.0" }, "minimum-stability": "dev", "prefer-stable" : true, diff --git a/php-80/Dockerfile b/php-80/Dockerfile index ab39fec5..e302a39c 100644 --- a/php-80/Dockerfile +++ b/php-80/Dockerfile @@ -465,6 +465,16 @@ RUN cp ${INSTALL_DIR}/sbin/php-fpm /opt/bin/php-fpm RUN php /bref/lib-copy/copy-dependencies.php /opt/bin/php-fpm /opt/lib +# Embed the https://github.com/brefphp/php-fpm-runtime in the layer +FROM composer:2 as fpm-runtime + +RUN mkdir -p /opt/bref/php-fpm-runtime +WORKDIR /opt/bref/php-fpm-runtime + +COPY --link layers/fpm/composer.json composer.json +RUN composer install --ignore-platform-req=ext-posix --ignore-platform-req=ext-simplexml + + FROM isolation as fpm COPY --link --from=fpm-extension /opt /opt @@ -478,4 +488,4 @@ RUN chmod +x /opt/bootstrap && chmod +x /var/runtime/bootstrap COPY --link layers/fpm/php-fpm.conf /opt/bref/etc/php-fpm.conf -COPY --link --from=bref/fpm-internal-src /opt/bref/php-fpm-runtime /opt/bref/php-fpm-runtime +COPY --link --from=fpm-runtime /opt/bref/php-fpm-runtime /opt/bref/php-fpm-runtime diff --git a/php-81/Dockerfile b/php-81/Dockerfile index f34274ad..923ef247 100644 --- a/php-81/Dockerfile +++ b/php-81/Dockerfile @@ -465,6 +465,16 @@ RUN cp ${INSTALL_DIR}/sbin/php-fpm /opt/bin/php-fpm RUN php /bref/lib-copy/copy-dependencies.php /opt/bin/php-fpm /opt/lib +# Embed the https://github.com/brefphp/php-fpm-runtime in the layer +FROM composer:2 as fpm-runtime + +RUN mkdir -p /opt/bref/php-fpm-runtime +WORKDIR /opt/bref/php-fpm-runtime + +COPY --link layers/fpm/composer.json composer.json +RUN composer install --ignore-platform-req=ext-posix --ignore-platform-req=ext-simplexml + + FROM isolation as fpm COPY --link --from=fpm-extension /opt /opt @@ -478,4 +488,4 @@ RUN chmod +x /opt/bootstrap && chmod +x /var/runtime/bootstrap COPY --link layers/fpm/php-fpm.conf /opt/bref/etc/php-fpm.conf -COPY --link --from=bref/fpm-internal-src /opt/bref/php-fpm-runtime /opt/bref/php-fpm-runtime +COPY --link --from=fpm-runtime /opt/bref/php-fpm-runtime /opt/bref/php-fpm-runtime diff --git a/php-82/Dockerfile b/php-82/Dockerfile index dfc16003..69af8624 100644 --- a/php-82/Dockerfile +++ b/php-82/Dockerfile @@ -465,6 +465,16 @@ RUN cp ${INSTALL_DIR}/sbin/php-fpm /opt/bin/php-fpm RUN php /bref/lib-copy/copy-dependencies.php /opt/bin/php-fpm /opt/lib +# Embed the https://github.com/brefphp/php-fpm-runtime in the layer +FROM composer:2 as fpm-runtime + +RUN mkdir -p /opt/bref/php-fpm-runtime +WORKDIR /opt/bref/php-fpm-runtime + +COPY --link layers/fpm/composer.json composer.json +RUN composer install --ignore-platform-req=ext-posix --ignore-platform-req=ext-simplexml + + FROM isolation as fpm COPY --link --from=fpm-extension /opt /opt @@ -478,4 +488,4 @@ RUN chmod +x /opt/bootstrap && chmod +x /var/runtime/bootstrap COPY --link layers/fpm/php-fpm.conf /opt/bref/etc/php-fpm.conf -COPY --link --from=bref/fpm-internal-src /opt/bref/php-fpm-runtime /opt/bref/php-fpm-runtime +COPY --link --from=fpm-runtime /opt/bref/php-fpm-runtime /opt/bref/php-fpm-runtime From f67bc6c95264d061ae6f95b00946c15d520a5a26 Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Mon, 23 Jan 2023 17:30:38 +0100 Subject: [PATCH 04/17] Use the new `depot bake` command --- .github/workflows/tests.yml | 3 ++- cpu-arm.Makefile | 48 ++----------------------------------- cpu-x86.Makefile | 48 ++----------------------------------- docker-bake.hcl | 8 +++++++ 4 files changed, 14 insertions(+), 93 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 93de790b..c4bfef07 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,13 +27,14 @@ jobs: - uses: depot/setup-action@v1 - name: Build Docker images - run: make -f cpu-${{ matrix.cpu }}.Makefile docker-images + run: depot bake --load env: DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }} PHP_VERSION: ${{ matrix.php_version }} CPU: ${{ matrix.cpu }} CPU_PREFIX: ${{ (matrix.cpu == 'arm') && 'arm-' || '' }} IMAGE_VERSION_SUFFIX: ${{ (matrix.cpu == 'arm') && 'arm64' || 'x86_64' }} + DOCKER_PLATFORM: ${{ (matrix.cpu == 'arm') && 'arm64' || 'amd64' }} - name: Test that layers can be exported run: | diff --git a/cpu-arm.Makefile b/cpu-arm.Makefile index de5a25f6..32b10a16 100644 --- a/cpu-arm.Makefile +++ b/cpu-arm.Makefile @@ -4,6 +4,7 @@ export # export all variables defined in .env export CPU = arm export CPU_PREFIX = arm- export IMAGE_VERSION_SUFFIX = arm64 +export DOCKER_PLATFORM = arm64 # Build all Docker images and layers *locally* @@ -14,52 +15,7 @@ default: docker-images layers # Build Docker images *locally* docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 docker-images-php-%: - # build - depot build \ - --platform=linux/arm64 \ - --build-arg=CPU=${CPU} \ - --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ - --load \ - --tag=bref/${CPU_PREFIX}build-php-$* \ - --file=php-$*/Dockerfile \ - --target=build-environment \ - . - # php - depot build \ - --platform=linux/arm64 \ - --build-arg=CPU=${CPU} \ - --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ - --load \ - --tag=bref/${CPU_PREFIX}php-$* \ - --file=php-$*/Dockerfile \ - --target=function \ - . - # php-fpm - depot build \ - --platform=linux/arm64 \ - --build-arg=CPU=${CPU} \ - --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ - --load \ - --tag=bref/${CPU_PREFIX}php-$*-fpm \ - --file=php-$*/Dockerfile \ - --target=fpm \ - . - # console - depot build \ - --platform=linux/arm64 \ - --build-arg=PHP_VERSION=$* \ - --build-arg=CPU_PREFIX=${CPU_PREFIX} \ - --load \ - --tag=bref/${CPU_PREFIX}php-$*-console \ - layers/console - # php-fpm-dev - depot build \ - --platform=linux/arm64 \ - --build-arg=PHP_VERSION=$* \ - --build-arg=CPU_PREFIX=${CPU_PREFIX} \ - --load \ - --tag=bref/${CPU_PREFIX}php-$*-fpm-dev \ - layers/fpm-dev + PHP_VERSION=$* depot bake # Build Lambda layers (zip files) *locally* diff --git a/cpu-x86.Makefile b/cpu-x86.Makefile index b5e5aea1..254beaf3 100644 --- a/cpu-x86.Makefile +++ b/cpu-x86.Makefile @@ -4,6 +4,7 @@ export # export all variables defined in .env export CPU = x86 export CPU_PREFIX = export IMAGE_VERSION_SUFFIX = x86_64 +export DOCKER_PLATFORM = amd64 # Build all Docker images and layers *locally* @@ -14,52 +15,7 @@ default: docker-images layers # Build Docker images *locally* docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 docker-images-php-%: - # build - depot build \ - --platform=linux/amd64 \ - --build-arg=CPU=${CPU} \ - --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ - --load \ - --tag=bref/${CPU_PREFIX}build-php-$* \ - --file=php-$*/Dockerfile \ - --target=build-environment \ - . - # php - depot build \ - --platform=linux/amd64 \ - --build-arg=CPU=${CPU} \ - --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ - --load \ - --tag=bref/${CPU_PREFIX}php-$* \ - --file=php-$*/Dockerfile \ - --target=function \ - . - # php-fpm - depot build \ - --platform=linux/amd64 \ - --build-arg=CPU=${CPU} \ - --build-arg=IMAGE_VERSION_SUFFIX=${IMAGE_VERSION_SUFFIX} \ - --load \ - --tag=bref/${CPU_PREFIX}php-$*-fpm \ - --file=php-$*/Dockerfile \ - --target=fpm \ - . - # console - depot build \ - --platform=linux/amd64 \ - --build-arg=PHP_VERSION=$* \ - --build-arg=CPU_PREFIX=${CPU_PREFIX} \ - --load \ - --tag=bref/${CPU_PREFIX}php-$*-console \ - layers/console - # php-fpm-dev - depot build \ - --platform=linux/amd64 \ - --build-arg=PHP_VERSION=$* \ - --build-arg=CPU_PREFIX=${CPU_PREFIX} \ - --load \ - --tag=bref/${CPU_PREFIX}php-$*-fpm-dev \ - layers/fpm-dev + PHP_VERSION=$* depot bake # Build Lambda layers (zip files) *locally* diff --git a/docker-bake.hcl b/docker-bake.hcl index a01196f1..655c9bbd 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -14,6 +14,9 @@ variable "PHP_VERSION" { variable "IMAGE_VERSION_SUFFIX" { default = "x86_64" } +variable "DOCKER_PLATFORM" { + default = "amd64" +} target "build-php" { dockerfile = "php-${PHP_VERSION}/Dockerfile" @@ -23,6 +26,7 @@ target "build-php" { "CPU" = "${CPU}" "IMAGE_VERSION_SUFFIX" = "${IMAGE_VERSION_SUFFIX}" } + platforms = ["linux/${DOCKER_PLATFORM}"] } target "php" { @@ -36,6 +40,7 @@ target "php" { contexts = { "bref/${CPU_PREFIX}build-php-${PHP_VERSION}" = "target:build-php" } + platforms = ["linux/${DOCKER_PLATFORM}"] } target "php-fpm" { @@ -50,6 +55,7 @@ target "php-fpm" { "bref/${CPU_PREFIX}build-php-${PHP_VERSION}" = "target:build-php" "bref/${CPU_PREFIX}php-${PHP_VERSION}" = "target:php" } + platforms = ["linux/${DOCKER_PLATFORM}"] } target "console" { @@ -64,6 +70,7 @@ target "console" { "bref/${CPU_PREFIX}build-php-${PHP_VERSION}" = "target:build-php" "bref/${CPU_PREFIX}php-${PHP_VERSION}" = "target:php" } + platforms = ["linux/${DOCKER_PLATFORM}"] } target "php-fpm-dev" { @@ -79,4 +86,5 @@ target "php-fpm-dev" { "bref/${CPU_PREFIX}php-${PHP_VERSION}-fpm" = "target:php-fpm" "bref/local-api-gateway" = "docker-image://bref/local-api-gateway:latest" } + platforms = ["linux/${DOCKER_PLATFORM}"] } From 61de798ec0b8990ba783dab57cc0ee097076d9cd Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Mon, 23 Jan 2023 17:42:02 +0100 Subject: [PATCH 05/17] Explicitly set the Docker platform in tests to fix the CI --- .github/workflows/tests.yml | 2 +- docker-bake.hcl | 12 ++++++------ tests/Makefile | 14 +++++++------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c4bfef07..2026af03 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: CPU: ${{ matrix.cpu }} CPU_PREFIX: ${{ (matrix.cpu == 'arm') && 'arm-' || '' }} IMAGE_VERSION_SUFFIX: ${{ (matrix.cpu == 'arm') && 'arm64' || 'x86_64' }} - DOCKER_PLATFORM: ${{ (matrix.cpu == 'arm') && 'arm64' || 'amd64' }} + DOCKER_PLATFORM: ${{ (matrix.cpu == 'arm') && 'linux/arm64' || 'linux/amd64' }} - name: Test that layers can be exported run: | diff --git a/docker-bake.hcl b/docker-bake.hcl index 655c9bbd..8169770c 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -15,7 +15,7 @@ variable "IMAGE_VERSION_SUFFIX" { default = "x86_64" } variable "DOCKER_PLATFORM" { - default = "amd64" + default = "linux/amd64" } target "build-php" { @@ -26,7 +26,7 @@ target "build-php" { "CPU" = "${CPU}" "IMAGE_VERSION_SUFFIX" = "${IMAGE_VERSION_SUFFIX}" } - platforms = ["linux/${DOCKER_PLATFORM}"] + platforms = ["${DOCKER_PLATFORM}"] } target "php" { @@ -40,7 +40,7 @@ target "php" { contexts = { "bref/${CPU_PREFIX}build-php-${PHP_VERSION}" = "target:build-php" } - platforms = ["linux/${DOCKER_PLATFORM}"] + platforms = ["${DOCKER_PLATFORM}"] } target "php-fpm" { @@ -55,7 +55,7 @@ target "php-fpm" { "bref/${CPU_PREFIX}build-php-${PHP_VERSION}" = "target:build-php" "bref/${CPU_PREFIX}php-${PHP_VERSION}" = "target:php" } - platforms = ["linux/${DOCKER_PLATFORM}"] + platforms = ["${DOCKER_PLATFORM}"] } target "console" { @@ -70,7 +70,7 @@ target "console" { "bref/${CPU_PREFIX}build-php-${PHP_VERSION}" = "target:build-php" "bref/${CPU_PREFIX}php-${PHP_VERSION}" = "target:php" } - platforms = ["linux/${DOCKER_PLATFORM}"] + platforms = ["${DOCKER_PLATFORM}"] } target "php-fpm-dev" { @@ -86,5 +86,5 @@ target "php-fpm-dev" { "bref/${CPU_PREFIX}php-${PHP_VERSION}-fpm" = "target:php-fpm" "bref/local-api-gateway" = "docker-image://bref/local-api-gateway:latest" } - platforms = ["linux/${DOCKER_PLATFORM}"] + platforms = ["${DOCKER_PLATFORM}"] } diff --git a/tests/Makefile b/tests/Makefile index 8978add1..8a96111f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -7,19 +7,19 @@ test: test-80 test-81 test-82 # This rule matches with a wildcard, for example `test-80`. # The `$*` variable will contained the matched part, in this case `80`. test-%: vendor - docker run --rm -v=$(PWD):/var/task:ro --entrypoint=php bref/${CPU_PREFIX}php-$* \ + docker run --platform=${DOCKER_PLATFORM} --rm -v=$(PWD):/var/task:ro --entrypoint=php bref/${CPU_PREFIX}php-$* \ test_1_binary.php $* - docker run --rm -v=$(PWD):/var/task:ro --entrypoint=php bref/${CPU_PREFIX}php-$* \ + docker run --platform=${DOCKER_PLATFORM} --rm -v=$(PWD):/var/task:ro --entrypoint=php bref/${CPU_PREFIX}php-$* \ test_2_extensions.php - docker run --rm -v=$(PWD):/var/task:ro --entrypoint=php \ + docker run --platform=${DOCKER_PLATFORM} --rm -v=$(PWD):/var/task:ro --entrypoint=php \ -e PHP_INI_SCAN_DIR="/opt/bref/etc/php/conf.d/:/var/task/" bref/${CPU_PREFIX}php-$* \ test_3_manual_enabling_extensions.php # Test function handler docker stop test-${CPU_PREFIX}php-$* 2> /dev/null || true # silence errors - docker run --rm --detach -v=$(PWD):/var/task:ro --name test-${CPU_PREFIX}php-$* \ + docker run --platform=${DOCKER_PLATFORM} --rm --detach -v=$(PWD):/var/task:ro --name test-${CPU_PREFIX}php-$* \ bref/${CPU_PREFIX}php-$* test_4_function_handler.php docker exec test-${CPU_PREFIX}php-$* php test_4_function_invocation.php \ || (docker logs test-${CPU_PREFIX}php-$* && exit 1) @@ -27,7 +27,7 @@ test-%: vendor # Test FPM handler docker stop test-${CPU_PREFIX}php-$*-fpm 2> /dev/null || true # silence errors - docker run --rm --detach -v=$(PWD):/var/task:ro --name test-${CPU_PREFIX}php-$*-fpm \ + docker run --platform=${DOCKER_PLATFORM} --rm --detach -v=$(PWD):/var/task:ro --name test-${CPU_PREFIX}php-$*-fpm \ bref/${CPU_PREFIX}php-$*-fpm test_5_fpm_handler.php docker exec test-${CPU_PREFIX}php-$*-fpm php test_5_fpm_invocation.php \ || (docker logs test-${CPU_PREFIX}php-$*-fpm && exit 1) # print logs in case of failure @@ -35,7 +35,7 @@ test-%: vendor # Test console handler docker stop test-${CPU_PREFIX}php-$*-console 2> /dev/null || true # silence errors - docker run --rm --detach -v=$(PWD):/var/task:ro --name test-${CPU_PREFIX}php-$*-console \ + docker run --platform=${DOCKER_PLATFORM} --rm --detach -v=$(PWD):/var/task:ro --name test-${CPU_PREFIX}php-$*-console \ bref/${CPU_PREFIX}php-$*-console test_6_console_handler.php docker exec test-${CPU_PREFIX}php-$*-console php test_6_console_invocation.php \ || (docker logs test-${CPU_PREFIX}php-$*-console && exit 1) # print logs in case of failure @@ -43,7 +43,7 @@ test-%: vendor # Test that we can override PHP_INI_SCAN_DIR docker stop test-${CPU_PREFIX}php-$*-test7 2> /dev/null || true # silence errors - docker run --rm --detach -v=$(PWD):/var/task:ro --name test-${CPU_PREFIX}php-$*-test7 \ + docker run --platform=${DOCKER_PLATFORM} --rm --detach -v=$(PWD):/var/task:ro --name test-${CPU_PREFIX}php-$*-test7 \ -e PHP_INI_SCAN_DIR="/opt/bref/etc/php/conf.d/:/var/task/" \ bref/${CPU_PREFIX}php-$* test_4_function_handler.php docker exec test-${CPU_PREFIX}php-$*-test7 php test_7_custom_ini_scan_dir.php \ From 7ac54fad5733d645ce986352dda17f2fcadfe2f0 Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Mon, 23 Jan 2023 17:47:23 +0100 Subject: [PATCH 06/17] Fix Docker platform --- cpu-arm.Makefile | 2 +- cpu-x86.Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpu-arm.Makefile b/cpu-arm.Makefile index 32b10a16..320c91c3 100644 --- a/cpu-arm.Makefile +++ b/cpu-arm.Makefile @@ -4,7 +4,7 @@ export # export all variables defined in .env export CPU = arm export CPU_PREFIX = arm- export IMAGE_VERSION_SUFFIX = arm64 -export DOCKER_PLATFORM = arm64 +export DOCKER_PLATFORM = linux/arm64 # Build all Docker images and layers *locally* diff --git a/cpu-x86.Makefile b/cpu-x86.Makefile index 254beaf3..f9426cbd 100644 --- a/cpu-x86.Makefile +++ b/cpu-x86.Makefile @@ -4,7 +4,7 @@ export # export all variables defined in .env export CPU = x86 export CPU_PREFIX = export IMAGE_VERSION_SUFFIX = x86_64 -export DOCKER_PLATFORM = amd64 +export DOCKER_PLATFORM = linux/amd64 # Build all Docker images and layers *locally* From fb81096091f68ac414472e268aa4ce1a4066057e Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Mon, 23 Jan 2023 17:51:20 +0100 Subject: [PATCH 07/17] Allow debugging in CI --- tests/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 8a96111f..8a5207dd 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,7 +1,5 @@ export CPU_PREFIX ?= -.SILENT: test test-80 test-81 test-82 vendor - test: test-80 test-81 test-82 # This rule matches with a wildcard, for example `test-80`. From 78fe7492e3aeb7e1ab514329d149a03e0f5c6f89 Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Mon, 23 Jan 2023 17:58:51 +0100 Subject: [PATCH 08/17] Simplify the GitHub Actions config --- .github/workflows/tests.yml | 11 +++-------- cpu-arm.Makefile | 2 +- cpu-x86.Makefile | 2 +- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2026af03..e0eb6227 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,24 +21,19 @@ jobs: - 81 - 82 steps: - - uses: actions/checkout@v3 - uses: depot/setup-action@v1 - name: Build Docker images - run: depot bake --load + run: make -f cpu-${{ matrix.cpu }}.Makefile docker-images-php-${{ matrix.php_version }} env: DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }} - PHP_VERSION: ${{ matrix.php_version }} - CPU: ${{ matrix.cpu }} - CPU_PREFIX: ${{ (matrix.cpu == 'arm') && 'arm-' || '' }} - IMAGE_VERSION_SUFFIX: ${{ (matrix.cpu == 'arm') && 'arm64' || 'x86_64' }} - DOCKER_PLATFORM: ${{ (matrix.cpu == 'arm') && 'linux/arm64' || 'linux/amd64' }} - name: Test that layers can be exported run: | make -f cpu-${{ matrix.cpu }}.Makefile layer-php-${{ matrix.php_version }} make -f cpu-${{ matrix.cpu }}.Makefile layer-php-${{ matrix.php_version }}-fpm - - run: make -f cpu-${{ matrix.cpu }}.Makefile test-${{ matrix.php_version }} + - name: Run tests + run: make -f cpu-${{ matrix.cpu }}.Makefile test-${{ matrix.php_version }} diff --git a/cpu-arm.Makefile b/cpu-arm.Makefile index 320c91c3..f34f080f 100644 --- a/cpu-arm.Makefile +++ b/cpu-arm.Makefile @@ -15,7 +15,7 @@ default: docker-images layers # Build Docker images *locally* docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 docker-images-php-%: - PHP_VERSION=$* depot bake + PHP_VERSION=$* depot bake --load # Build Lambda layers (zip files) *locally* diff --git a/cpu-x86.Makefile b/cpu-x86.Makefile index f9426cbd..031a9d66 100644 --- a/cpu-x86.Makefile +++ b/cpu-x86.Makefile @@ -15,7 +15,7 @@ default: docker-images layers # Build Docker images *locally* docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 docker-images-php-%: - PHP_VERSION=$* depot bake + PHP_VERSION=$* depot bake --load # Build Lambda layers (zip files) *locally* From c16205a3654e74f854f020ac9adabe47bfafdc12 Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Sat, 28 Jan 2023 13:26:34 +0000 Subject: [PATCH 09/17] Try forcing the Depot platform to hopefully fix the CI https://github.com/depot/cli/pull/56 --- cpu-arm.Makefile | 2 +- cpu-x86.Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpu-arm.Makefile b/cpu-arm.Makefile index f34f080f..be459cca 100644 --- a/cpu-arm.Makefile +++ b/cpu-arm.Makefile @@ -15,7 +15,7 @@ default: docker-images layers # Build Docker images *locally* docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 docker-images-php-%: - PHP_VERSION=$* depot bake --load + PHP_VERSION=$* depot bake --build-platform=${DOCKER_PLATFORM} --load # Build Lambda layers (zip files) *locally* diff --git a/cpu-x86.Makefile b/cpu-x86.Makefile index 031a9d66..e06604db 100644 --- a/cpu-x86.Makefile +++ b/cpu-x86.Makefile @@ -15,7 +15,7 @@ default: docker-images layers # Build Docker images *locally* docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 docker-images-php-%: - PHP_VERSION=$* depot bake --load + PHP_VERSION=$* depot bake --build-platform=${DOCKER_PLATFORM} --load # Build Lambda layers (zip files) *locally* From 82c1eee07c55501addd7d3ecc1ef0ba67c2848cc Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Sat, 28 Jan 2023 17:03:01 +0000 Subject: [PATCH 10/17] CI --- .github/workflows/tests.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e0eb6227..44dbd5cc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,6 +23,16 @@ jobs: steps: - uses: actions/checkout@v3 + # See https://stackoverflow.com/questions/70312490/github-actions-runner-environment-doesnt-build-for-arm-images + - name: Set up QEMU to build and run ARM images + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker buildx to use BuildKit features + uses: docker/setup-buildx-action@v2 + with: + # Sets up `docker build` command as an alias to `docker buildx` + install: true + - uses: depot/setup-action@v1 - name: Build Docker images From e324c636be6b7430b53f135f6201398248545257 Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Sat, 28 Jan 2023 17:05:56 +0000 Subject: [PATCH 11/17] Cleanup previous changes that didn't help make the CI work --- cpu-arm.Makefile | 2 +- cpu-x86.Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpu-arm.Makefile b/cpu-arm.Makefile index be459cca..f34f080f 100644 --- a/cpu-arm.Makefile +++ b/cpu-arm.Makefile @@ -15,7 +15,7 @@ default: docker-images layers # Build Docker images *locally* docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 docker-images-php-%: - PHP_VERSION=$* depot bake --build-platform=${DOCKER_PLATFORM} --load + PHP_VERSION=$* depot bake --load # Build Lambda layers (zip files) *locally* diff --git a/cpu-x86.Makefile b/cpu-x86.Makefile index e06604db..031a9d66 100644 --- a/cpu-x86.Makefile +++ b/cpu-x86.Makefile @@ -15,7 +15,7 @@ default: docker-images layers # Build Docker images *locally* docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 docker-images-php-%: - PHP_VERSION=$* depot bake --build-platform=${DOCKER_PLATFORM} --load + PHP_VERSION=$* depot bake --load # Build Lambda layers (zip files) *locally* From 57658417d40d6b523f97d6942724570f43bd70da Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Sat, 28 Jan 2023 17:06:20 +0000 Subject: [PATCH 12/17] Simplify CI --- .github/workflows/tests.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 44dbd5cc..417e8c08 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,15 +24,9 @@ jobs: - uses: actions/checkout@v3 # See https://stackoverflow.com/questions/70312490/github-actions-runner-environment-doesnt-build-for-arm-images - - name: Set up QEMU to build and run ARM images + - name: Set up QEMU to run ARM images (that were built with Depot) uses: docker/setup-qemu-action@v2 - - name: Set up Docker buildx to use BuildKit features - uses: docker/setup-buildx-action@v2 - with: - # Sets up `docker build` command as an alias to `docker buildx` - install: true - - uses: depot/setup-action@v1 - name: Build Docker images From d8f5ee14ccb15fdc4b0ab0d33e7458ae4d226bd0 Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Sat, 28 Jan 2023 17:11:06 +0000 Subject: [PATCH 13/17] Edit CI message --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 417e8c08..1bd7537f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,7 +8,7 @@ on: jobs: tests: - name: Build and tests layers + name: Build and tests ${{ matrix.cpu }} PHP ${{ matrix.php_version }} layers runs-on: ubuntu-latest strategy: fail-fast: false From 20ad0e397b4796d6db1feb0261f040096fc89ec2 Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Sat, 28 Jan 2023 18:31:54 +0000 Subject: [PATCH 14/17] Update the documentation --- README.md | 72 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 06694675..fc4ba386 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ If you are submitting a pull request to this repository, you probably want to te 2. Run the test scripts. 3. Publish the Lambda layers to your AWS account and test them in a real Lambda. +**For minor changes** (e.g. upgrading a version) it is faster and easier to open a pull request. The layers will be built faster in CI and the test results will be available in a few minutes. + ### Requirements - `make` @@ -35,15 +37,19 @@ If you are submitting a pull request to this repository, you probably want to te > **Warning:** > -> On macOS, do not enable [the experimental Rosetta emulation](https://docs.docker.com/desktop/release-notes/#4160). This causes a Segmentation Fault when running `php-fpm` in the Docker images. +> On macOS, do not enable [the experimental Rosetta emulation](https://docs.docker.com/desktop/release-notes/#4160). This causes a Segmentation Fault when running `php-fpm` in the Docker images (as of January 2023, this may have been fixed since). You can build Docker images and Lambda layers locally: ```sh +# Make x86 layers (the default) make + +# Make ARM layers +make CPU=arm ``` -The process takes about 4 minutes. It will create the Docker images on your machine, and generate the Lambda layer zip files in `./output`. +It will create the Docker images on your machine, and generate the Lambda layer zip files in `./output`. It takes some time to build the Docker images (especially to build the images on the other platform, e.g. the ARM images if you are on an Intel processor). ### Testing @@ -51,6 +57,8 @@ After building the images, run the automated tests: ```sh make test +# and/or +make test CPU=arm ``` > **Note** @@ -72,16 +80,11 @@ cp .env.example .env # Then build layers: make +make CPU=arm # Then publish layers: make upload-layers -``` - -You can also limit to ARM or X86 layers: - -```sh -make -f cpu-x86.Makefile -make -f cpu-x86.Makefile upload-layers +make upload-layers CPU=arm ``` The published Lambda layers will be public (they are readonly anyway). You can find them in your AWS console (AWS Lambda service). Feel free to delete them afterwards. @@ -98,13 +101,13 @@ docker run --rm -it --entrypoint=bash bref/php-80 > > `ldd` is a linux utility that will show libraries (`.so` files) used by a binary/library. For example: `ldd /opt/bin/php` or `ldd /opt/bref/extensions/curl.so`. That helps to make sure we include all the libraries needed by PHP extensions in the layers. > -> However, `ldd` fails when running on another CPU architecture. So instead of `ldd`, we use `objdump -p /usr/bin/bash | grep NEEDED` (that needs to be installed with `yum install binutils`). +> However, `ldd` fails when running on another CPU architecture. So instead of `ldd`, we can use `objdump -p /usr/bin/bash | grep NEEDED` (that needs to be installed with `yum install binutils`). Related: `utils/lib-check` is a small utility-tool to check whether we're copying unnecessary `.so` files into the layer (i.e. `.so` files that already exist in Lambda). ### Supporting a new PHP version -The general idea is to copy `php-81` into `php-82`. Search/replace `php-81` with `php-82`, change PHP_VERSION in `Makefile`, and adapt anything else if needed. +The general idea is to copy `php-82` into `php-83`. Search/replace `php-82` with `php-83`, update the PHP version, update the `Makefile`, and adapt anything else if needed. ### Supporting new regions @@ -163,32 +166,25 @@ Anything we want to make available in AWS Lambda is possible by preparing the ri ### The php-xx folders -The Dockerfile attempts at a best-effort to follow a top-down execution process for easier reading. It starts from -an AWS-provided Docker Image and installs PHP. Some standard files (such as the php binary) can already be -isolated into the `/bref` folder. The use of multiple Docker Layers helps with investigations -because the developer can have a faster feedback loop by checking each step of the process incrementally instead -of trying to figure out why an entire build is failing. +The Dockerfile attempts at a best-effort to follow a top-down execution process for easier reading. -The 2nd layer is the `isolation` layer where we'll start from the standard AWS-provided image all over again -(getting rid of any residual unnecessary file) and then copying `/bref` into `/opt`. PHP Configurations are -copied here as well. +It starts from an AWS-provided Docker image and compiles the system libraries that we will need to use to compile PHP (the PHP build requires more recent version than what `yum install` provides, so we need to compile them, which is slow. -The 3rd layer is the `function` layer where everything is packet together and the `bootstrap` file is loaded. +Then, PHP is compiled. All the compilation happens in `/tmp`. -The 4th layer is `zip-function`, where we get a small and fast Linux (Alpine) just to install and zip the entire -`/opt` content. We use docker compose volumes to map `/tmp/bref-zip` from host to the container so that we can -zip everything and get the zipped file out of the container. +We then copy the PHP binary in `/opt/bin` and all PHP extensions in `/opt/...`. Indeed, `/opt` is the target directory of AWS Lambda layers. -The 5th layer goes back to `extensions` and start `fpm-extension`. Here we're back at step 2 so that we can install -`fpm`. +Then, we need to copy to `/opt` all the system libraries (`*.so` files) used by PHP and the extensions. To do so, we have a script that parses all the system dependencies of `/opt/bin/php` and extensions, and automatically copies them to `/opt/lib` (a directory automatically scanned by AWS Lambda). -The 6th layer goes back to `isolation` and start `fpm`. It mimics steps 3th and 4th but for the FPM Layer. +Finally, for each layer, we re-start from scratch from the empty AWS Lambda Docker images (using `FROM`, i.e. multi-stage builds) and we copy `/opt`. That gives us an "empty" Docker images with only `/opt` populated, just like on AWS Lambda when the PHP layers are unzipped. -Lastly, layer 7 zips FPM and pack everything ready for AWS Lambda. +That will also let us zip `/opt` to create the layers. ## Design decisions log -##### Installing PHP from a distribution +### Installing PHP from a distribution + +#### First iteration Compiling PHP is a complex process for the average PHP Developer. It takes a fair amount of time and can be cumbersome. Using `remi-collet` as a PHP distributor greatly simplifies and help the @@ -205,6 +201,18 @@ Useful links: - https://blog.remirepo.net/pages/English-FAQ#scl - https://rpms.remirepo.net/wizard/ +#### Second iteration + +We discovered an issue with using Remi's built images ([#42](https://github.com/brefphp/aws-lambda-layers/issues/42)): HTTP2 support was not compiled in CURL. Remi's packages explicitly don't intent to support it, and our only choice is to compile PHP (it's not an extension that can be installed after the fact). + +The previous decision (use Remi's repo) is reverted during Bref v2's beta and we go back to compiling PHP from scratch. + +Some benefits: + +- We can have identical compilation scripts between x86 and ARM, which simplifies the code a lot +- We can provide recent PHP versions for ARM, including PHP 8.2 (wasn't supported by Amazon Linux Extra before) +- We have identical system libraries and dependencies on x86 and ARM, which should avoid weird differences and bugs + ##### Bundling extensions While developing a new Runtime, the first attempt was to provide an "alpine-like" Bref Layer: only the PHP @@ -215,7 +223,7 @@ The benefits of maintaining a lightweight layer long-term didn't outweigh the co ##### Variables vs Repetitive Code -Before landing on the current architecture, there was several attempts (7 to be exact) on a back-and-forth +Before landing on the current architecture, there was several attempts (9 to be exact) on a back-and-forth between more environment variables vs more repetitive code. Environment variables grows complexity because they require contributors to understand how they intertwine with each other. We have layers, php version and CPU architecture. A more "reusable" Dockerfile or docker compose requires a more complex Makefile. In contrast, @@ -236,9 +244,11 @@ regions in chunks (7 to be precise) and tries to publish 7 layers at a time. AWS CodeBuild was used for Bref v1 builds, as it lets us use large instances to build faster. Bref 1 layers took an hour to build. Additionally, using CodeBuild allowed to avoid using AWS access keys to publish layers. -However, Bref 2's build only take 10 minutes, and runs really well in GitHub Actions. Additionally, using OIDC we can authorize this repository to publish into the AWS account with very restricted permissions _without_ using AWS access keys (assume role, just like CodeBuild). +For Bref v2, we now build in GitHub Actions because it's simpler, entirely public and easier to follow for maintainers and contributors. + +To make builds 10× to 20× faster, we use https://depot.dev thanks who generously offered to support Bref for free ❤️ -As such, we use GitHub Actions as it's simpler to set up, entirely public and much easier to follow. +Additionally, using OIDC we can authorize this repository to publish into the AWS account with very restricted permissions _without_ using AWS access keys (assume role, just like CodeBuild). ##### Automation tests From f0ecf71564ae9b56e914d360fbdc3567c30d9cb8 Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Sat, 28 Jan 2023 18:32:11 +0000 Subject: [PATCH 15/17] Merge the Makefiles using variables --- .env.example | 5 ++ .github/workflows/release.yml | 67 ++++++++--------- .github/workflows/tests.yml | 14 +++- Makefile | 136 ++++++++++++++++++++++++---------- cpu-arm.Makefile | 90 ---------------------- cpu-x86.Makefile | 95 ------------------------ 6 files changed, 141 insertions(+), 266 deletions(-) delete mode 100644 cpu-arm.Makefile delete mode 100644 cpu-x86.Makefile diff --git a/.env.example b/.env.example index 14542b71..df3bc473 100644 --- a/.env.example +++ b/.env.example @@ -7,3 +7,8 @@ # Limit the parallelization of layer publication. # Default is 7, we recommend a lower number when publishing from a laptop. MAX_PARALLEL_PUBLISH=3 + +# In the CI and on local machines in the core team we build using https://depot.dev +# as it is much faster to build cross-platform images. +# Do not uncomment this line, unless you have a depot.dev account. +#USE_DEPOT=1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dc1d8d47..11e8cf04 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,12 +15,22 @@ permissions: contents: read # This is required for actions/checkout jobs: - release-x86: - name: Publish x86 layers + + release: + name: Publish ${{ matrix.cpu }} layers runs-on: ubuntu-latest + strategy: + matrix: + cpu: + - x86 + - arm steps: - uses: actions/checkout@v3 + # See https://stackoverflow.com/questions/70312490/github-actions-runner-environment-doesnt-build-for-arm-images + - name: Set up QEMU to run ARM images (that were built with Depot) + uses: docker/setup-qemu-action@v2 + - uses: depot/setup-action@v1 - name: Configure AWS credentials @@ -37,49 +47,32 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Build Docker images - run: make -f cpu-${{ matrix.cpu }}.Makefile docker-images + run: make docker-images env: + CPU: ${{ matrix.cpu }} + USE_DEPOT: 1 DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }} - PHP_VERSION: ${{ matrix.php_version }} + + - run: make layers + env: CPU: ${{ matrix.cpu }} - CPU_PREFIX: ${{ (matrix.cpu == 'arm') && 'arm-' || '' }} - IMAGE_VERSION_SUFFIX: ${{ (matrix.cpu == 'arm') && 'arm64' || 'x86_64' }} - - run: make -f cpu-x86.Makefile layers - - run: make -f cpu-x86.Makefile test - - run: make -f cpu-x86.Makefile upload-layers - - run: make -f cpu-x86.Makefile upload-to-docker-hub + - run: make test + env: + CPU: ${{ matrix.cpu }} - release-arm: - name: Publish ARM layers - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: arn:aws:iam::534081306603:role/bref-layer-publisher-github-actions - role-session-name: bref-layer-publisher-github-actions - aws-region: us-east-1 - - name: Configure Docker Hub credentials - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - # See https://stackoverflow.com/questions/70312490/github-actions-runner-environment-doesnt-build-for-arm-images - - name: Set up QEMU to build ARM images - uses: docker/setup-qemu-action@v2 - - name: Set up Docker buildx to build ARM images - uses: docker/setup-buildx-action@v2 - - run: make -f cpu-arm.Makefile layers - - run: make -f cpu-arm.Makefile test - - run: make -f cpu-arm.Makefile upload-layers - - run: make -f cpu-arm.Makefile upload-to-docker-hub + - run: make upload-layers + env: + CPU: ${{ matrix.cpu }} + + - run: make upload-to-docker-hub + env: + CPU: ${{ matrix.cpu }} update-layer-versions: name: Update layer versions in brefphp/bref runs-on: ubuntu-latest - needs: [ release-x86, release-arm ] + needs: [ release ] steps: - name: Trigger layer update in brefphp/bref uses: actions/github-script@v6 @@ -96,7 +89,7 @@ jobs: update-layer-js-versions: name: Update layer versions in brefphp/layers.js runs-on: ubuntu-latest - needs: [ release-x86, release-arm ] + needs: [ release ] steps: - name: Trigger release in brefphp/layers.js uses: actions/github-script@v6 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1bd7537f..26dec691 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,14 +30,20 @@ jobs: - uses: depot/setup-action@v1 - name: Build Docker images - run: make -f cpu-${{ matrix.cpu }}.Makefile docker-images-php-${{ matrix.php_version }} + run: make docker-images-php-${{ matrix.php_version }} env: + CPU: ${{ matrix.cpu }} + USE_DEPOT: 1 DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }} - name: Test that layers can be exported run: | - make -f cpu-${{ matrix.cpu }}.Makefile layer-php-${{ matrix.php_version }} - make -f cpu-${{ matrix.cpu }}.Makefile layer-php-${{ matrix.php_version }}-fpm + make layer-php-${{ matrix.php_version }} + make layer-php-${{ matrix.php_version }}-fpm + env: + CPU: ${{ matrix.cpu }} - name: Run tests - run: make -f cpu-${{ matrix.cpu }}.Makefile test-${{ matrix.php_version }} + run: make test-${{ matrix.php_version }} + env: + CPU: ${{ matrix.cpu }} diff --git a/Makefile b/Makefile index 4f7a3c39..247b8052 100644 --- a/Makefile +++ b/Makefile @@ -2,46 +2,102 @@ -include .env export # export all variables defined in .env -# - Build all layers -# - Publish all Docker images to Docker Hub -# - Publish all layers to AWS Lambda +# Define all the environment variables depending on the CPU +# Set CPU= (empty) to build for x86 +# Set CPU=arm to build for ARM +ifeq ($(CPU), arm) # if $CPU=="arm" + $(info "⚠️ Building for ARM") # Print a message + export CPU = arm + export CPU_PREFIX = arm- + export IMAGE_VERSION_SUFFIX = arm64 + export DOCKER_PLATFORM = linux/arm64 +else + $(info "⚠️ Building for x86") # Print a message + export CPU = x86 + export CPU_PREFIX = + export IMAGE_VERSION_SUFFIX = x86_64 + export DOCKER_PLATFORM = linux/amd64 +endif + +# By default, Docker images are built using `docker buildx bake` +# But we use https://depot.dev in CI (super fast) by setting USE_DEPOT=1 +ifeq ($(USE_DEPOT), 1) # if $USE_DEPOT=="1" + $(info "⚠️ Building using depot.dev") # Print a message + export BAKE_COMMAND = depot bake +else + export BAKE_COMMAND = docker buildx bake +endif + + +# Build all Docker images and layers *locally* +# Use this to test your changes +default: docker-images layers + + +# Build Docker images *locally* +docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 +docker-images-php-%: + PHP_VERSION=$* ${BAKE_COMMAND} --load + + +# Build Lambda layers (zip files) *locally* +layers: layer-php-80 layer-php-81 layer-php-82 layer-php-80-fpm layer-php-81-fpm layer-php-82-fpm + # Build the console layer only once (x86 and single PHP version) + @if [ ${CPU} = "x86" ]; then \ + ./utils/docker-zip-dir.sh bref/php-80-console-zip console; \ + fi +# This rule matches with a wildcard, for example `layer-php-80`. +# The `$*` variable will contained the matched part, in this case `php-80`. +layer-%: + ./utils/docker-zip-dir.sh bref/${CPU_PREFIX}$* ${CPU_PREFIX}$* + + +# Upload the layers to AWS Lambda # Uses the current AWS_PROFILE. Most users will not want to use this option # as this will publish all layers to all regions + publish all Docker images. -everything: - $(MAKE) -f cpu-x86.Makefile everything - $(MAKE) -f cpu-arm.Makefile everything +upload-layers: upload-layers-php-80 upload-layers-php-81 upload-layers-php-82 + # Upload the console layer only once (x86 and single PHP version) + @if [ ${CPU} = "x86" ]; then \ + LAYER_NAME=console $(MAKE) -C ./utils/lambda-publish publish-parallel; \ + fi +upload-layers-php-%: + # Upload the function layers to AWS + LAYER_NAME=${CPU_PREFIX}php-$* $(MAKE) -C ./utils/lambda-publish publish-parallel + # Upload the FPM layers to AWS + LAYER_NAME=${CPU_PREFIX}php-$*-fpm $(MAKE) -C ./utils/lambda-publish publish-parallel -# Build Docker images *locally* -docker-images: - $(MAKE) -f cpu-x86.Makefile docker-images - $(MAKE) -f cpu-arm.Makefile docker-images - -# Build Lambda layers (zip files) *locally* (will also build Docker images) -layers: - $(MAKE) -f cpu-x86.Makefile layers - $(MAKE) -f cpu-arm.Makefile layers - -# Upload the layers to AWS Lambda (will also build Docker images and layers) -upload-layers: - $(MAKE) -f cpu-x86.Makefile upload-layers - $(MAKE) -f cpu-arm.Makefile upload-layers - -# Build and publish Docker images to Docker Hub. -# Only publishes the `latest` version. -# This process is executed when a merge to `main` happens. -# When a release tag is created, GitHub Actions -# will download the latest images, tag them with the version number -# and re-upload them with the right tag. -upload-to-docker-hub: - $(MAKE) -f cpu-x86.Makefile upload-to-docker-hub - $(MAKE) -f cpu-arm.Makefile upload-to-docker-hub - -test: - $(MAKE) -f cpu-x86.Makefile test - $(MAKE) -f cpu-arm.Makefile test - -clean: - $(MAKE) -f cpu-x86.Makefile clean - $(MAKE) -f cpu-arm.Makefile clean - -.PHONY: layers + +# Publish Docker images to Docker Hub. +upload-to-docker-hub: upload-to-docker-hub-php-80 upload-to-docker-hub-php-81 upload-to-docker-hub-php-82 +upload-to-docker-hub-php-%: + # While in beta we tag and push the `:2` version, later we'll push `:latest` as well + for image in \ + "bref/${CPU_PREFIX}php-$*" "bref/${CPU_PREFIX}php-$*-fpm" "bref/${CPU_PREFIX}php-$*-console" \ + "bref/${CPU_PREFIX}build-php-$*" "bref/${CPU_PREFIX}php-$*-fpm-dev"; \ + do \ + docker tag $$image $$image:2 ; \ + docker push $$image:2 ; \ + done + # TODO: when v2 becomes "latest", we should also push "latest" tags + # We could actually use `docker push --all-tags` at the end probably? + + +test: test-80 test-81 test-82 +test-%: + cd tests && $(MAKE) test-$* + + +clean: clean-80 clean-81 clean-82 + # Clear the build cache, else all images will be rebuilt using cached layers + docker builder prune + # Remove zip files + rm -f output/${CPU_PREFIX}*.zip +clean-%: + # Clean Docker images to force rebuilding them + docker image rm --force bref/${CPU_PREFIX}build-php-$* \ + bref/${CPU_PREFIX}php-$* \ + bref/${CPU_PREFIX}php-$*-zip \ + bref/${CPU_PREFIX}php-$*-fpm \ + bref/${CPU_PREFIX}php-$*-fpm-zip \ + bref/${CPU_PREFIX}php-$*-fpm-dev \ + bref/${CPU_PREFIX}php-$*-console diff --git a/cpu-arm.Makefile b/cpu-arm.Makefile deleted file mode 100644 index f34f080f..00000000 --- a/cpu-arm.Makefile +++ /dev/null @@ -1,90 +0,0 @@ -# Load .env file if it exists --include .env -export # export all variables defined in .env -export CPU = arm -export CPU_PREFIX = arm- -export IMAGE_VERSION_SUFFIX = arm64 -export DOCKER_PLATFORM = linux/arm64 - - -# Build all Docker images and layers *locally* -# Use this to test your changes -default: docker-images layers - - -# Build Docker images *locally* -docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 -docker-images-php-%: - PHP_VERSION=$* depot bake --load - - -# Build Lambda layers (zip files) *locally* -layers: layer-php-80 layer-php-81 layer-php-82 layer-php-80-fpm layer-php-81-fpm layer-php-82-fpm -# This rule matches with a wildcard, for example `layer-php-80`. -# The `$*` variable will contained the matched part, in this case `php-80`. -layer-%: - ./utils/docker-zip-dir.sh bref/arm-$* arm-$* - - -# Upload the layers to AWS Lambda -# Uses the current AWS_PROFILE. Most users will not want to use this option -# as this will publish all layers to all regions + publish all Docker images. -upload-layers: - # Upload the function layers to AWS - LAYER_NAME=arm-php-80 $(MAKE) -C ./utils/lambda-publish publish-parallel - LAYER_NAME=arm-php-81 $(MAKE) -C ./utils/lambda-publish publish-parallel - LAYER_NAME=arm-php-82 $(MAKE) -C ./utils/lambda-publish publish-parallel - - # Upload the FPM layers to AWS - LAYER_NAME=arm-php-80-fpm $(MAKE) -C ./utils/lambda-publish publish-parallel - LAYER_NAME=arm-php-81-fpm $(MAKE) -C ./utils/lambda-publish publish-parallel - LAYER_NAME=arm-php-82-fpm $(MAKE) -C ./utils/lambda-publish publish-parallel - - -# Publish Docker images to Docker Hub. -upload-to-docker-hub: - # While in beta we tag and push the `:2` version, later we'll push `:latest` as well - for image in \ - "bref/arm-php-80" "bref/arm-php-80-fpm" "bref/arm-php-80-console" "bref/arm-build-php-80" "bref/arm-php-80-fpm-dev" \ - "bref/arm-php-81" "bref/arm-php-81-fpm" "bref/arm-php-81-console" "bref/arm-build-php-81" "bref/arm-php-81-fpm-dev"; \ - "bref/arm-php-82" "bref/arm-php-82-fpm" "bref/arm-php-82-console" "bref/arm-build-php-82" "bref/arm-php-82-fpm-dev"; \ - do \ - docker tag $$image $$image:2 ; \ - docker push $$image:2 ; \ - done - # TODO: when v2 becomes "latest", we should also push "latest" tags - # We could actually use `docker push --all-tags` at the end probably? - - -test: test-80 test-81 test-82 -test-%: - cd tests && $(MAKE) test-$* - - -clean: - # Remove zip files - rm -f output/arm-*.zip - # Clean Docker images to force rebuilding them - docker image rm --force bref/arm-build-php-80 - docker image rm --force bref/arm-build-php-81 - docker image rm --force bref/arm-build-php-82 - docker image rm --force bref/arm-php-80 - docker image rm --force bref/arm-php-81 - docker image rm --force bref/arm-php-82 - docker image rm --force bref/arm-php-80-zip - docker image rm --force bref/arm-php-81-zip - docker image rm --force bref/arm-php-82-zip - docker image rm --force bref/arm-php-80-fpm - docker image rm --force bref/arm-php-81-fpm - docker image rm --force bref/arm-php-82-fpm - docker image rm --force bref/arm-php-80-fpm-zip - docker image rm --force bref/arm-php-81-fpm-zip - docker image rm --force bref/arm-php-82-fpm-zip - docker image rm --force bref/arm-php-80-fpm-dev - docker image rm --force bref/arm-php-81-fpm-dev - docker image rm --force bref/arm-php-82-fpm-dev - docker image rm --force bref/arm-php-80-console - docker image rm --force bref/arm-php-81-console - docker image rm --force bref/arm-php-82-console - # Clear the build cache, else all images will be rebuilt using cached layers - docker builder prune diff --git a/cpu-x86.Makefile b/cpu-x86.Makefile deleted file mode 100644 index 031a9d66..00000000 --- a/cpu-x86.Makefile +++ /dev/null @@ -1,95 +0,0 @@ -# Load .env file if it exists --include .env -export # export all variables defined in .env -export CPU = x86 -export CPU_PREFIX = -export IMAGE_VERSION_SUFFIX = x86_64 -export DOCKER_PLATFORM = linux/amd64 - - -# Build all Docker images and layers *locally* -# Use this to test your changes -default: docker-images layers - - -# Build Docker images *locally* -docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82 -docker-images-php-%: - PHP_VERSION=$* depot bake --load - - -# Build Lambda layers (zip files) *locally* -layers: layer-php-80 layer-php-81 layer-php-82 layer-php-80-fpm layer-php-81-fpm layer-php-82-fpm - # Handle this layer specifically - ./utils/docker-zip-dir.sh bref/php-80-console-zip console -# This rule matches with a wildcard, for example `layer-php-80`. -# The `$*` variable will contained the matched part, in this case `php-80`. -layer-%: - ./utils/docker-zip-dir.sh bref/$* $* - - -# Upload the layers to AWS Lambda -# Uses the current AWS_PROFILE. Most users will not want to use this option -# as this will publish all layers to all regions + publish all Docker images. -upload-layers: - # Upload the function layers to AWS - LAYER_NAME=php-80 $(MAKE) -C ./utils/lambda-publish publish-parallel - LAYER_NAME=php-81 $(MAKE) -C ./utils/lambda-publish publish-parallel - LAYER_NAME=php-82 $(MAKE) -C ./utils/lambda-publish publish-parallel - - # Upload the FPM layers to AWS - LAYER_NAME=php-80-fpm $(MAKE) -C ./utils/lambda-publish publish-parallel - LAYER_NAME=php-81-fpm $(MAKE) -C ./utils/lambda-publish publish-parallel - LAYER_NAME=php-82-fpm $(MAKE) -C ./utils/lambda-publish publish-parallel - - # Upload the console layer to AWS - LAYER_NAME=console $(MAKE) -C ./utils/lambda-publish publish-parallel - - -# Publish Docker images to Docker Hub. -upload-to-docker-hub: - # While in beta we tag and push the `:2` version, later we'll push `:latest` as well - for image in \ - "bref/php-80" "bref/php-80-fpm" "bref/php-80-console" "bref/build-php-80" "bref/php-80-fpm-dev" \ - "bref/php-81" "bref/php-81-fpm" "bref/php-81-console" "bref/build-php-81" "bref/php-81-fpm-dev" \ - "bref/php-82" "bref/php-82-fpm" "bref/php-82-console" "bref/build-php-82" "bref/php-82-fpm-dev"; \ - do \ - docker tag $$image $$image:2 ; \ - docker push $$image:2 ; \ - done - # TODO: when v2 becomes "latest", we should also push "latest" tags - # We could actually use `docker push --all-tags` at the end probably? - - -test: test-80 test-81 test-82 -test-%: - cd tests && $(MAKE) test-$* - - -clean: - # Remove zip files - rm -f output/*.zip - # Clean Docker images to force rebuilding them - docker image rm --force bref/build-php-80 - docker image rm --force bref/build-php-81 - docker image rm --force bref/build-php-82 - docker image rm --force bref/php-80 - docker image rm --force bref/php-81 - docker image rm --force bref/php-82 - docker image rm --force bref/php-80-zip - docker image rm --force bref/php-81-zip - docker image rm --force bref/php-82-zip - docker image rm --force bref/php-80-fpm - docker image rm --force bref/php-81-fpm - docker image rm --force bref/php-82-fpm - docker image rm --force bref/php-80-fpm-zip - docker image rm --force bref/php-81-fpm-zip - docker image rm --force bref/php-82-fpm-zip - docker image rm --force bref/php-80-fpm-dev - docker image rm --force bref/php-81-fpm-dev - docker image rm --force bref/php-82-fpm-dev - docker image rm --force bref/php-80-console - docker image rm --force bref/php-81-console - docker image rm --force bref/php-82-console - # Clear the build cache, else all images will be rebuilt using cached layers - docker builder prune From 6101ee2b462a35a6fa37d83c5fe6a97d0658e101 Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Sat, 28 Jan 2023 18:37:44 +0000 Subject: [PATCH 16/17] Change CI name --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 26dec691..e1bc7728 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,7 +8,7 @@ on: jobs: tests: - name: Build and tests ${{ matrix.cpu }} PHP ${{ matrix.php_version }} layers + name: Build and tests PHP ${{ matrix.php_version }}, ${{ matrix.cpu }} runs-on: ubuntu-latest strategy: fail-fast: false From 8db39ace0036083974c72e9b25b0078250a12004 Mon Sep 17 00:00:00 2001 From: Matthieu Napoli Date: Sat, 28 Jan 2023 18:38:41 +0000 Subject: [PATCH 17/17] Upgrade PHP versions --- php-80/Dockerfile | 3 ++- php-81/Dockerfile | 1 + php-82/Dockerfile | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/php-80/Dockerfile b/php-80/Dockerfile index e302a39c..cfefbca4 100644 --- a/php-80/Dockerfile +++ b/php-80/Dockerfile @@ -5,7 +5,8 @@ ARG CPU # Can be "x86_64" or "arm64" ARG IMAGE_VERSION_SUFFIX -ARG VERSION_PHP=8.0.25 +# https://www.php.net/downloads +ARG VERSION_PHP=8.0.27 # Lambda uses a custom AMI named Amazon Linux 2 diff --git a/php-81/Dockerfile b/php-81/Dockerfile index 923ef247..9c7f65ee 100644 --- a/php-81/Dockerfile +++ b/php-81/Dockerfile @@ -5,6 +5,7 @@ ARG CPU # Can be "x86_64" or "arm64" ARG IMAGE_VERSION_SUFFIX +# https://www.php.net/downloads ARG VERSION_PHP=8.1.14 diff --git a/php-82/Dockerfile b/php-82/Dockerfile index 69af8624..22136e12 100644 --- a/php-82/Dockerfile +++ b/php-82/Dockerfile @@ -5,7 +5,8 @@ ARG CPU # Can be "x86_64" or "arm64" ARG IMAGE_VERSION_SUFFIX -ARG VERSION_PHP=8.2.0 +# https://www.php.net/downloads +ARG VERSION_PHP=8.2.1 # Lambda uses a custom AMI named Amazon Linux 2