From ac44f139825d34425e3f900c5a62dd59f8888196 Mon Sep 17 00:00:00 2001 From: Serein Pfeiffer Date: Wed, 2 Jul 2025 21:23:35 +0200 Subject: [PATCH 1/4] docker: Support arm64 manylinux image creation. --- Dockerfile.template | 18 +++++++++++++----- README.md | 33 +++++++++++++++++++++++++++++---- deploy.bash | 6 ++++++ python-wheel-globals.cmake | 22 ++++++++++++++++++++-- 4 files changed, 68 insertions(+), 11 deletions(-) diff --git a/Dockerfile.template b/Dockerfile.template index f81c05a..761e4a5 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -21,11 +21,19 @@ RUN yum update -y && yum install -y \ RUN rm -f /usr/local/bin/cmake /usr/local/bin/ctest /usr/local/bin/cpack # Install CMake 3.31.8 (latest 3.x version) -ADD https://github.com/Kitware/CMake/releases/download/v3.31.8/cmake-3.31.8-linux-x86_64.tar.gz /tmp/ -RUN cd /tmp && \ - tar -xzf cmake-3.31.8-linux-x86_64.tar.gz && \ - cp -r cmake-3.31.8-linux-x86_64/bin/* /usr/local/bin/ && \ - cp -r cmake-3.31.8-linux-x86_64/share/* /usr/local/share/ && \ +# Map Docker architecture to CMake architecture naming +RUN if [ "${architecture}" = "x86_64" ]; then \ + CMAKE_ARCH="x86_64"; \ + elif [ "${architecture}" = "aarch64" ]; then \ + CMAKE_ARCH="aarch64"; \ + else \ + echo "Unsupported architecture: ${architecture}"; exit 1; \ + fi && \ + curl -L https://github.com/Kitware/CMake/releases/download/v3.31.8/cmake-3.31.8-linux-${CMAKE_ARCH}.tar.gz -o /tmp/cmake.tar.gz && \ + cd /tmp && \ + tar -xzf cmake.tar.gz && \ + cp -r cmake-3.31.8-linux-${CMAKE_ARCH}/bin/* /usr/local/bin/ && \ + cp -r cmake-3.31.8-linux-${CMAKE_ARCH}/share/* /usr/local/share/ && \ rm -rf /tmp/cmake* # Install Python diff --git a/README.md b/README.md index 88c6280..d356a2f 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,24 @@ add_wheel_test(mylib-test ) ``` +## Building Docker Images + +To build the manylinux Docker images for different architectures: + +```bash +# Build x86_64 images (default) +./deploy.bash + +# Build ARM64/aarch64 images +./deploy.bash --arch aarch64 + +# Build and push images with version tag +./deploy.bash --version 2025.1 --push + +# Build, tag as latest, and push +./deploy.bash --version 2025.1 --push --latest +``` + ## CI Utilities This repository also provides several utilities to facilitate additional wheel deployment steps that are needed on macOS and Linux. @@ -116,16 +134,22 @@ This repository also provides several utilities to facilitate additional wheel d For CI jobs, this repo provides the following docker images: +**x86_64 architecture:** * `manylinux-cpp17-py3.9-x86_64` * `manylinux-cpp17-py3.10-x86_64` * `manylinux-cpp17-py3.11-x86_64` * `manylinux-cpp17-py3.12-x86_64` * `manylinux-cpp17-py3.13-x86_64` -This images are based on GLIBC 2.28, so e.g. the minimum Ubuntu version -for wheels from your CI will be 21.04. +**aarch64 (ARM64) architecture:** +* `manylinux-cpp17-py3.9-aarch64` +* `manylinux-cpp17-py3.10-aarch64` +* `manylinux-cpp17-py3.11-aarch64` +* `manylinux-cpp17-py3.12-aarch64` +* `manylinux-cpp17-py3.13-aarch64` -Note: `aarch64` images are not yet deployed. Let us know if you need them! +These images are based on GLIBC 2.28, so e.g. the minimum Ubuntu version +for wheels from your CI will be 21.04. You may use a Github Actions Snippet like this to build your wheels: @@ -135,8 +159,9 @@ jobs: strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + architecture: ["x86_64", "aarch64"] runs-on: ubuntu-latest - container: ghcr.io/klebert-engineering/manylinux-cpp17-py${{ matrix.python-version }}-x86_64:latest + container: ghcr.io/klebert-engineering/manylinux-cpp17-py${{ matrix.python-version }}-${{ matrix.architecture }}:latest steps: - uses: actions/checkout@v3 with: diff --git a/deploy.bash b/deploy.bash index 1216529..8b285fe 100755 --- a/deploy.bash +++ b/deploy.bash @@ -30,6 +30,12 @@ while [[ $# -gt 0 ]]; do esac done +# Validate architecture +if [[ "$architecture" != "x86_64" && "$architecture" != "aarch64" ]]; then + echo "Error: Unsupported architecture '$architecture'. Supported architectures are: x86_64, aarch64" + exit 1 +fi + for pyver_long in "${python_versions[@]}"; do diff --git a/python-wheel-globals.cmake b/python-wheel-globals.cmake index cc0cfcc..6608827 100644 --- a/python-wheel-globals.cmake +++ b/python-wheel-globals.cmake @@ -31,9 +31,27 @@ elseif (APPLE) "-c" "import platform; print('_'.join(platform.mac_ver()[0].split('.')[:2]))" OUTPUT_VARIABLE MACOS_VER OUTPUT_STRIP_TRAILING_WHITESPACE) - set(PY_WHEEL_PLATFORM "macosx_${MACOS_VER}_x86_64") + execute_process( + COMMAND + "${Python3_EXECUTABLE}" + "-c" "import platform; print(platform.machine())" + OUTPUT_VARIABLE MACHINE_ARCH + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(MACHINE_ARCH STREQUAL "arm64") + set(PY_WHEEL_PLATFORM "macosx_${MACOS_VER}_arm64") + else() + set(PY_WHEEL_PLATFORM "macosx_${MACOS_VER}_x86_64") + endif() else() - if(CMAKE_SIZEOF_VOID_P EQUAL 8) + execute_process( + COMMAND + "${Python3_EXECUTABLE}" + "-c" "import platform; print(platform.machine())" + OUTPUT_VARIABLE MACHINE_ARCH + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(MACHINE_ARCH STREQUAL "aarch64") + set(PY_WHEEL_PLATFORM "linux_aarch64") + elseif(CMAKE_SIZEOF_VOID_P EQUAL 8) set(PY_WHEEL_PLATFORM "linux_x86_64") else() set(PY_WHEEL_PLATFORM "linux_i686") From a3968d7c551e5d732350179a3468b917d66f97c7 Mon Sep 17 00:00:00 2001 From: Serein Pfeiffer Date: Tue, 8 Jul 2025 14:43:11 +0200 Subject: [PATCH 2/4] docker: Use Python that comes with the base many-linux image. --- Dockerfile.template | 24 ++++++++++++++---------- deploy.bash | 15 ++++++--------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Dockerfile.template b/Dockerfile.template index 761e4a5..32a3937 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -36,13 +36,17 @@ RUN if [ "${architecture}" = "x86_64" ]; then \ cp -r cmake-3.31.8-linux-${CMAKE_ARCH}/share/* /usr/local/share/ && \ rm -rf /tmp/cmake* -# Install Python -ADD https://www.python.org/ftp/python/${pyver_long}/Python-${pyver_long}.tgz /usr/local/src -RUN cd /usr/local/src && \ - tar -xzvf Python-${pyver_long}.tgz && \ - cd Python-${pyver_long} && \ - ./configure --enable-shared && \ - make -j$(nproc) install -RUN ln -sf /usr/local/src/Python-${pyver_long}/libpython${pyver_short}.so.1.0 /lib64/libpython${pyver_short}.so.1.0 -RUN pip${pyver_short} install wheel -RUN pip${pyver_short} install -U pip +# Use pre-installed Python from manylinux image +# The manylinux images already have Python versions in /opt/python/ +# Create symlinks to make the Python version available system-wide +RUN if [ -d /opt/python/cp$(echo ${pyver_short} | tr -d '.')-cp$(echo ${pyver_short} | tr -d '.') ]; then \ + ln -sf /opt/python/cp$(echo ${pyver_short} | tr -d '.')-cp$(echo ${pyver_short} | tr -d '.')/bin/python /usr/local/bin/python${pyver_short} && \ + ln -sf /opt/python/cp$(echo ${pyver_short} | tr -d '.')-cp$(echo ${pyver_short} | tr -d '.')/bin/pip /usr/local/bin/pip${pyver_short} && \ + echo "Using pre-installed Python ${pyver_short}"; \ + else \ + echo "Python ${pyver_short} not found in /opt/python/"; \ + exit 1; \ + fi + +# Ensure wheel is installed (should already be present in manylinux images) +RUN pip${pyver_short} install --upgrade pip wheel diff --git a/deploy.bash b/deploy.bash index 8b285fe..a6bc75e 100755 --- a/deploy.bash +++ b/deploy.bash @@ -4,7 +4,7 @@ image_name="manylinux-cpp17-py" version="2025.1" push="" latest="" -python_versions=(3.9.13 3.10.9 3.11.1 3.12.4 3.13.1) +python_versions=(3.9 3.10 3.11 3.12 3.13) architecture=x86_64 while [[ $# -gt 0 ]]; do @@ -37,20 +37,17 @@ if [[ "$architecture" != "x86_64" && "$architecture" != "aarch64" ]]; then fi -for pyver_long in "${python_versions[@]}"; do +for pyver in "${python_versions[@]}"; do - pyver_short=$(echo "$pyver_long" | sed "s/\\.[0-9]\+\$//") + echo "Building $architecture manylinux Docker image for Python $pyver..." - echo "Building $architecture manylinux Docker image for Python $pyver_short ($pyver_long)..." + dockerfile="Dockerfile-$pyver-$architecture" - dockerfile="Dockerfile-$pyver_long-$architecture" - - sed -e "s/\${pyver_long}/$pyver_long/g" \ - -e "s/\${pyver_short}/$pyver_short/g" \ + sed -e "s/\${pyver_short}/$pyver/g" \ -e "s/\${architecture}/$architecture/g" \ Dockerfile.template > $dockerfile - image_name_full="ghcr.io/klebert-engineering/$image_name$pyver_short-$architecture" + image_name_full="ghcr.io/klebert-engineering/$image_name$pyver-$architecture" docker build -t "$image_name_full:$version" -f $dockerfile . if [[ -n "$latest" ]]; then From 9af467ec9bb27ee1b4ac1488dbc6a94a0d23041e Mon Sep 17 00:00:00 2001 From: Serein Pfeiffer Date: Tue, 8 Jul 2025 15:47:20 +0200 Subject: [PATCH 3/4] docker: Ensure that the right Python version gets activated. --- Dockerfile.template | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Dockerfile.template b/Dockerfile.template index 32a3937..7da4d47 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -40,8 +40,15 @@ RUN if [ "${architecture}" = "x86_64" ]; then \ # The manylinux images already have Python versions in /opt/python/ # Create symlinks to make the Python version available system-wide RUN if [ -d /opt/python/cp$(echo ${pyver_short} | tr -d '.')-cp$(echo ${pyver_short} | tr -d '.') ]; then \ + # Create versioned symlinks ln -sf /opt/python/cp$(echo ${pyver_short} | tr -d '.')-cp$(echo ${pyver_short} | tr -d '.')/bin/python /usr/local/bin/python${pyver_short} && \ ln -sf /opt/python/cp$(echo ${pyver_short} | tr -d '.')-cp$(echo ${pyver_short} | tr -d '.')/bin/pip /usr/local/bin/pip${pyver_short} && \ + # Override the generic python3 to point to our specific version + ln -sf /opt/python/cp$(echo ${pyver_short} | tr -d '.')-cp$(echo ${pyver_short} | tr -d '.')/bin/python /usr/local/bin/python3 && \ + ln -sf /opt/python/cp$(echo ${pyver_short} | tr -d '.')-cp$(echo ${pyver_short} | tr -d '.')/bin/pip /usr/local/bin/pip3 && \ + # Also create python symlink for maximum compatibility + ln -sf /opt/python/cp$(echo ${pyver_short} | tr -d '.')-cp$(echo ${pyver_short} | tr -d '.')/bin/python /usr/local/bin/python && \ + ln -sf /opt/python/cp$(echo ${pyver_short} | tr -d '.')-cp$(echo ${pyver_short} | tr -d '.')/bin/pip /usr/local/bin/pip && \ echo "Using pre-installed Python ${pyver_short}"; \ else \ echo "Python ${pyver_short} not found in /opt/python/"; \ @@ -50,3 +57,7 @@ RUN if [ -d /opt/python/cp$(echo ${pyver_short} | tr -d '.')-cp$(echo ${pyver_sh # Ensure wheel is installed (should already be present in manylinux images) RUN pip${pyver_short} install --upgrade pip wheel + +# Set environment variables to help CMake find the correct Python +# Note: We set PATH to prioritize /usr/local/bin where our symlinks are +ENV PATH=/usr/local/bin:$PATH From 6e7b831e2a31359820d1371f184385fbbfb8e6d5 Mon Sep 17 00:00:00 2001 From: Serein Pfeiffer Date: Tue, 8 Jul 2025 17:05:48 +0200 Subject: [PATCH 4/4] cmake: Use Development.Module dependency (instead of just Development) as this is what manylinux is designed for. --- python-wheel-globals.cmake | 2 +- python-wheel.cmake | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python-wheel-globals.cmake b/python-wheel-globals.cmake index 6608827..6479257 100644 --- a/python-wheel-globals.cmake +++ b/python-wheel-globals.cmake @@ -1,4 +1,4 @@ -find_package(Python3 COMPONENTS Interpreter Development REQUIRED) +find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED) # Some RPATH setup for macOS if (APPLE) diff --git a/python-wheel.cmake b/python-wheel.cmake index 16937cf..e29f3f4 100644 --- a/python-wheel.cmake +++ b/python-wheel.cmake @@ -35,7 +35,7 @@ endfunction() function (add_wheel WHEEL_TARGET) set(Python_FIND_VIRTUALENV FIRST) # Favor venv over system install - find_package(Python3 COMPONENTS Interpreter Development REQUIRED) + find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED) # Parse arguments cmake_parse_arguments(WHEEL @@ -189,7 +189,7 @@ endfunction() function (add_wheel_test TEST_NAME) set(Python_FIND_VIRTUALENV FIRST) # Favor venv over system install - find_package(Python3 COMPONENTS Interpreter Development REQUIRED) + find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED) # Parse arguments cmake_parse_arguments(WHEEL_TEST