From fcf80fea1d407d81bb1f270b9ae0995e5fc2d6fb Mon Sep 17 00:00:00 2001 From: mattip Date: Thu, 25 Jul 2024 12:03:53 +0300 Subject: [PATCH 1/8] use DYNAMIC_LIST to limit the number of kernels compiled into the wheel --- tools/build_openblas.sh | 5 ++++- tools/build_steps.sh | 15 +++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tools/build_openblas.sh b/tools/build_openblas.sh index 253c6366..58d50583 100644 --- a/tools/build_openblas.sh +++ b/tools/build_openblas.sh @@ -64,12 +64,14 @@ if [ "$build_bits" == 64 ]; then extra="-fno-asynchronous-unwind-tables" vc_arch="X64" plat_tag="win_amd64" + dynamic_list="PRESCOTT NEHALEM SANDYBRIDGE HASWELL SKYLAKEX" else march=pentium4 extra="-mfpmath=sse -msse2" fextra="-m32" vc_arch="i386" plat_tag="win32" + dynamic_list="PRESCOTT NEHALEM SANDYBRIDGE HASWELL" fi cflags="-O2 -march=$march -mtune=generic $extra" fflags="$fextra $cflags -frecursive -ffpe-summary=invalid,zero" @@ -94,7 +96,8 @@ OPENBLAS_VERSION=$(git describe --tags --abbrev=8) # Variable used in creating output libraries make BINARY=$build_bits DYNAMIC_ARCH=1 USE_THREAD=1 USE_OPENMP=0 \ NUM_THREADS=24 NO_WARMUP=1 NO_AFFINITY=1 CONSISTENT_FPCSR=1 \ - BUILD_LAPACK_DEPRECATED=1 TARGET=PRESCOTT BUFFERSIZE=20\ + BUILD_LAPACK_DEPRECATED=1 TARGET=PRESCOTT BUFFERSIZE=20 \ + DYNAMIC_LIST="$dynamic_list" \ LDFLAGS="$LDFLAGS" \ COMMON_OPT="$cflags" \ FCOMMON_OPT="$fflags" \ diff --git a/tools/build_steps.sh b/tools/build_steps.sh index bd738d45..46c0313a 100644 --- a/tools/build_steps.sh +++ b/tools/build_steps.sh @@ -124,32 +124,35 @@ function do_build_lib { case $(get_os)-$plat in Linux-x86_64) local bitness=64 - local target_flags="TARGET=PRESCOTT" + local target=PRESCOTT + local dynamic_list="PRESCOTT NEHALEM SANDYBRIDGE HASWELL SKYLAKEX" ;; Darwin-x86_64) local bitness=64 - local target_flags="TARGET=CORE2" + local target_flags=CORE2 + local dynamic_list="CORE2 NEHALEM SANDYBRIDGE HASWELL SKYLAKEX" # Pick up the gfortran runtime libraries export DYLD_LIBRARY_PATH=/usr/local/lib:$DYLD_LIBRARY_PATH ;; *-i686) local bitness=32 - local target_flags="TARGET=PRESCOTT" + local target_flags=PRESCOTT + local dynamic_list="PRESCOTT NEHALEM SANDYBRIDGE HASWELL" ;; Linux-aarch64) local bitness=64 - local target_flags="TARGET=ARMV8" + local target=ARMV8 ;; Darwin-arm64) local bitness=64 - local target_flags="TARGET=VORTEX" + local target=VORTEX ;; *-s390x) local bitness=64 ;; *-ppc64le) local bitness=64 - local target_flags="TARGET=POWER8" + local target=POWER8 ;; *) echo "Strange plat value $plat"; exit 1 ;; esac From 3a5fdd01aa88d7642862900f531b9c443ebd654b Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 29 Jul 2024 16:48:48 +0300 Subject: [PATCH 2/8] add dynamic_list to aarch64 builds, issue 170 --- tools/build_steps.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tools/build_steps.sh b/tools/build_steps.sh index 46c0313a..6e1e378f 100644 --- a/tools/build_steps.sh +++ b/tools/build_steps.sh @@ -142,6 +142,9 @@ function do_build_lib { Linux-aarch64) local bitness=64 local target=ARMV8 + # temporarily limit the kernels for travis until there is data + # see https://github.com/MacPython/openblas-libs/issues/170 + local dynamic_list="ARMV8 CORTEXA57 NEOVERSEV1 THUNDERX" ;; Darwin-arm64) local bitness=64 @@ -175,23 +178,25 @@ function do_build_lib { git config --global --add safe.directory '*' pushd OpenBLAS patch_source - echo start building if [ -v dynamic_list ]; then CFLAGS="$CFLAGS -fvisibility=protected -Wno-uninitialized" \ make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ USE_OPENMP=0 NUM_THREADS=64 \ DYNAMIC_LIST="$dynamic_list" \ BINARY=$bitness $interface_flags $target_flags shared 2>&1 1>/dev/null + make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ + USE_OPENMP=0 NUM_THREADS=64 \ + DYNAMIC_LIST="$dynamic_list" \ + BINARY=$bitness $interface_flags $target_flags tests else CFLAGS="$CFLAGS -fvisibility=protected -Wno-uninitialized" \ make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ USE_OPENMP=0 NUM_THREADS=64 \ BINARY=$bitness $interface_flags $target_flags shared 2>&1 1>/dev/null + make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ + USE_OPENMP=0 NUM_THREADS=64 \ + BINARY=$bitness $interface_flags $target_flags tests fi - echo done building, now testing - make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ - USE_OPENMP=0 NUM_THREADS=64 \ - BINARY=$bitness $interface_flags $target_flags tests make PREFIX=$BUILD_PREFIX $interface_flags install popd if [ "$nightly" = "1" ]; then From e4ce5348d7935cf3d3d917451ba68cbd4f7ca1fb Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 30 Jul 2024 12:26:09 +0300 Subject: [PATCH 3/8] update wheel version to 0.3.27.350.1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5a3a84b8..39e348b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "scipy-openblas64" # v0.3.27-350-gd11e7340 -version = "0.3.27.350.0" +version = "0.3.27.350.1" requires-python = ">=3.7" description = "Provides OpenBLAS for python packaging" readme = "README.md" From e9a7924ad41edefb38c82d414b4ff8094b48e54b Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 30 Jul 2024 16:17:56 +0300 Subject: [PATCH 4/8] fix bad rebase merge --- tools/build_steps.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/build_steps.sh b/tools/build_steps.sh index 6e1e378f..9869dbbb 100644 --- a/tools/build_steps.sh +++ b/tools/build_steps.sh @@ -129,14 +129,14 @@ function do_build_lib { ;; Darwin-x86_64) local bitness=64 - local target_flags=CORE2 + local target=CORE2 local dynamic_list="CORE2 NEHALEM SANDYBRIDGE HASWELL SKYLAKEX" # Pick up the gfortran runtime libraries export DYLD_LIBRARY_PATH=/usr/local/lib:$DYLD_LIBRARY_PATH ;; *-i686) local bitness=32 - local target_flags=PRESCOTT + local target=PRESCOTT local dynamic_list="PRESCOTT NEHALEM SANDYBRIDGE HASWELL" ;; Linux-aarch64) @@ -183,19 +183,19 @@ function do_build_lib { make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ USE_OPENMP=0 NUM_THREADS=64 \ DYNAMIC_LIST="$dynamic_list" \ - BINARY=$bitness $interface_flags $target_flags shared 2>&1 1>/dev/null + BINARY=$bitness $interface_flags TARGET="$target" shared 2>&1 1>/dev/null make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ USE_OPENMP=0 NUM_THREADS=64 \ DYNAMIC_LIST="$dynamic_list" \ - BINARY=$bitness $interface_flags $target_flags tests + BINARY=$bitness $interface_flags TARGET="$target" tests else CFLAGS="$CFLAGS -fvisibility=protected -Wno-uninitialized" \ make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ USE_OPENMP=0 NUM_THREADS=64 \ - BINARY=$bitness $interface_flags $target_flags shared 2>&1 1>/dev/null + BINARY=$bitness $interface_flags TARGET="$target" shared 2>&1 1>/dev/null make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ USE_OPENMP=0 NUM_THREADS=64 \ - BINARY=$bitness $interface_flags $target_flags tests + BINARY=$bitness $interface_flags TARGET="$target" tests fi make PREFIX=$BUILD_PREFIX $interface_flags install popd From fa286b38bd618415342382aad1257f57e7e16cd8 Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 30 Jul 2024 19:56:33 +0300 Subject: [PATCH 5/8] build aarch64 on github via docker --- .github/workflows/posix.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index ca6c41df..8ad96c68 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -46,24 +46,24 @@ jobs: MB_ML_LIBC: musllinux MB_ML_VER: _1_1 - # - os: ubuntu-latest - # PLAT: aarch64 - # INTERFACE64: '0' - # MB_ML_VER: '2014' - # - os: ubuntu-latest - # PLAT: aarch64 - # INTERFACE64: '1' - # MB_ML_VER: '2014' - # - os: ubuntu-latest - # PLAT: aarch64 - # INTERFACE64: '0' - # MB_ML_LIBC: musllinux - # MB_ML_VER: _1_1 - # - os: ubuntu-latest - # PLAT: aarch64 - # INTERFACE64: '1' - # MB_ML_LIBC: musllinux - # MB_ML_VER: _1_1 + - os: ubuntu-latest + PLAT: aarch64 + INTERFACE64: '0' + MB_ML_VER: '2014' + - os: ubuntu-latest + PLAT: aarch64 + INTERFACE64: '1' + MB_ML_VER: '2014' + - os: ubuntu-latest + PLAT: aarch64 + INTERFACE64: '0' + MB_ML_LIBC: musllinux + MB_ML_VER: _1_1 + - os: ubuntu-latest + PLAT: aarch64 + INTERFACE64: '1' + MB_ML_LIBC: musllinux + MB_ML_VER: _1_1 exclude: - PLAT: i686 From fab6282546bc9eabf0eb97542f46719836ac069d Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 30 Jul 2024 22:54:53 +0300 Subject: [PATCH 6/8] limit musllinux_1_1_aarch64 kernels, use QUIET_MAKE --- tools/build_steps.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/build_steps.sh b/tools/build_steps.sh index 9869dbbb..9659baae 100644 --- a/tools/build_steps.sh +++ b/tools/build_steps.sh @@ -144,7 +144,11 @@ function do_build_lib { local target=ARMV8 # temporarily limit the kernels for travis until there is data # see https://github.com/MacPython/openblas-libs/issues/170 - local dynamic_list="ARMV8 CORTEXA57 NEOVERSEV1 THUNDERX" + if [ "$MB_ML_LIBC" == "musllinux" ]; then + local dynamic_list="ARMV8 CORTEXA57 THUNDERX" + else + local dynamic_list="ARMV8 CORTEXA57 NEOVERSEV1 THUNDERX" + fi ;; Darwin-arm64) local bitness=64 @@ -180,14 +184,10 @@ function do_build_lib { patch_source if [ -v dynamic_list ]; then CFLAGS="$CFLAGS -fvisibility=protected -Wno-uninitialized" \ - make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ + make BUFFERSIZE=20 DYNAMIC_ARCH=1 QUIET_MAKE=1 \ USE_OPENMP=0 NUM_THREADS=64 \ DYNAMIC_LIST="$dynamic_list" \ - BINARY=$bitness $interface_flags TARGET="$target" shared 2>&1 1>/dev/null - make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ - USE_OPENMP=0 NUM_THREADS=64 \ - DYNAMIC_LIST="$dynamic_list" \ - BINARY=$bitness $interface_flags TARGET="$target" tests + BINARY=$bitness $interface_flags TARGET="$target" else CFLAGS="$CFLAGS -fvisibility=protected -Wno-uninitialized" \ make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ From de7b9a655856d9261cf611bd8b6d18e7724110cb Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 31 Jul 2024 11:00:21 +0300 Subject: [PATCH 7/8] add musllinux_1_2 builds --- .github/workflows/posix.yml | 14 ++++++++++++-- tools/build_steps.sh | 4 ++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 8ad96c68..d6313f01 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -13,6 +13,7 @@ env: REPO_DIR: OpenBLAS OPENBLAS_COMMIT: "d11e7340" MACOSX_DEPLOYMENT_TARGET: 10.9 + MB_PYTHON_VERSION: 3.11 jobs: build: @@ -46,6 +47,7 @@ jobs: MB_ML_LIBC: musllinux MB_ML_VER: _1_1 + # Move these 6 builds to a faster option - os: ubuntu-latest PLAT: aarch64 INTERFACE64: '0' @@ -64,6 +66,16 @@ jobs: INTERFACE64: '1' MB_ML_LIBC: musllinux MB_ML_VER: _1_1 + - os: ubuntu-latest + PLAT: aarch64 + INTERFACE64: '0' + MB_ML_LIBC: musllinux + MB_ML_VER: _1_2 + - os: ubuntu-latest + PLAT: aarch64 + INTERFACE64: '1' + MB_ML_LIBC: musllinux + MB_ML_VER: _1_2 exclude: - PLAT: i686 @@ -72,8 +84,6 @@ jobs: INTERFACE64: '1' env: NIGHTLY: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} - MB_PYTHON_VERSION: ${{ matrix.python-version }} - TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} MB_ML_LIBC: ${{ matrix.MB_ML_LIBC }} MB_ML_VER: ${{ matrix.MB_ML_VER }} INTERFACE64: ${{ matrix.INTERFACE64 }} diff --git a/tools/build_steps.sh b/tools/build_steps.sh index 9659baae..66494276 100644 --- a/tools/build_steps.sh +++ b/tools/build_steps.sh @@ -142,9 +142,9 @@ function do_build_lib { Linux-aarch64) local bitness=64 local target=ARMV8 - # temporarily limit the kernels for travis until there is data + # temporarily limit the kernels for aarch64 until there is data # see https://github.com/MacPython/openblas-libs/issues/170 - if [ "$MB_ML_LIBC" == "musllinux" ]; then + if [ "$MB_ML_LIBC" == "musllinux" -a "$MB_ML_VER" == "_1_1" ]; then local dynamic_list="ARMV8 CORTEXA57 THUNDERX" else local dynamic_list="ARMV8 CORTEXA57 NEOVERSEV1 THUNDERX" From 46b71e12e52790802a5e95d8aadfa53edfb1d5b3 Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 31 Jul 2024 14:12:05 +0300 Subject: [PATCH 8/8] use more specific upload name --- .github/workflows/posix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index d6313f01..efcda0cc 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -168,12 +168,12 @@ jobs: - uses: actions/upload-artifact@v4.3.0 with: - name: wheels-${{ matrix.os }}-${{ matrix.PLAT }}-${{ matrix.INTERFACE64 }}-${{ matrix.MB_ML_LIBC }} + name: wheels-${{ matrix.os }}-${{ matrix.PLAT }}-${{ matrix.INTERFACE64 }}-${{ matrix.MB_ML_LIBC }}-${{ matrix.MB_ML_VER }} path: dist/scipy_openblas*.whl - uses: actions/upload-artifact@v4.3.0 with: - name: openblas-${{ matrix.os }}-${{ matrix.PLAT }}-${{ matrix.INTERFACE64 }}-${{ matrix.MB_ML_LIBC }} + name: openblas-${{ matrix.os }}-${{ matrix.PLAT }}-${{ matrix.INTERFACE64 }}-${{ matrix.MB_ML_LIBC }}-${{ matrix.MB_ML_VER }} path: libs/openblas*.tar.gz - uses: conda-incubator/setup-miniconda@v3.0.4