From 5ce4e14bb06e594ac9125e9bc4d5469297c280b0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 28 Mar 2023 02:24:03 -0700 Subject: [PATCH 1/4] Wheels: 0.15.0.post2 - zlib builds: remove `.dll`: As in Unix builds, avoid that a `.dll` can ever be picked up. - bump version to `0.15.0.post2` --- .github/workflows/build.yml | 2 +- .travis.yml | 2 +- library_builders.bat | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ed57c1b3ce..7bdd890177 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,7 +118,7 @@ jobs: uses: suisei-cn/actions-download-file@v1 id: setupversion with: - url: "https://gist.githubusercontent.com/ax3l/5e83edefe4b05cf6aa2a971649285fe0/raw/b510a363aac6b68651ec8081efbb40cb3bda1d4b/0001-Bump-setup.py.patch" + url: "https://gist.githubusercontent.com/ax3l/5e83edefe4b05cf6aa2a971649285fe0/raw/023cbd69e06715e5191ece741bb1de56560a9f96/0001-Bump-setup.py.patch" target: src/.patch/ - name: Apply Patches diff --git a/.travis.yml b/.travis.yml index 6fe7b89564..31c80dbb97 100644 --- a/.travis.yml +++ b/.travis.yml @@ -145,7 +145,7 @@ before_script: - curl -sOL https://github.com/ax3l/openPMD-api/commit/b622cc5ea770f866c1e373185a9e389c04bdb54c.patch - curl -sOL https://github.com/ax3l/openPMD-api/commit/a1aac530dbc8261656cf6a07900374bc6ac6fcab.patch - curl -sOL https://github.com/ax3l/openPMD-api/commit/43c4bf4327616e9cf7e7992d91d9b8ccd4ee3a83.patch - - curl -sOL https://gist.githubusercontent.com/ax3l/5e83edefe4b05cf6aa2a971649285fe0/raw/b510a363aac6b68651ec8081efbb40cb3bda1d4b/0001-Bump-setup.py.patch + - curl -sOL https://gist.githubusercontent.com/ax3l/5e83edefe4b05cf6aa2a971649285fe0/raw/023cbd69e06715e5191ece741bb1de56560a9f96/0001-Bump-setup.py.patch - cd .. - python3 -m patch .patch/b622cc5ea770f866c1e373185a9e389c04bdb54c.patch - python3 -m patch .patch/a1aac530dbc8261656cf6a07900374bc6ac6fcab.patch diff --git a/library_builders.bat b/library_builders.bat index 02f08702cb..5c7b552ec6 100644 --- a/library_builders.bat +++ b/library_builders.bat @@ -161,6 +161,11 @@ exit /b 0 cmake --build build-zlib --target install --config Release if errorlevel 1 exit 1 + set "zlib_dll=%BUILD_PREFIX:~1,-1%/zlib/bin/zlib.dll" + set "zlib_dll=%zlib_dll:/=\%" + del "%zlib_dll%" + if errorlevel 1 exit 1 + break > zlib-stamp if errorlevel 1 exit 1 exit /b 0 From 5bc62b98a437561cec6cc69a4b0deb3175c2781d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 28 Mar 2023 22:44:46 -0700 Subject: [PATCH 2/4] [Hack] Win w/ only HDF5 w/ static Zlib requries CMake 3.24+ --- .github/workflows/build.yml | 43 +++++++++++++++++++------------------ library_builders.bat | 37 +++++++++++++++++++++++-------- library_builders.sh | 20 ++++++++++++----- 3 files changed, 65 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7bdd890177..4a0a29f9f1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,11 +10,11 @@ jobs: fail-fast: false matrix: include: - - os: ubuntu-20.04 - arch: "x86_64" + #- os: ubuntu-20.04 + # arch: "x86_64" - - os: ubuntu-20.04 - arch: "i686" + #- os: ubuntu-20.04 + # arch: "i686" # builds faster on Travis-CI: #- os: ubuntu-20.04 @@ -34,25 +34,25 @@ jobs: arch: "AMD64" # x86 (32bit) - - os: windows-2019 - arch: "x86" - env: - CMAKE_GENERATOR: "Visual Studio 16 2019" - CMAKE_GENERATOR_PLATFORM: "Win32" - - - os: macos-10.15 - arch: "x86_64" - env: - MACOSX_DEPLOYMENT_TARGET: 10.15 + #- os: windows-2019 + # arch: "x86" + # env: + # CMAKE_GENERATOR: "Visual Studio 16 2019" + # CMAKE_GENERATOR_PLATFORM: "Win32" + + #- os: macos-10.15 + # arch: "x86_64" + # env: + # MACOSX_DEPLOYMENT_TARGET: 10.15 # Apple Silicon M1/arm64/aarch64 builds: # https://cibuildwheel.readthedocs.io/en/stable/faq/#apple-silicon # https://github.com/pypa/cibuildwheel/pull/704 - - os: macos-11 - arch: "arm64" - env: - CMAKE_OSX_ARCHITECTURES: "arm64" - MACOSX_DEPLOYMENT_TARGET: 11.0 + #- os: macos-11 + # arch: "arm64" + # env: + # CMAKE_OSX_ARCHITECTURES: "arm64" + # MACOSX_DEPLOYMENT_TARGET: 11.0 # Apple universal builds that contain x86-64 and arm64 binary code # Needs extra treatment of all librarys that are not CMake @@ -134,6 +134,7 @@ jobs: env: # (1) Disable PyPy (manylinux image: yum repo issues) # https://github.com/pypa/manylinux/issues/899 + CIBW_BUILD: "cp311-win_amd64" CIBW_SKIP: "pp*-manylinux*" CIBW_ARCHS: "${{ matrix.arch }}" CIBW_PROJECT_REQUIRES_PYTHON: ">=3.7" @@ -143,8 +144,8 @@ jobs: CIBW_BEFORE_BUILD_WINDOWS: 'cmd /E:ON /V:ON /C .github\library_builders.bat' # for the openPMD-api build, CMake shall search for # static dependencies of HDF5 and ADIOS1 (see setup.py) - CIBW_ENVIRONMENT: HDF5_USE_STATIC_LIBRARIES='ON' ADIOS_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' - CIBW_ENVIRONMENT_WINDOWS: HDF5_USE_STATIC_LIBRARIES='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' CMAKE_PREFIX_PATH='C:/Program Files (x86)/ADIOS2;C:/Program Files (x86)/blosc;C:/Program Files (x86)/HDF5;C:/Program Files (x86)/ZFP;C:/Program Files (x86)/zlib' + CIBW_ENVIRONMENT: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' ADIOS_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' + CIBW_ENVIRONMENT_WINDOWS: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='OFF' CMAKE_PREFIX_PATH='C:/Program Files (x86)/ADIOS2;C:/Program Files (x86)/blosc;C:/Program Files (x86)/HDF5;C:/Program Files (x86)/ZFP;C:/Program Files (x86)/zlib' # C++17 support in macOS 10.13+ (partial) and 10.14+ (std::visit) and 10.15+ (std::filesystem::path) # https://cibuildwheel.readthedocs.io/en/stable/cpp_standards/#macos-and-deployment-target-versions # arm64 Python interpreters are built with 11.0 diff --git a/library_builders.bat b/library_builders.bat index 5c7b552ec6..4452ca40cd 100644 --- a/library_builders.bat +++ b/library_builders.bat @@ -51,6 +51,9 @@ exit /b 0 cmake --build build-adios2 --target install --config Release if errorlevel 1 exit 1 + rmdir /s /q build-adios2 + if errorlevel 1 exit 1 + break > adios2-stamp if errorlevel 1 exit 1 exit /b 0 @@ -68,6 +71,8 @@ exit /b 0 -DBUILD_SHARED=OFF ^ -DBUILD_STATIC=ON ^ -DBUILD_TESTS=OFF ^ + -DPREFER_EXTERNAL_ZLIB=ON ^ + -DZLIB_USE_STATIC_LIBS=ON ^ -DDEACTIVATE_SNAPPY=ON if errorlevel 1 exit 1 @@ -77,6 +82,9 @@ exit /b 0 cmake --build build-blosc --target install --config Release if errorlevel 1 exit 1 + rmdir /s /q build-blosc + if errorlevel 1 exit 1 + break > blosc-stamp if errorlevel 1 exit 1 exit /b 0 @@ -93,6 +101,7 @@ exit /b 0 -DCMAKE_VERBOSE_MAKEFILE=ON ^ -DBUILD_SHARED_LIBS=OFF ^ -DBUILD_TESTING=OFF ^ + -DTEST_SHELL_SCRIPTS=OFF ^ -DHDF5_BUILD_CPP_LIB=OFF ^ -DHDF5_BUILD_EXAMPLES=OFF ^ -DHDF5_BUILD_FORTRAN=OFF ^ @@ -101,6 +110,7 @@ exit /b 0 -DHDF5_ENABLE_PARALLEL=OFF ^ -DHDF5_ENABLE_SZIP_SUPPORT=OFF ^ -DHDF5_ENABLE_Z_LIB_SUPPORT=ON ^ + -DZLIB_USE_STATIC_LIBS=ON ^ -DCMAKE_INSTALL_PREFIX=%BUILD_PREFIX%/HDF5 if errorlevel 1 exit 1 @@ -110,6 +120,9 @@ exit /b 0 cmake --build build-hdf5 --target install --config Release if errorlevel 1 exit 1 + rmdir /s /q build-hdf5 + if errorlevel 1 exit 1 + break > hdf5-stamp if errorlevel 1 exit 1 exit /b 0 @@ -137,6 +150,9 @@ exit /b 0 cmake --build build-zfp --target install --config Release if errorlevel 1 exit 1 + rmdir /s /q build-zfp + if errorlevel 1 exit 1 + break > zfp-stamp if errorlevel 1 exit 1 exit /b 0 @@ -144,12 +160,12 @@ exit /b 0 :build_zlib if exist zlib-stamp exit /b 0 - curl -sLo zlib-1.2.12.zip ^ - https://github.com/madler/zlib/archive/v1.2.12.zip - powershell Expand-Archive zlib-1.2.12.zip -DestinationPath dep-zlib + curl -sLo zlib-1.2.13.zip ^ + https://github.com/madler/zlib/archive/v1.2.13.zip + powershell Expand-Archive zlib-1.2.13.zip -DestinationPath dep-zlib - cmake -S dep-zlib/zlib-1.2.12 -B build-zlib ^ - -DBUILD_SHARED_LIBS=OFF ^ + cmake -S dep-zlib/zlib-1.2.13 -B build-zlib ^ + -DBUILD_SHARED_LIBS=ON ^ -DCMAKE_BUILD_TYPE=Release if errorlevel 1 exit 1 :: Manually-specified variables were not used by the project: @@ -161,11 +177,14 @@ exit /b 0 cmake --build build-zlib --target install --config Release if errorlevel 1 exit 1 - set "zlib_dll=%BUILD_PREFIX:~1,-1%/zlib/bin/zlib.dll" + set "zlib_dll=%BUILD_PREFIX:~1,-1%/zlib/bin/zlib1.dll" set "zlib_dll=%zlib_dll:/=\%" del "%zlib_dll%" if errorlevel 1 exit 1 + rmdir /s /q build-zlib + if errorlevel 1 exit 1 + break > zlib-stamp if errorlevel 1 exit 1 exit /b 0 @@ -175,7 +194,7 @@ call :install_buildessentials call :build_zlib :: build_bzip2 :: build_szip -call :build_blosc -call :build_zfp +::call :build_blosc +::call :build_zfp call :build_hdf5 -call :build_adios2 +::call :build_adios2 diff --git a/library_builders.sh b/library_builders.sh index 9377d57f03..38c83888d2 100755 --- a/library_builders.sh +++ b/library_builders.sh @@ -118,8 +118,8 @@ function build_adios2 { ADIOS2_USE_SST=OFF fi - mkdir build-ADIOS2 - cd build-ADIOS2 + mkdir build-adios2 + cd build-adios2 PY_BIN=$(which python3) CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" if [ "$(uname -s)" = "Linux" ] @@ -153,6 +153,8 @@ function build_adios2 { make install cd - + rm -rf build-adios2 + touch adios2-stamp } @@ -178,8 +180,8 @@ function build_blosc { DEACTIVATE_SSE2=ON fi - mkdir build-c-blosc - cd build-c-blosc + mkdir build-blosc + cd build-blosc PY_BIN=$(which python3) CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" PATH=${CMAKE_BIN}:${PATH} cmake \ @@ -190,11 +192,15 @@ function build_blosc { -DBUILD_BENCHMARKS=OFF \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} \ + -DPREFER_EXTERNAL_ZLIB=ON \ + -DZLIB_USE_STATIC_LIBS=ON \ ../c-blosc-* make -j${CPU_COUNT} make install cd - + rm -rf build-blosc + touch blosc-stamp } @@ -221,13 +227,15 @@ function build_zfp { make install cd - + rm -rf build-zfp + touch zfp-stamp } function build_zlib { if [ -e zlib-stamp ]; then return; fi - ZLIB_VERSION="1.2.12" + ZLIB_VERSION="1.2.13" curl -sLO https://zlib.net/fossils/zlib-$ZLIB_VERSION.tar.gz file zlib*.tar.gz @@ -247,6 +255,8 @@ function build_zlib { PATH=${CMAKE_BIN}:${PATH} cmake --build build-zlib --target install rm -rf ${BUILD_PREFIX}/lib/libz.*dylib ${BUILD_PREFIX}/lib/libz.*so + rm -rf build-zlib + touch zlib-stamp } From 7523314d51eb1713d819d4d5232e39d6a6247f21 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 30 Mar 2023 00:52:41 -0700 Subject: [PATCH 3/4] Win: Activate ADIOS2, too --- .github/workflows/build.yml | 2 +- library_builders.bat | 7 +++---- library_builders.sh | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4a0a29f9f1..32918a1112 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -145,7 +145,7 @@ jobs: # for the openPMD-api build, CMake shall search for # static dependencies of HDF5 and ADIOS1 (see setup.py) CIBW_ENVIRONMENT: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' ADIOS_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' - CIBW_ENVIRONMENT_WINDOWS: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='OFF' CMAKE_PREFIX_PATH='C:/Program Files (x86)/ADIOS2;C:/Program Files (x86)/blosc;C:/Program Files (x86)/HDF5;C:/Program Files (x86)/ZFP;C:/Program Files (x86)/zlib' + CIBW_ENVIRONMENT_WINDOWS: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' CMAKE_PREFIX_PATH='C:/Program Files (x86)/ADIOS2;C:/Program Files (x86)/blosc;C:/Program Files (x86)/HDF5;C:/Program Files (x86)/ZFP;C:/Program Files (x86)/zlib' # C++17 support in macOS 10.13+ (partial) and 10.14+ (std::visit) and 10.15+ (std::filesystem::path) # https://cibuildwheel.readthedocs.io/en/stable/cpp_standards/#macos-and-deployment-target-versions # arm64 Python interpreters are built with 11.0 diff --git a/library_builders.bat b/library_builders.bat index 4452ca40cd..383e2d13d1 100644 --- a/library_builders.bat +++ b/library_builders.bat @@ -71,7 +71,6 @@ exit /b 0 -DBUILD_SHARED=OFF ^ -DBUILD_STATIC=ON ^ -DBUILD_TESTS=OFF ^ - -DPREFER_EXTERNAL_ZLIB=ON ^ -DZLIB_USE_STATIC_LIBS=ON ^ -DDEACTIVATE_SNAPPY=ON if errorlevel 1 exit 1 @@ -194,7 +193,7 @@ call :install_buildessentials call :build_zlib :: build_bzip2 :: build_szip -::call :build_blosc -::call :build_zfp +call :build_blosc +call :build_zfp call :build_hdf5 -::call :build_adios2 +call :build_adios2 diff --git a/library_builders.sh b/library_builders.sh index 38c83888d2..6f7dba2ed1 100755 --- a/library_builders.sh +++ b/library_builders.sh @@ -192,7 +192,6 @@ function build_blosc { -DBUILD_BENCHMARKS=OFF \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} \ - -DPREFER_EXTERNAL_ZLIB=ON \ -DZLIB_USE_STATIC_LIBS=ON \ ../c-blosc-* make -j${CPU_COUNT} From 2058077990b1da1776b688bcb6f40fb1096ee8c1 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 30 Mar 2023 12:28:05 -0700 Subject: [PATCH 4/4] Reactivate All Builds --- .github/workflows/build.yml | 41 ++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 32918a1112..06034e6b8f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,11 +10,11 @@ jobs: fail-fast: false matrix: include: - #- os: ubuntu-20.04 - # arch: "x86_64" + - os: ubuntu-20.04 + arch: "x86_64" - #- os: ubuntu-20.04 - # arch: "i686" + - os: ubuntu-20.04 + arch: "i686" # builds faster on Travis-CI: #- os: ubuntu-20.04 @@ -34,25 +34,25 @@ jobs: arch: "AMD64" # x86 (32bit) - #- os: windows-2019 - # arch: "x86" - # env: - # CMAKE_GENERATOR: "Visual Studio 16 2019" - # CMAKE_GENERATOR_PLATFORM: "Win32" - - #- os: macos-10.15 - # arch: "x86_64" - # env: - # MACOSX_DEPLOYMENT_TARGET: 10.15 + - os: windows-2019 + arch: "x86" + env: + CMAKE_GENERATOR: "Visual Studio 16 2019" + CMAKE_GENERATOR_PLATFORM: "Win32" + + - os: macos-10.15 + arch: "x86_64" + env: + MACOSX_DEPLOYMENT_TARGET: 10.15 # Apple Silicon M1/arm64/aarch64 builds: # https://cibuildwheel.readthedocs.io/en/stable/faq/#apple-silicon # https://github.com/pypa/cibuildwheel/pull/704 - #- os: macos-11 - # arch: "arm64" - # env: - # CMAKE_OSX_ARCHITECTURES: "arm64" - # MACOSX_DEPLOYMENT_TARGET: 11.0 + - os: macos-11 + arch: "arm64" + env: + CMAKE_OSX_ARCHITECTURES: "arm64" + MACOSX_DEPLOYMENT_TARGET: 11.0 # Apple universal builds that contain x86-64 and arm64 binary code # Needs extra treatment of all librarys that are not CMake @@ -113,7 +113,7 @@ jobs: url: "https://github.com/ax3l/openPMD-api/commit/43c4bf4327616e9cf7e7992d91d9b8ccd4ee3a83.patch" target: src/.patch/ - # 0.15.0.post1 bump + # 0.15.0.post2 bump - name: Download Patch 4/4 uses: suisei-cn/actions-download-file@v1 id: setupversion @@ -134,7 +134,6 @@ jobs: env: # (1) Disable PyPy (manylinux image: yum repo issues) # https://github.com/pypa/manylinux/issues/899 - CIBW_BUILD: "cp311-win_amd64" CIBW_SKIP: "pp*-manylinux*" CIBW_ARCHS: "${{ matrix.arch }}" CIBW_PROJECT_REQUIRES_PYTHON: ">=3.7"