From 5fdb28a76df51169e83542eedce23cfb2ee0b3d7 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Sun, 27 Nov 2022 13:29:56 +0800 Subject: [PATCH 1/3] Supoort prebuild macos-arm64 arch binaries. --- .github/workflows/ci-build-release-napi.yml | 17 +++++++++++---- .github/workflows/ci-pr-validation.yml | 23 +++++++++++---------- dependencies.yaml | 2 +- pkg/mac/build-cpp-deps-lib.sh | 8 +++---- pkg/mac/build-cpp-lib.sh | 2 +- 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci-build-release-napi.yml b/.github/workflows/ci-build-release-napi.yml index 9d6a2f61..4ea46a8b 100644 --- a/.github/workflows/ci-build-release-napi.yml +++ b/.github/workflows/ci-build-release-napi.yml @@ -19,8 +19,8 @@ jobs: fail-fast: false matrix: arch: - - x86_64 -# - arm64 + - x64 + - arm64 nodejs: - 18 steps: @@ -38,16 +38,24 @@ jobs: path: pkg/mac/build key: ${{ runner.os }}-${{ matrix.arch }}-mac-${{ hashFiles('pkg/mac/build-cpp-deps-lib.sh') }} + - name: Add arch env vars + run: | + if [ "${{ matrix.arch }}" = "x64" ]; then + echo "ARCH=x86_64" >> $GITHUB_ENV + else + echo "ARCH=${{ matrix.target }}" >> $GITHUB_ENV + fi + - name: Build CPP dependencies lib if: steps.cache-dependencies.outputs.cache-hit != 'true' run: | - export ARCH=${{ matrix.arch }} + export ARCH=${{ env.ARCH }} pkg/mac/build-cpp-deps-lib.sh - name: Build CPP lib if: steps.cache-pulsar.outputs.cache-hit != 'true' run: | - export ARCH=${{ matrix.arch }} + export ARCH=${{ env.ARCH }} pkg/mac/build-cpp-lib.sh - name: Build Node binaries lib @@ -57,6 +65,7 @@ jobs: npx node-pre-gyp build --target_arch=${{ matrix.arch }} - name: Test loading Node binaries lib + if: matrix.arch == 'x64' run: | node pkg/load_test.js diff --git a/.github/workflows/ci-pr-validation.yml b/.github/workflows/ci-pr-validation.yml index bd22c205..8f353373 100644 --- a/.github/workflows/ci-pr-validation.yml +++ b/.github/workflows/ci-pr-validation.yml @@ -48,7 +48,8 @@ jobs: fail-fast: false matrix: arch: - - x86_64 + - x64 + - arm64 nodejs: - 18 steps: @@ -66,40 +67,40 @@ jobs: path: pkg/mac/build key: ${{ runner.os }}-${{ matrix.arch }}-mac-${{ hashFiles('pkg/mac/build-cpp-deps-lib.sh') }} - - name: Add env vars - shell: bash + - name: Add arch env vars run: | - if [ "${{ matrix.target }}" = "x86_64" ]; then - echo "TARGET=x64" >> $GITHUB_ENV + if [ "${{ matrix.arch }}" = "x64" ]; then + echo "ARCH=x86_64" >> $GITHUB_ENV else - echo "TARGET=${{ matrix.target }}" >> $GITHUB_ENV + echo "ARCH=${{ matrix.target }}" >> $GITHUB_ENV fi - name: Build CPP dependencies lib if: steps.cache-dependencies.outputs.cache-hit != 'true' run: | - export ARCH=${{ matrix.arch }} + export ARCH=${{ env.ARCH }} pkg/mac/build-cpp-deps-lib.sh - name: Build CPP lib if: steps.cache-pulsar.outputs.cache-hit != 'true' run: | - export ARCH=${{ matrix.arch }} + export ARCH=${{ env.ARCH }} pkg/mac/build-cpp-lib.sh - name: Build Node binaries lib run: | npm install --ignore-scripts - npx node-pre-gyp configure --target_arch=${{ env.TARGET }} - npx node-pre-gyp build --target_arch=${{ env.TARGET }} + npx node-pre-gyp configure --target_arch=${{ matrix.arch }} + npx node-pre-gyp build --target_arch=${{ matrix.arch }} - name: Test loading Node binaries lib + if: matrix.arch == 'x64' run: | node pkg/load_test.js - name: Package Node binaries lib run: | - npx node-pre-gyp package --target_arch=${{ env.TARGET }} + npx node-pre-gyp package --target_arch=${{ matrix.arch }} linux-napi: name: Build NAPI ${{matrix.image}} - Node ${{matrix.nodejs}} - ${{matrix.cpu.platform}} diff --git a/dependencies.yaml b/dependencies.yaml index a704c97b..c3969c5d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -24,5 +24,5 @@ protobuf: 3.20.0 zlib: 1.2.13 zstd: 1.5.2 snappy: 1.1.9 -openssl: 1.1.1q +openssl: 1.1.1s curl: 7.85.0 diff --git a/pkg/mac/build-cpp-deps-lib.sh b/pkg/mac/build-cpp-deps-lib.sh index 9226bce3..e2078d12 100755 --- a/pkg/mac/build-cpp-deps-lib.sh +++ b/pkg/mac/build-cpp-deps-lib.sh @@ -59,7 +59,6 @@ if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} - echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c if [ $ARCH = 'arm64' ]; then PLATFORM=darwin64-arm64-cc else @@ -147,8 +146,8 @@ if [ ! -f snappy-${SNAPPY_VERSION}.done ]; then curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz tar xfz ${SNAPPY_VERSION}.tar.gz pushd snappy-${SNAPPY_VERSION} - CXXFLAGS="-fPIC -O3 -arch ${ARCH} -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - cmake . -DCMAKE_INSTALL_PREFIX=$PREFIX -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF + CXXFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + cmake . -DCMAKE_OSX_ARCHITECTURES=${ARCH} -DCMAKE_INSTALL_PREFIX=$PREFIX -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF make -j16 make install touch .done @@ -175,7 +174,8 @@ if [ ! -f curl-${CURL_VERSION}.done ]; then --without-brotli \ --without-secure-transport \ --disable-ipv6 \ - --prefix=$PREFIX + --prefix=$PREFIX \ + --host=$ARCH-apple-darwin make -j16 install popd diff --git a/pkg/mac/build-cpp-lib.sh b/pkg/mac/build-cpp-lib.sh index ab829278..a0f4289e 100755 --- a/pkg/mac/build-cpp-lib.sh +++ b/pkg/mac/build-cpp-lib.sh @@ -29,9 +29,9 @@ cd $PULSAR_DIR curl -O -L "$BASE_URL"/apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz tar xfz apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz pushd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} - chmod +x ./build-support/merge_archives.sh rm -f CMakeCache.txt cmake . \ + -DCMAKE_OSX_ARCHITECTURES=${ARCH} \ -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \ -DCMAKE_INSTALL_PREFIX=$PULSAR_PREFIX \ -DCMAKE_BUILD_TYPE=Release \ From fd16e06487fad655544b780e6f0b02f674922a3b Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Mon, 28 Nov 2022 10:48:48 +0800 Subject: [PATCH 2/3] Revert upgrade the OpenSSL version. --- dependencies.yaml | 2 +- pkg/mac/build-cpp-deps-lib.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index c3969c5d..a704c97b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -24,5 +24,5 @@ protobuf: 3.20.0 zlib: 1.2.13 zstd: 1.5.2 snappy: 1.1.9 -openssl: 1.1.1s +openssl: 1.1.1q curl: 7.85.0 diff --git a/pkg/mac/build-cpp-deps-lib.sh b/pkg/mac/build-cpp-deps-lib.sh index e2078d12..d6092024 100755 --- a/pkg/mac/build-cpp-deps-lib.sh +++ b/pkg/mac/build-cpp-deps-lib.sh @@ -59,6 +59,7 @@ if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} + echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c if [ $ARCH = 'arm64' ]; then PLATFORM=darwin64-arm64-cc else From b95df05502d36c77578e36e0efbe42d3ccc98101 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Thu, 1 Dec 2022 11:47:54 +0800 Subject: [PATCH 3/3] Fix code review. --- .github/workflows/ci-build-release-napi.yml | 2 +- .github/workflows/ci-pr-validation.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-build-release-napi.yml b/.github/workflows/ci-build-release-napi.yml index 4ea46a8b..25dc9f92 100644 --- a/.github/workflows/ci-build-release-napi.yml +++ b/.github/workflows/ci-build-release-napi.yml @@ -43,7 +43,7 @@ jobs: if [ "${{ matrix.arch }}" = "x64" ]; then echo "ARCH=x86_64" >> $GITHUB_ENV else - echo "ARCH=${{ matrix.target }}" >> $GITHUB_ENV + echo "ARCH=${{ matrix.arch }}" >> $GITHUB_ENV fi - name: Build CPP dependencies lib diff --git a/.github/workflows/ci-pr-validation.yml b/.github/workflows/ci-pr-validation.yml index 8f353373..98e2edaf 100644 --- a/.github/workflows/ci-pr-validation.yml +++ b/.github/workflows/ci-pr-validation.yml @@ -72,7 +72,7 @@ jobs: if [ "${{ matrix.arch }}" = "x64" ]; then echo "ARCH=x86_64" >> $GITHUB_ENV else - echo "ARCH=${{ matrix.target }}" >> $GITHUB_ENV + echo "ARCH=${{ matrix.arch }}" >> $GITHUB_ENV fi - name: Build CPP dependencies lib