From 475422857e946377c84d1a31fba4bac2102cabe9 Mon Sep 17 00:00:00 2001 From: jackkav Date: Mon, 20 Jan 2025 17:10:09 +0100 Subject: [PATCH 1/9] add target arch flag --- .github/workflows/build-and-release.yaml | 2 +- .github/workflows/build-lint-test.yaml | 9 +++++++-- .github/workflows/new-version.yml | 2 +- .github/workflows/publish.yaml | 2 +- .github/workflows/sast.yml | 2 +- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 93bd94808..d75db981d 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -22,7 +22,7 @@ jobs: permissions: packages: write contents: write # publish sbom to GH releases/tag assets - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/build-lint-test.yaml b/.github/workflows/build-lint-test.yaml index bb5759afb..03fd27db6 100644 --- a/.github/workflows/build-lint-test.yaml +++ b/.github/workflows/build-lint-test.yaml @@ -17,7 +17,7 @@ jobs: permissions: packages: write contents: write # publish sbom to GH releases/tag assets - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v4 @@ -42,6 +42,7 @@ jobs: os: - macos-13-xlarge - ubuntu-22.04 + - ubuntu-22.04-arm libcurl-release: - 7.86.0 node-libcurl-cpp-std: @@ -50,7 +51,7 @@ jobs: - 20.18.0 include: # Lint - - os: ubuntu-latest + - os: ubuntu-22.04 node: 20.18.0 node-libcurl-cpp-std: c++20 libcurl-release: 7.86.0 @@ -103,6 +104,7 @@ jobs: RUN_TESTS=false \ RUN_PREGYP_CLEAN=false \ PUBLISH_BINARY=false \ + TARGET_ARCH=${{ matrix.os=='ubuntu-22.04-arm' && 'arm' || 'x64' }} \ ./scripts/ci/build.sh - name: 'Run lint' if: matrix.run-lint-and-tsc && steps.run_result.outputs.run_result != 'success' @@ -140,6 +142,7 @@ jobs: os: - macos-13-xlarge - ubuntu-22.04 + - ubuntu-22.04-arm libcurl-release: - 7.86.0 node: @@ -204,6 +207,8 @@ jobs: RUN_TESTS=true \ RUN_PREGYP_CLEAN=false \ PUBLISH_BINARY=false \ + PUBLISH_BINARY=false \ + TARGET_ARCH=${{ matrix.os=='ubuntu-22.04-arm' && 'arm' || 'x64' }} \ ./scripts/ci/build.sh - name: Upload artifacts if: always() && steps.run_result.outputs.run_result != 'success' diff --git a/.github/workflows/new-version.yml b/.github/workflows/new-version.yml index 9f994590f..c13602bd5 100644 --- a/.github/workflows/new-version.yml +++ b/.github/workflows/new-version.yml @@ -14,7 +14,7 @@ env: jobs: release: permissions: write-all - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 0f0fecd9d..d945003bb 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -7,7 +7,7 @@ on: jobs: publish: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 permissions: contents: write id-token: write # For using token to sign images diff --git a/.github/workflows/sast.yml b/.github/workflows/sast.yml index 9aaaa8dbd..071ea3427 100644 --- a/.github/workflows/sast.yml +++ b/.github/workflows/sast.yml @@ -13,7 +13,7 @@ on: jobs: semgrep: name: Semgrep SAST - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 permissions: # required for all workflows security-events: write From 5abe94771b96278dbec4e00aa74cf4fb11f9c59c Mon Sep 17 00:00:00 2001 From: jackkav Date: Tue, 21 Jan 2025 15:23:02 +0100 Subject: [PATCH 2/9] fix cache --- .github/workflows/build-and-release.yaml | 16 ++++++++-------- .github/workflows/build-lint-test.yaml | 19 +++++++++---------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index d75db981d..8238776f3 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -82,16 +82,16 @@ jobs: path: | ~/.node-gyp ~/deps - key: v4-${{ runner.os }}-libcurl-deps-cache-node-${{ matrix.node }}-fork + key: v4-${{ matrix.os }}-libcurl-deps-cache-node-${{ matrix.node }}-fork restore-keys: | - v4-${{ runner.os }}-libcurl-deps-cache-node-${{ matrix.node }}-fork + v4-${{ matrix.os }}-libcurl-deps-cache-node-${{ matrix.node }}-fork - name: 'Set GIT_TAG' if: startsWith(github.ref, 'refs/tags') run: echo "GIT_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: 'Publish Binary' run: | pip install packaging - GIT_COMMIT=${{ github.sha }} GIT_TAG=$GIT_TAG RUNNER_OS=${{ runner.os }} ./scripts/ci/build.sh + GIT_COMMIT=${{ github.sha }} GIT_TAG=$GIT_TAG RUNNER_OS=${{ matrix.os }} ./scripts/ci/build.sh - name: Upload artifacts if: always() uses: actions/upload-artifact@v4 @@ -139,10 +139,10 @@ jobs: uses: actions/cache@v4 with: path: ~/Library/Caches/electron - key: v1-${{ runner.os }}-electron-cache-${{ matrix.electron-version }} + key: v1-${{ matrix.os }}-electron-cache-${{ matrix.electron-version }} restore-keys: | - v1-${{ runner.os }}-electron-cache-${{ matrix.electron-version }} - v1-${{ runner.os }}-electron-cache- + v1-${{ matrix.os }}-electron-cache-${{ matrix.electron-version }} + v1-${{ matrix.os }}-electron-cache- - name: Restore libcurl deps cache uses: actions/cache@v4 id: libcurl-deps-cache @@ -150,9 +150,9 @@ jobs: path: | ~/.node-gyp ~/deps - key: v4-${{ runner.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }}-fork + key: v4-${{ matrix.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }}-fork restore-keys: | - v4-${{ runner.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }}-fork + v4-${{ matrix.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }}-fork - name: 'Set GIT_TAG' if: startsWith(github.ref, 'refs/tags') run: echo "GIT_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV diff --git a/.github/workflows/build-lint-test.yaml b/.github/workflows/build-lint-test.yaml index 03fd27db6..ab2c00d53 100644 --- a/.github/workflows/build-lint-test.yaml +++ b/.github/workflows/build-lint-test.yaml @@ -1,13 +1,12 @@ name: build-lint-test -on: - pull_request: +on: [push, workflow_dispatch] env: LIBCURL_RELEASE: 'LATEST' concurrency: - group: ${{ github.head_ref }} + group: ${{ github.ref }} cancel-in-progress: true # all jobs here must have a matrix identical to the ones inside build-and-release.yaml @@ -95,9 +94,9 @@ jobs: path: | ~/.node-gyp ~/deps - key: v4-build-lint-test-${{ runner.os }}-libcurl-deps-cache-node-${{ matrix.node }} + key: v4-build-lint-test-${{ matrix.os }}-libcurl-deps-cache-node-${{ matrix.node }} restore-keys: | - v4-build-lint-test-${{ runner.os }}-libcurl-deps-cache-node-${{ matrix.node }} + v4-build-lint-test-${{ matrix.os }}-libcurl-deps-cache-node-${{ matrix.node }} - name: Build node-libcurl ${{ matrix.node }} if: steps.run_result.outputs.run_result != 'success' run: | @@ -184,10 +183,10 @@ jobs: uses: actions/cache@v4 with: path: ~/Library/Caches/electron - key: v1-${{ runner.os }}-electron-cache-${{ matrix.electron-version }} + key: v1-${{ matrix.os }}-electron-cache-${{ matrix.electron-version }} restore-keys: | - v1-${{ runner.os }}-electron-cache-${{ matrix.electron-version }} - v1-${{ runner.os }}-electron-cache- + v1-${{ matrix.os }}-electron-cache-${{ matrix.electron-version }} + v1-${{ matrix.os }}-electron-cache- - name: Restore libcurl deps cache uses: actions/cache@v4 id: libcurl-deps-cache @@ -195,9 +194,9 @@ jobs: path: | ~/.node-gyp ~/deps - key: v4-build-lint-test-${{ runner.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }} + key: v4-build-lint-test-${{ matrix.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }} restore-keys: | - v4-build-lint-test-${{ runner.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }} + v4-build-lint-test-${{ matrix.os }}-libcurl-deps-cache-electron-${{ matrix.electron-version }} - name: 'Set GIT_TAG' if: startsWith(github.ref, 'refs/tags') run: echo "GIT_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV From 8bcade8f0a6bcba3abea4aefe532dbfe3f808090 Mon Sep 17 00:00:00 2001 From: jackkav Date: Tue, 21 Jan 2025 15:25:10 +0100 Subject: [PATCH 3/9] fix target_arch --- .github/workflows/build-lint-test.yaml | 4 ++-- scripts/ci/build.sh | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-lint-test.yaml b/.github/workflows/build-lint-test.yaml index ab2c00d53..e29c7b1b2 100644 --- a/.github/workflows/build-lint-test.yaml +++ b/.github/workflows/build-lint-test.yaml @@ -103,7 +103,7 @@ jobs: RUN_TESTS=false \ RUN_PREGYP_CLEAN=false \ PUBLISH_BINARY=false \ - TARGET_ARCH=${{ matrix.os=='ubuntu-22.04-arm' && 'arm' || 'x64' }} \ + TARGET_ARCH=${{ matrix.os=='ubuntu-22.04-arm' && 'arm64' || 'x64' }} \ ./scripts/ci/build.sh - name: 'Run lint' if: matrix.run-lint-and-tsc && steps.run_result.outputs.run_result != 'success' @@ -207,7 +207,7 @@ jobs: RUN_PREGYP_CLEAN=false \ PUBLISH_BINARY=false \ PUBLISH_BINARY=false \ - TARGET_ARCH=${{ matrix.os=='ubuntu-22.04-arm' && 'arm' || 'x64' }} \ + TARGET_ARCH=${{ matrix.os=='ubuntu-22.04-arm' && 'arm64' || 'x64' }} \ ./scripts/ci/build.sh - name: Upload artifacts if: always() && steps.run_result.outputs.run_result != 'success' diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index e997adf01..d23b62753 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -60,6 +60,12 @@ LOGS_FOLDER=${BUILD_LOGS_FOLDER:-./logs} mkdir -p $LOGS_FOLDER +# ia32, x64, armv7, etc +target_arch=${TARGET_ARCH:-"x64"} +# the build system of some dependencies (e.g. zstd and openldap) seems to read +# and use this variable, so unset it here to avoid making them fail +unset TARGET_ARCH + # The following two, libunistring and libidn2, are only necessary if building libcurl >= 7.53 # However we are going to build then anyway, they are not that slow to build. @@ -243,8 +249,8 @@ fi LIBCURL_DEST_FOLDER=$PREFIX_DIR/deps/libcurl echo "Building libcurl v$LIBCURL_RELEASE - Latest is v$LATEST_LIBCURL_RELEASE" ./scripts/ci/build-libcurl.sh $LIBCURL_RELEASE $LIBCURL_DEST_FOLDER || (echo "libcurl failed build log:" && cat $LIBCURL_DEST_FOLDER/source/$LIBCURL_RELEASE/config.log && exit 1) -echo "libcurl successful build log:" -cat $LIBCURL_DEST_FOLDER/source/$LIBCURL_RELEASE/config.log +# echo "libcurl successful build log:" +# cat $LIBCURL_DEST_FOLDER/source/$LIBCURL_RELEASE/config.log export LIBCURL_BUILD_FOLDER=$LIBCURL_DEST_FOLDER/build/$LIBCURL_RELEASE ls -al $LIBCURL_BUILD_FOLDER/lib @@ -307,8 +313,6 @@ else fi target=`echo $target | sed 's/^v//'` -# ia32, x64, armv7, etc -target_arch=${TARGET_ARCH:-"x64"} NODE_LIBCURL_CPP_STD=${NODE_LIBCURL_CPP_STD:-$(node $curr_dirname/../cpp-std.js)} From 6dd0c0a5dc8d98ed90d1e4a878b6080a094ccc21 Mon Sep 17 00:00:00 2001 From: jackkav Date: Tue, 21 Jan 2025 15:29:18 +0100 Subject: [PATCH 4/9] fix electron jobs --- .github/workflows/build-lint-test.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-lint-test.yaml b/.github/workflows/build-lint-test.yaml index e29c7b1b2..764747e9d 100644 --- a/.github/workflows/build-lint-test.yaml +++ b/.github/workflows/build-lint-test.yaml @@ -166,11 +166,12 @@ jobs: ${{ github.run_id }}-${{ github.job }}- - id: run_result run: cat run_result 2>/dev/null || echo 'default' - - name: Set up Homebrew - id: set-up-homebrew - uses: Homebrew/actions/setup-homebrew@master - - name: Install Needed packages + - if: runner.os == 'macOS' + name: Install Needed packages on macOS run: brew install coreutils wget automake libtool cmake gnu-sed m4 autoconf groff + - if: runner.os == 'Linux' + name: Install Needed packages on Linux + run: sudo apt-get install -y cmake - name: Checkout uses: actions/checkout@v4 - name: Setup Node @@ -182,7 +183,9 @@ jobs: - name: Restore Electron Cache uses: actions/cache@v4 with: - path: ~/Library/Caches/electron + path: | + ~/Library/Caches/electron + ~/.cache/electron key: v1-${{ matrix.os }}-electron-cache-${{ matrix.electron-version }} restore-keys: | v1-${{ matrix.os }}-electron-cache-${{ matrix.electron-version }} From 5c36128a4b68d4b23a4a5990cd658511be9fdd33 Mon Sep 17 00:00:00 2001 From: jackkav Date: Wed, 22 Jan 2025 16:54:41 +0100 Subject: [PATCH 5/9] skip failing exception tests --- test/curl/easy.spec.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/curl/easy.spec.ts b/test/curl/easy.spec.ts index 6667a1d3f..c985a2ee2 100644 --- a/test/curl/easy.spec.ts +++ b/test/curl/easy.spec.ts @@ -66,8 +66,8 @@ describe('easy', () => { 'Return value from the HEADER callback must be an integer.', ) }) - - it('READFUNCTION - should rethrow error', () => { + // Below tests are skipped because we don't yet have an explanation why recent changes to github actions are causing them to fail + it.skip('READFUNCTION - should rethrow error', () => { curl.setOpt('UPLOAD', true) // @ts-ignore curl.setOpt('READFUNCTION', () => { @@ -76,7 +76,7 @@ describe('easy', () => { const perform = () => curl.perform() perform.should.throw('Error thrown on callback') }) - it('READFUNCTION - should throw error if has invalid return type', () => { + it.skip('READFUNCTION - should throw error if has invalid return type', () => { curl.setOpt('UPLOAD', true) // @ts-ignore curl.setOpt('READFUNCTION', () => { @@ -89,7 +89,7 @@ describe('easy', () => { }) if (Curl.isVersionGreaterOrEqualThan(7, 64, 0)) { - it('TRAILERFUNCTION - should rethrow error', () => { + it.skip('TRAILERFUNCTION - should rethrow error', () => { curl.setOpt('UPLOAD', true) curl.setOpt('HTTPHEADER', ['x-random-header: random-value']) // @ts-ignore @@ -107,7 +107,7 @@ describe('easy', () => { const perform = () => curl.perform() perform.should.throw('Error thrown on callback') }) - it('TRAILERFUNCTION - should throw error if has invalid return type', () => { + it.skip('TRAILERFUNCTION - should throw error if has invalid return type', () => { curl.setOpt('UPLOAD', true) curl.setOpt('HTTPHEADER', ['x-random-header: random-value']) // @ts-ignore From c25fba0a0da7e00ec238b88a69cfe14249ffdb53 Mon Sep 17 00:00:00 2001 From: jackkav Date: Wed, 22 Jan 2025 17:38:59 +0100 Subject: [PATCH 6/9] add arm to publish action --- .github/workflows/build-and-release.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 8238776f3..103de1394 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -49,6 +49,7 @@ jobs: os: - macos-13-xlarge - ubuntu-22.04 + - ubuntu-22.04-arm libcurl-release: - 7.86.0 node-libcurl-cpp-std: @@ -109,6 +110,7 @@ jobs: os: - macos-13-xlarge - ubuntu-22.04 + - ubuntu-22.04-arm libcurl-release: - 7.86.0 node: From 0083eccb5d9c77c11773bbf162aa86e87b68c9eb Mon Sep 17 00:00:00 2001 From: Filipe Freire Date: Thu, 23 Jan 2025 10:58:26 +0000 Subject: [PATCH 7/9] remove test skip and change url on easy.spec.ts --- test/curl/easy.spec.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/curl/easy.spec.ts b/test/curl/easy.spec.ts index c985a2ee2..c2f3b30ef 100644 --- a/test/curl/easy.spec.ts +++ b/test/curl/easy.spec.ts @@ -8,7 +8,7 @@ import 'should' import { Curl, CurlCode, Easy } from '../../lib' -const url = 'http://example.com/' +const url = 'http://httpbin.org/put' // This is the only test that does not uses a express server // It makes a request to a live server, which can cause issues if there are network problems @@ -67,7 +67,7 @@ describe('easy', () => { ) }) // Below tests are skipped because we don't yet have an explanation why recent changes to github actions are causing them to fail - it.skip('READFUNCTION - should rethrow error', () => { + it('READFUNCTION - should rethrow error', () => { curl.setOpt('UPLOAD', true) // @ts-ignore curl.setOpt('READFUNCTION', () => { @@ -76,7 +76,7 @@ describe('easy', () => { const perform = () => curl.perform() perform.should.throw('Error thrown on callback') }) - it.skip('READFUNCTION - should throw error if has invalid return type', () => { + it('READFUNCTION - should throw error if has invalid return type', () => { curl.setOpt('UPLOAD', true) // @ts-ignore curl.setOpt('READFUNCTION', () => { @@ -89,7 +89,7 @@ describe('easy', () => { }) if (Curl.isVersionGreaterOrEqualThan(7, 64, 0)) { - it.skip('TRAILERFUNCTION - should rethrow error', () => { + it('TRAILERFUNCTION - should rethrow error', () => { curl.setOpt('UPLOAD', true) curl.setOpt('HTTPHEADER', ['x-random-header: random-value']) // @ts-ignore @@ -107,7 +107,7 @@ describe('easy', () => { const perform = () => curl.perform() perform.should.throw('Error thrown on callback') }) - it.skip('TRAILERFUNCTION - should throw error if has invalid return type', () => { + it('TRAILERFUNCTION - should throw error if has invalid return type', () => { curl.setOpt('UPLOAD', true) curl.setOpt('HTTPHEADER', ['x-random-header: random-value']) // @ts-ignore From 343ef29150d7ff3909bdc625e4dba96743d49cbf Mon Sep 17 00:00:00 2001 From: Filipe Freire Date: Thu, 23 Jan 2025 10:59:46 +0000 Subject: [PATCH 8/9] revert concurrency group setting ref to head_ref --- .github/workflows/build-lint-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-lint-test.yaml b/.github/workflows/build-lint-test.yaml index 764747e9d..0e024e3f6 100644 --- a/.github/workflows/build-lint-test.yaml +++ b/.github/workflows/build-lint-test.yaml @@ -6,7 +6,7 @@ env: LIBCURL_RELEASE: 'LATEST' concurrency: - group: ${{ github.ref }} + group: ${{ github.head_ref }} cancel-in-progress: true # all jobs here must have a matrix identical to the ones inside build-and-release.yaml From 23d9e79a3e0ccb7a2af62418464b4b55208215ee Mon Sep 17 00:00:00 2001 From: Filipe Freire Date: Thu, 23 Jan 2025 11:33:51 +0000 Subject: [PATCH 9/9] Update test/curl/easy.spec.ts --- test/curl/easy.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/curl/easy.spec.ts b/test/curl/easy.spec.ts index c2f3b30ef..a2665444a 100644 --- a/test/curl/easy.spec.ts +++ b/test/curl/easy.spec.ts @@ -66,7 +66,6 @@ describe('easy', () => { 'Return value from the HEADER callback must be an integer.', ) }) - // Below tests are skipped because we don't yet have an explanation why recent changes to github actions are causing them to fail it('READFUNCTION - should rethrow error', () => { curl.setOpt('UPLOAD', true) // @ts-ignore