From 6cc928c08ebd8de72a5b549173e6825a23ebbd80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 19 May 2025 17:26:23 +0200 Subject: [PATCH 1/8] GH-3: Enable CI --- .github/workflows/test.yaml | 113 ++++++++++++++++++++++++++++++ LICENSE.txt | 0 NOTICE.txt | 0 ci/docker/debian-12-js.dockerfile | 28 ++++++++ ci/scripts/js_build.sh | 60 ++++++++++++++++ ci/scripts/js_test.sh | 31 ++++++++ docker-compose.yml | 54 ++++++++++++++ 7 files changed, 286 insertions(+) create mode 100644 .github/workflows/test.yaml create mode 100644 LICENSE.txt create mode 100644 NOTICE.txt create mode 100644 ci/docker/debian-12-js.dockerfile create mode 100755 ci/scripts/js_build.sh create mode 100755 ci/scripts/js_test.sh create mode 100644 docker-compose.yml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 00000000..965ab32d --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: NodeJS + + +on: + push: + branches: + - '**' + - '!dependabot/**' + tags: + - '**' + pull_request: + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + + docker: + name: AMD64 Debian 12 NodeJS 18 + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + steps: + - name: Checkout Arrow JS + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + with: + fetch-depth: 0 + - name: Execute Docker Build + run: | + docker compose run debian-js + + macos: + name: AMD64 macOS 13 NodeJS ${{ matrix.node }} + runs-on: macos-13 + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + node: [18] + steps: + - name: Checkout Arrow JS + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Jest Cache + uses: actions/cache@v4 + with: + path: js/.jest-cache + key: js-jest-cache-${{ runner.os }}-${{ hashFiles('js/src/**/*.ts', 'js/test/**/*.ts', 'js/yarn.lock') }} + restore-keys: js-jest-cache-${{ runner.os }}- + - name: Install NodeJS + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - name: Build + shell: bash + run: ci/scripts/js_build.sh $(pwd) build + - name: Test + shell: bash + run: ci/scripts/js_test.sh $(pwd) build + + windows: + name: AMD64 Windows NodeJS ${{ matrix.node }} + runs-on: windows-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + node: [18] + steps: + - name: Checkout Arrow JS + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Jest Cache + uses: actions/cache@v4 + with: + path: js/.jest-cache + key: js-jest-cache-${{ runner.os }}-${{ hashFiles('js/src/**/*.ts', 'js/test/**/*.ts', 'js/yarn.lock') }} + restore-keys: js-jest-cache-${{ runner.os }}- + - name: Install NodeJS + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - name: Build + shell: bash + run: ci/scripts/js_build.sh $(pwd) build + - name: Test + shell: bash + run: ci/scripts/js_test.sh $(pwd) build \ No newline at end of file diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000..e69de29b diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 00000000..e69de29b diff --git a/ci/docker/debian-12-js.dockerfile b/ci/docker/debian-12-js.dockerfile new file mode 100644 index 00000000..1c49e77e --- /dev/null +++ b/ci/docker/debian-12-js.dockerfile @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +ARG node=18 +FROM ${arch}/node:${node} + +ENV NODE_NO_WARNINGS=1 + +# install rsync for copying the generated documentation +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends rsync && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/ci/scripts/js_build.sh b/ci/scripts/js_build.sh new file mode 100755 index 00000000..69738c03 --- /dev/null +++ b/ci/scripts/js_build.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex + +source_dir=${1} +build_dir=${2} + +: ${BUILD_DOCS_JS:=OFF} + +# https://github.com/apache/arrow/issues/41429 +# TODO: We want to out-of-source build. This is a workaround. We copy +# all needed files to the build directory from the source directory +# and build in the build directory. +rm -rf ${build_dir}/js +mkdir -p ${build_dir} +# To be uncommented when https://github.com/apache/arrow-js/issues/2 +cp -aL ${source_dir}/LICENSE.txt ${build_dir}/ +cp -aL ${source_dir}/NOTICE.txt ${build_dir}/ +cp -aL ${source_dir} ${build_dir}/js +pushd ${build_dir}/js + +yarn --immutable +yarn lint:ci +yarn build + +if [ "${BUILD_DOCS_JS}" == "ON" ]; then + # If apache or upstream are defined use those as remote. + # Otherwise use origin which could be a fork on PRs. + if [ "$(git -C ${source_dir} config --get remote.apache.url)" == "git@github.com:apache/arrow-js.git" ]; then + yarn doc --gitRemote apache + elif [[ "$(git -C ${source_dir}config --get remote.upstream.url)" =~ "https://github.com/apache/arrow-js" ]]; then + yarn doc --gitRemote upstream + elif [[ "$(basename -s .git $(git -C ${source_dir} config --get remote.origin.url))" == "arrow-js" ]]; then + yarn doc + else + echo "Failed to build docs because the remote is not set correctly. Please set the origin or upstream remote to https://github.com/apache/arrow-js.git or the apache remote to git@github.com:apache/arrow-js.git." + exit 0 + fi + mkdir -p ${build_dir}/docs/js + rsync -a doc/ ${build_dir}/docs/js +fi + +popd diff --git a/ci/scripts/js_test.sh b/ci/scripts/js_test.sh new file mode 100755 index 00000000..3dbe2dd9 --- /dev/null +++ b/ci/scripts/js_test.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex + +source_dir=${1} +build_dir=${2}/js + +pushd ${build_dir} + +yarn lint +yarn test +yarn test:bundle + +popd diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..bb1f3d7d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Usage +# ----- +# +# The docker compose file is parametrized using environment variables, the +# defaults are set in .env file. +# +# Example: +# $ ARCH=arm64v8 docker compose build java +# $ ARCH=arm64v8 docker compose run java + +volumes: + debian-ccache: + name: ${ARCH}-debian-${DEBIAN}-ccache + +services: + debian-js: + # Usage: + # docker compose build debian-js + # docker compose run debian-js + image: ${REPO}:${ARCH}-debian-${DEBIAN}-js-${NODE} + build: + context: . + dockerfile: ci/docker/debian-${DEBIAN}-js.dockerfile + cache_from: + - ${REPO}:${ARCH}-debian-${DEBIAN}-js-${NODE} + args: + arch: ${ARCH} + node: ${NODE} + volumes: &debian-volumes + - .:/arrow-js:delegated + - ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated + environment: + BUILD_DOCS_JS: "ON" + command: &js-command > + /bin/bash -c " + /arrow-js/ci/scripts/js_build.sh /arrow-js /build && + /arrow-js/ci/scripts/js_test.sh /arrow-js /build" \ No newline at end of file From 38ef4f831179e1d321970b31fdc1873e5ecee269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 19 May 2025 17:48:44 +0200 Subject: [PATCH 2/8] Minor improvements + add .env file --- .env | 48 +++++++++++++++++++++++++++++++++++++ .github/workflows/test.yaml | 8 +++---- 2 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 00000000..0fe316bc --- /dev/null +++ b/.env @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# All of the following environment variables are required to set default values +# for the parameters in docker-compose.yml. + +# empty prefix means that the docker-compose configuration will use named +# volumes which potentially improves the performance on docker for macos and +# docker for windows, it also prevents the contamination of the source +# directory +# a non-empty prefix means that directories from the host are bind-mounted +# into the container, it should be set to ".docker/" on github actions to keep +# the cache plugin functional +DOCKER_VOLUME_PREFIX= + +# turn on inline build cache, this is a docker buildx feature documented +# at https://github.com/docker/buildx#--cache-tonametypetypekeyvalue +BUILDKIT_INLINE_CACHE=1 +COMPOSE_DOCKER_CLI_BUILD=1 +DOCKER_BUILDKIT=1 + +# different architecture notations +ARCH=amd64 +ARCH_ALIAS=x86_64 +ARCH_SHORT=amd64 + +# The default Debian version to use for the docker image +DEBIAN=12 + +# Default repository to pull and push images from +REPO=ghcr.io/apache/arrow-js-dev + +# Default versions for various dependencies +NODE=18 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 965ab32d..32761f4c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -67,8 +67,8 @@ jobs: - name: Jest Cache uses: actions/cache@v4 with: - path: js/.jest-cache - key: js-jest-cache-${{ runner.os }}-${{ hashFiles('js/src/**/*.ts', 'js/test/**/*.ts', 'js/yarn.lock') }} + path: .jest-cache + key: js-jest-cache-${{ runner.os }}-${{ hashFiles('src/**/*.ts', 'test/**/*.ts', 'yarn.lock') }} restore-keys: js-jest-cache-${{ runner.os }}- - name: Install NodeJS uses: actions/setup-node@v4 @@ -98,8 +98,8 @@ jobs: - name: Jest Cache uses: actions/cache@v4 with: - path: js/.jest-cache - key: js-jest-cache-${{ runner.os }}-${{ hashFiles('js/src/**/*.ts', 'js/test/**/*.ts', 'js/yarn.lock') }} + path: .jest-cache + key: js-jest-cache-${{ runner.os }}-${{ hashFiles('src/**/*.ts', 'test/**/*.ts', 'yarn.lock') }} restore-keys: js-jest-cache-${{ runner.os }}- - name: Install NodeJS uses: actions/setup-node@v4 From 97c63079d8960fc91e262c609f595eaf76e2fb36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 19 May 2025 19:05:33 +0200 Subject: [PATCH 3/8] Try fixing macOS build --- ci/scripts/js_build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/scripts/js_build.sh b/ci/scripts/js_build.sh index 69738c03..d4910b0f 100755 --- a/ci/scripts/js_build.sh +++ b/ci/scripts/js_build.sh @@ -36,7 +36,10 @@ cp -aL ${source_dir}/NOTICE.txt ${build_dir}/ cp -aL ${source_dir} ${build_dir}/js pushd ${build_dir}/js -yarn --immutable +# Clear yarn cache first +yarn cache clean + +yarn install --check-files --immutable yarn lint:ci yarn build From 42ed6bde5bb7bd5462aa264afcbad1bd1270379b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 19 May 2025 19:08:56 +0200 Subject: [PATCH 4/8] Try fixing eslint-plugin-gunicorn on macos --- ci/scripts/js_build.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ci/scripts/js_build.sh b/ci/scripts/js_build.sh index d4910b0f..b52a4e45 100755 --- a/ci/scripts/js_build.sh +++ b/ci/scripts/js_build.sh @@ -36,10 +36,13 @@ cp -aL ${source_dir}/NOTICE.txt ${build_dir}/ cp -aL ${source_dir} ${build_dir}/js pushd ${build_dir}/js -# Clear yarn cache first -yarn cache clean +yarn --immutable + +if ! yarn list --pattern eslint-plugin-unicorn | grep -q eslint-plugin-unicorn; then + echo "eslint-plugin-unicorn not installed properly, trying explicit installation" + yarn add --dev eslint-plugin-unicorn +fi -yarn install --check-files --immutable yarn lint:ci yarn build From a4c8205b3d18daee4eb7a9b28acffcbc8d5c3481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 19 May 2025 19:28:43 +0200 Subject: [PATCH 5/8] Minor fixes and use /tmp/build instead of internal dir --- .github/workflows/test.yaml | 11 +++++++---- ci/scripts/js_test.sh | 3 +-- docker-compose.yml | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 32761f4c..fc40fa6b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -34,6 +34,9 @@ concurrency: permissions: contents: read +env: + DOCKER_VOLUME_PREFIX: ".docker/" + jobs: docker: @@ -76,10 +79,10 @@ jobs: node-version: ${{ matrix.node }} - name: Build shell: bash - run: ci/scripts/js_build.sh $(pwd) build + run: ci/scripts/js_build.sh $(pwd) /tmp/build - name: Test shell: bash - run: ci/scripts/js_test.sh $(pwd) build + run: ci/scripts/js_test.sh /tmp/build windows: name: AMD64 Windows NodeJS ${{ matrix.node }} @@ -107,7 +110,7 @@ jobs: node-version: ${{ matrix.node }} - name: Build shell: bash - run: ci/scripts/js_build.sh $(pwd) build + run: ci/scripts/js_build.sh $(pwd) /tmp/build - name: Test shell: bash - run: ci/scripts/js_test.sh $(pwd) build \ No newline at end of file + run: ci/scripts/js_test.sh /tmp/build diff --git a/ci/scripts/js_test.sh b/ci/scripts/js_test.sh index 3dbe2dd9..a25178d1 100755 --- a/ci/scripts/js_test.sh +++ b/ci/scripts/js_test.sh @@ -19,8 +19,7 @@ set -ex -source_dir=${1} -build_dir=${2}/js +build_dir=${1}/js pushd ${build_dir} diff --git a/docker-compose.yml b/docker-compose.yml index bb1f3d7d..b5860f51 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,4 +51,4 @@ services: command: &js-command > /bin/bash -c " /arrow-js/ci/scripts/js_build.sh /arrow-js /build && - /arrow-js/ci/scripts/js_test.sh /arrow-js /build" \ No newline at end of file + /arrow-js/ci/scripts/js_test.sh /build" From 27ae089e6b6ff44b17767d21b2f774cd03512ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 19 May 2025 19:46:49 +0200 Subject: [PATCH 6/8] Remove possibly unnecessary check for eslint-plugin-unicorn and fix comment --- ci/scripts/js_build.sh | 5 ----- docker-compose.yml | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/ci/scripts/js_build.sh b/ci/scripts/js_build.sh index b52a4e45..58574297 100755 --- a/ci/scripts/js_build.sh +++ b/ci/scripts/js_build.sh @@ -38,11 +38,6 @@ pushd ${build_dir}/js yarn --immutable -if ! yarn list --pattern eslint-plugin-unicorn | grep -q eslint-plugin-unicorn; then - echo "eslint-plugin-unicorn not installed properly, trying explicit installation" - yarn add --dev eslint-plugin-unicorn -fi - yarn lint:ci yarn build diff --git a/docker-compose.yml b/docker-compose.yml index b5860f51..0786ef22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,8 +22,8 @@ # defaults are set in .env file. # # Example: -# $ ARCH=arm64v8 docker compose build java -# $ ARCH=arm64v8 docker compose run java +# $ ARCH=amd64 docker compose build debian-js +# $ ARCH=amd64 docker compose run debian-js volumes: debian-ccache: From 095d6716d1b752a4e2d41a1172c9cfc5dc3176a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 19 May 2025 20:12:37 +0200 Subject: [PATCH 7/8] Reduce unnecessary long timeout --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fc40fa6b..eaf9c941 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -43,7 +43,7 @@ jobs: name: AMD64 Debian 12 NodeJS 18 runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 + timeout-minutes: 45 steps: - name: Checkout Arrow JS uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 From a8858b1faf6513644d6852e4cd7d09aa28a4f525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 19 May 2025 20:16:34 +0200 Subject: [PATCH 8/8] Remove stray comment from tests --- ci/scripts/js_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/js_build.sh b/ci/scripts/js_build.sh index 58574297..52d06f0f 100755 --- a/ci/scripts/js_build.sh +++ b/ci/scripts/js_build.sh @@ -30,7 +30,7 @@ build_dir=${2} # and build in the build directory. rm -rf ${build_dir}/js mkdir -p ${build_dir} -# To be uncommented when https://github.com/apache/arrow-js/issues/2 + cp -aL ${source_dir}/LICENSE.txt ${build_dir}/ cp -aL ${source_dir}/NOTICE.txt ${build_dir}/ cp -aL ${source_dir} ${build_dir}/js