From 975a3918e5c82cf9629137b27685a5bab0315c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 28 Oct 2021 13:52:57 +0200 Subject: [PATCH 01/24] Exercise documentation builds on the main branch --- .github/workflows/docs.yml | 62 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000000..0605827d3d6 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Docs + +on: + push: + +env: + ARROW_ENABLE_TIMING_TESTS: OFF + DOCKER_VOLUME_PREFIX: ".docker/" + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + +jobs: + + docker: + name: AMD64 Ubuntu 20.10 Docs & Reference + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + run: ci/scripts/util_cleanup.sh + - name: Cache Docker Volumes + uses: actions/cache@v2 + with: + path: .docker + key: ubuntu-docs-${{ hashFiles('cpp/**') }} + restore-keys: ubuntu-docs- + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: archery docker run ubuntu-docs + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + run: archery docker push ubuntu-docs \ No newline at end of file From 91d09586e33eeee0a3d92acbeb2886774471c398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 28 Oct 2021 13:59:51 +0200 Subject: [PATCH 02/24] Trigger on pull request --- .github/workflows/docs.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0605827d3d6..1c40f575c98 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -19,6 +19,19 @@ name: Docs on: push: + pull_request: + paths: + - '.github/workflows/docs.yml' + - 'ci/docker/linux-apt-docs.dockerfile' + - 'ci/docker/linux-apt-python-3.dockerfile' + - 'ci/docker/ubuntu-20.10-cpp.dockerfile' + - 'ci/scripts/c_glib_build.sh' + - 'ci/scripts/cpp_build.sh' + - 'ci/scripts/docs_build.sh' + - 'ci/scripts/java_build.sh' + - 'ci/scripts/js_build.sh' + - 'ci/scripts/python_build.sh' + - 'ci/scripts/r_build.sh' env: ARROW_ENABLE_TIMING_TESTS: OFF @@ -59,4 +72,4 @@ jobs: - name: Docker Push if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' continue-on-error: true - run: archery docker push ubuntu-docs \ No newline at end of file + run: archery docker push ubuntu-docs From dc4dd10b46ad1360082f2998ea6984877b729e74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 28 Oct 2021 17:40:20 +0200 Subject: [PATCH 03/24] Build docs individually for each impls --- .github/workflows/docs.yml | 2 +- ci/scripts/c_glib_build.sh | 4 +++- ci/scripts/python_build.sh | 5 +++++ docker-compose.yml | 20 ++++++++++---------- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1c40f575c98..f844be14799 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -42,7 +42,7 @@ env: jobs: docker: - name: AMD64 Ubuntu 20.10 Docs & Reference + name: AMD64 Ubuntu 20.10 Complete Documentation runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 60 diff --git a/ci/scripts/c_glib_build.sh b/ci/scripts/c_glib_build.sh index ce3cea18e71..5e45bdaee78 100755 --- a/ci/scripts/c_glib_build.sh +++ b/ci/scripts/c_glib_build.sh @@ -21,7 +21,9 @@ set -ex source_dir=${1}/c_glib build_dir=${2}/c_glib -: ${ARROW_GLIB_GTK_DOC:=false} +with_docs=${3:-false} + +: ${ARROW_GLIB_GTK_DOC:=${with_docs}} : ${ARROW_GLIB_DEVELOPMENT_MODE:=false} export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig diff --git a/ci/scripts/python_build.sh b/ci/scripts/python_build.sh index ec6d723b2a7..6cd1880addf 100755 --- a/ci/scripts/python_build.sh +++ b/ci/scripts/python_build.sh @@ -19,6 +19,7 @@ set -ex +arrow_dir=${1} source_dir=${1}/python build_dir=${2}/python @@ -52,3 +53,7 @@ ${PYTHON:-python} \ --record $relative_build_dir/record.txt popd + +if [ "${with_docs}" == "true" ]; then + sphinx-build -b html -j ${ncpus} ${arrow_dir}/docs/source +fi diff --git a/docker-compose.yml b/docker-compose.yml index a38210cc5e0..106b6b725dd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -518,7 +518,7 @@ services: command: &c-glib-command > /bin/bash -c " /arrow/ci/scripts/cpp_build.sh /arrow /build && - /arrow/ci/scripts/c_glib_build.sh /arrow /build && + /arrow/ci/scripts/c_glib_build.sh /arrow /build true && /arrow/ci/scripts/c_glib_test.sh /arrow /build" ubuntu-c-glib: @@ -576,7 +576,7 @@ services: command: &ruby-command > /bin/bash -c " /arrow/ci/scripts/cpp_build.sh /arrow /build && - /arrow/ci/scripts/c_glib_build.sh /arrow /build && + /arrow/ci/scripts/c_glib_build.sh /arrow /build true && /arrow/ci/scripts/c_glib_test.sh /arrow /build && /arrow/ci/scripts/ruby_test.sh /arrow /build" @@ -631,7 +631,7 @@ services: volumes: *conda-volumes command: &python-conda-command ["/arrow/ci/scripts/cpp_build.sh /arrow /build && - /arrow/ci/scripts/python_build.sh /arrow /build && + /arrow/ci/scripts/python_build.sh /arrow /build true && /arrow/ci/scripts/python_test.sh /arrow"] ubuntu-cuda-python: @@ -1135,6 +1135,7 @@ services: /bin/bash -c " /arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/python_build.sh /arrow /build && + /arrow/ci/scripts/r_build.sh /arrow true && /arrow/ci/scripts/r_test.sh /arrow" ubuntu-r-only-r: @@ -1367,7 +1368,7 @@ services: volumes: *debian-volumes command: &js-command > /bin/bash -c " - /arrow/ci/scripts/js_build.sh /arrow && + /arrow/ci/scripts/js_build.sh /arrow true && /arrow/ci/scripts/js_test.sh /arrow" #################################### C# ##################################### @@ -1415,7 +1416,7 @@ services: - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated command: &java-command > /bin/bash -c " - /arrow/ci/scripts/java_build.sh /arrow /build && + /arrow/ci/scripts/java_build.sh /arrow /build true && /arrow/ci/scripts/java_test.sh /arrow /build" debian-java-jni: @@ -1506,17 +1507,16 @@ services: environment: <<: *ccache ARROW_CUDA: "ON" - ARROW_GLIB_GTK_DOC: "true" Protobuf_SOURCE: "BUNDLED" # Need Protobuf >= 3.15 volumes: *ubuntu-volumes command: &docs-command > /bin/bash -c " /arrow/ci/scripts/cpp_build.sh /arrow /build true && - /arrow/ci/scripts/c_glib_build.sh /arrow /build && - /arrow/ci/scripts/python_build.sh /arrow /build && - /arrow/ci/scripts/java_build.sh /arrow /build true && - /arrow/ci/scripts/js_build.sh /arrow true && + /arrow/ci/scripts/c_glib_build.sh /arrow /build true && + /arrow/ci/scripts/python_build.sh /arrow /build true && /arrow/ci/scripts/r_build.sh /arrow true && + /arrow/ci/scripts/js_build.sh /arrow true && + /arrow/ci/scripts/java_build.sh /arrow /build true && /arrow/ci/scripts/docs_build.sh /arrow /build" ################################# Tools ##################################### From 77d467141ebc56fc952604374e9d4dfbb9037c21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 28 Oct 2021 17:54:00 +0200 Subject: [PATCH 04/24] Only rsync the documentations from docs_build.sh --- ci/scripts/docs_build.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/ci/scripts/docs_build.sh b/ci/scripts/docs_build.sh index e6ee768ee87..ec166534e42 100755 --- a/ci/scripts/docs_build.sh +++ b/ci/scripts/docs_build.sh @@ -21,17 +21,6 @@ set -ex arrow_dir=${1} build_dir=${2}/docs -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} -export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig:${PKG_CONFIG_PATH} -export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0 -export CFLAGS="-DARROW_NO_DEPRECATED_API" -export CXXFLAGS="-DARROW_NO_DEPRECATED_API" - -ncpus=$(python3 -c "import os; print(os.cpu_count())") - -# Sphinx docs -sphinx-build -b html -j ${ncpus} ${arrow_dir}/docs/source ${build_dir} - # C++ - original doxygen # rsync -a ${arrow_dir}/cpp/apidoc/ ${build_dir}/cpp From d1fc0e3c75dc8e3ac7efedf22b8146c2d477ac79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 28 Oct 2021 17:56:26 +0200 Subject: [PATCH 05/24] Trigger python doc builds --- ci/scripts/python_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/python_build.sh b/ci/scripts/python_build.sh index 6cd1880addf..bcef5338734 100755 --- a/ci/scripts/python_build.sh +++ b/ci/scripts/python_build.sh @@ -22,6 +22,7 @@ set -ex arrow_dir=${1} source_dir=${1}/python build_dir=${2}/python +with_docs=${3:-false} if [ ! -z "${CONDA_PREFIX}" ]; then echo -e "===\n=== Conda environment for build\n===" From b6578e177d338134906e03f88c785397aca46dc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 28 Oct 2021 18:03:05 +0200 Subject: [PATCH 06/24] Install sphinx to the python docker image --- ci/docker/conda-python.dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/docker/conda-python.dockerfile b/ci/docker/conda-python.dockerfile index 3749372e4f4..7241aacbdc8 100644 --- a/ci/docker/conda-python.dockerfile +++ b/ci/docker/conda-python.dockerfile @@ -24,6 +24,7 @@ ARG python=3.6 COPY ci/conda_env_python.txt /arrow/ci/ RUN mamba install -q \ --file arrow/ci/conda_env_python.txt \ + --file arrow/ci/conda_env_sphinx.txt \ $([ "$python" == "3.6" -o "$python" == "3.7" ] && echo "pickle5") \ python=${python} \ nomkl && \ From c195666031f4ed2d8e95f3f7db1a4326e906ca90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 28 Oct 2021 18:18:01 +0200 Subject: [PATCH 07/24] Add sphinx conda env file --- ci/docker/conda-python.dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/docker/conda-python.dockerfile b/ci/docker/conda-python.dockerfile index 7241aacbdc8..3f041d6f411 100644 --- a/ci/docker/conda-python.dockerfile +++ b/ci/docker/conda-python.dockerfile @@ -21,7 +21,9 @@ FROM ${repo}:${arch}-conda-cpp # install python specific packages ARG python=3.6 -COPY ci/conda_env_python.txt /arrow/ci/ +COPY ci/conda_env_python.txt \ + ci/conda_env_sphinx.txt \ + /arrow/ci/ RUN mamba install -q \ --file arrow/ci/conda_env_python.txt \ --file arrow/ci/conda_env_sphinx.txt \ From 1e29cd1f74ae4329a0be8dc1557e2002f39408ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 28 Oct 2021 18:33:29 +0200 Subject: [PATCH 08/24] ncpus --- ci/scripts/python_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/python_build.sh b/ci/scripts/python_build.sh index bcef5338734..4880cb74364 100755 --- a/ci/scripts/python_build.sh +++ b/ci/scripts/python_build.sh @@ -56,5 +56,6 @@ ${PYTHON:-python} \ popd if [ "${with_docs}" == "true" ]; then + ncpus=$(python -c "import os; print(os.cpu_count())") sphinx-build -b html -j ${ncpus} ${arrow_dir}/docs/source fi From e265f1159c44896b9df6645283e97a086ed8e800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 28 Oct 2021 19:02:04 +0200 Subject: [PATCH 09/24] Remove docs_build.sh --- ci/scripts/c_glib_build.sh | 5 +++++ ci/scripts/docs_build.sh | 37 ------------------------------------- ci/scripts/java_build.sh | 6 ++++-- ci/scripts/js_build.sh | 3 ++- ci/scripts/python_build.sh | 2 +- ci/scripts/r_build.sh | 4 +++- docker-compose.yml | 12 ++++-------- 7 files changed, 19 insertions(+), 50 deletions(-) delete mode 100755 ci/scripts/docs_build.sh diff --git a/ci/scripts/c_glib_build.sh b/ci/scripts/c_glib_build.sh index 5e45bdaee78..211cca27e49 100755 --- a/ci/scripts/c_glib_build.sh +++ b/ci/scripts/c_glib_build.sh @@ -21,6 +21,7 @@ set -ex source_dir=${1}/c_glib build_dir=${2}/c_glib +build_root=${2} with_docs=${3:-false} : ${ARROW_GLIB_GTK_DOC:=${with_docs}} @@ -45,3 +46,7 @@ pushd ${build_dir} ninja ninja install popd + +if [ "${with_docs}" == "true" ]; then + rsync -a ${ARROW_HOME}/share/gtk-doc/html/ ${build_root}/docs/c_glib +fi diff --git a/ci/scripts/docs_build.sh b/ci/scripts/docs_build.sh deleted file mode 100755 index ec166534e42..00000000000 --- a/ci/scripts/docs_build.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -ex - -arrow_dir=${1} -build_dir=${2}/docs - -# C++ - original doxygen -# rsync -a ${arrow_dir}/cpp/apidoc/ ${build_dir}/cpp - -# R -rsync -a ${arrow_dir}/r/docs/ ${build_dir}/r - -# C GLib -rsync -a ${ARROW_HOME}/share/gtk-doc/html/ ${build_dir}/c_glib - -# Java -rsync -a ${arrow_dir}/java/target/site/apidocs/ ${build_dir}/java/reference - -# Javascript -rsync -a ${arrow_dir}/js/doc/ ${build_dir}/js diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh index 1ba37606d3c..2e5a14e1951 100755 --- a/ci/scripts/java_build.sh +++ b/ci/scripts/java_build.sh @@ -20,8 +20,9 @@ set -ex arrow_dir=${1} source_dir=${1}/java -cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug} -cdata_dist_dir=${2}/java/c +build_dir=${2} +cpp_build_dir=${build_dir}/cpp/${ARROW_BUILD_TYPE:-debug} +cdata_dist_dir=${build_dir}/java/c with_docs=${3:-false} if [[ "$(uname -s)" == "Linux" ]] && [[ "$(uname -m)" == "s390x" ]]; then @@ -102,6 +103,7 @@ fi if [ "${with_docs}" == "true" ]; then # HTTP pooling is turned of to avoid download issues https://issues.apache.org/jira/browse/ARROW-11633 ${mvn} -Dcheckstyle.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false install site + rsync -a ${arrow_dir}/java/target/site/apidocs/ ${build_dir}/docs/java/reference fi popd diff --git a/ci/scripts/js_build.sh b/ci/scripts/js_build.sh index 12f58d54bb8..6519514cf46 100755 --- a/ci/scripts/js_build.sh +++ b/ci/scripts/js_build.sh @@ -20,12 +20,12 @@ set -ex source_dir=${1}/js +build_dir=${2} with_docs=${2:-false} pushd ${source_dir} yarn --frozen-lockfile -# TODO(kszucs): linting should be moved to archery yarn lint:ci yarn build @@ -40,6 +40,7 @@ if [ "${with_docs}" == "true" ]; then echo "Failed to build docs because the remote is not set correctly. Please set the origin or upstream remote to https://github.com/apache/arrow.git or the apache remote to git@github.com:apache/arrow.git." exit 0 fi + rsync -a ${arrow_dir}/js/doc/ ${build_dir}/docs/js fi popd diff --git a/ci/scripts/python_build.sh b/ci/scripts/python_build.sh index 4880cb74364..f8dc2ea2686 100755 --- a/ci/scripts/python_build.sh +++ b/ci/scripts/python_build.sh @@ -57,5 +57,5 @@ popd if [ "${with_docs}" == "true" ]; then ncpus=$(python -c "import os; print(os.cpu_count())") - sphinx-build -b html -j ${ncpus} ${arrow_dir}/docs/source + sphinx-build -b html -j ${ncpus} ${arrow_dir}/docs/source ${build_dir}/docs fi diff --git a/ci/scripts/r_build.sh b/ci/scripts/r_build.sh index 2a2b9d7d161..ded395c4ea0 100755 --- a/ci/scripts/r_build.sh +++ b/ci/scripts/r_build.sh @@ -20,6 +20,7 @@ set -ex : ${R_BIN:=R} source_dir=${1}/r +build_dir=${2} with_docs=${2:-false} pushd ${source_dir} @@ -28,6 +29,7 @@ ${R_BIN} CMD INSTALL ${INSTALL_ARGS} . if [ "${with_docs}" == "true" ]; then ${R_BIN} -e "pkgdown::build_site(install = FALSE)" + rsync -a ${source_dir}/docs/ ${build_dir}/docs/r && fi -popd \ No newline at end of file +popd diff --git a/docker-compose.yml b/docker-compose.yml index 106b6b725dd..2fef01964c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1135,7 +1135,6 @@ services: /bin/bash -c " /arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/python_build.sh /arrow /build && - /arrow/ci/scripts/r_build.sh /arrow true && /arrow/ci/scripts/r_test.sh /arrow" ubuntu-r-only-r: @@ -1289,8 +1288,6 @@ services: command: > /bin/bash -c "/arrow/ci/scripts/r_revdepcheck.sh /arrow" - - ################################# Go ######################################## debian-go: @@ -1368,7 +1365,7 @@ services: volumes: *debian-volumes command: &js-command > /bin/bash -c " - /arrow/ci/scripts/js_build.sh /arrow true && + /arrow/ci/scripts/js_build.sh /arrow /build true && /arrow/ci/scripts/js_test.sh /arrow" #################################### C# ##################################### @@ -1514,10 +1511,9 @@ services: /arrow/ci/scripts/cpp_build.sh /arrow /build true && /arrow/ci/scripts/c_glib_build.sh /arrow /build true && /arrow/ci/scripts/python_build.sh /arrow /build true && - /arrow/ci/scripts/r_build.sh /arrow true && - /arrow/ci/scripts/js_build.sh /arrow true && - /arrow/ci/scripts/java_build.sh /arrow /build true && - /arrow/ci/scripts/docs_build.sh /arrow /build" + /arrow/ci/scripts/r_build.sh /arrow /build true && + /arrow/ci/scripts/js_build.sh /arrow /build true && + /arrow/ci/scripts/java_build.sh /arrow /build true" ################################# Tools ##################################### From d3b380821e55dbd0eadaa6bfa06462007c41b21e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 28 Oct 2021 19:26:13 +0200 Subject: [PATCH 10/24] Add rsync to the ubuntu images --- ci/docker/ubuntu-18.04-cpp.dockerfile | 1 + ci/docker/ubuntu-20.04-cpp.dockerfile | 1 + ci/docker/ubuntu-21.04-cpp.dockerfile | 1 + ci/scripts/r_build.sh | 2 +- 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/docker/ubuntu-18.04-cpp.dockerfile b/ci/docker/ubuntu-18.04-cpp.dockerfile index 0c05ac4ee6b..d3023f33794 100644 --- a/ci/docker/ubuntu-18.04-cpp.dockerfile +++ b/ci/docker/ubuntu-18.04-cpp.dockerfile @@ -83,6 +83,7 @@ RUN apt-get update -y -q && \ pkg-config \ protobuf-compiler \ rapidjson-dev \ + rsync \ tzdata && \ apt-get clean && \ rm -rf /var/lib/apt/lists* diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index c50f71bc913..55d7e86cd97 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -92,6 +92,7 @@ RUN apt-get update -y -q && \ python3-rados \ rados-objclass-dev \ rapidjson-dev \ + rsync \ tzdata \ wget && \ apt-get clean && \ diff --git a/ci/docker/ubuntu-21.04-cpp.dockerfile b/ci/docker/ubuntu-21.04-cpp.dockerfile index 2017d26b5bd..0fa145f5d97 100644 --- a/ci/docker/ubuntu-21.04-cpp.dockerfile +++ b/ci/docker/ubuntu-21.04-cpp.dockerfile @@ -89,6 +89,7 @@ RUN apt-get update -y -q && \ protobuf-compiler-grpc \ python3-pip \ rapidjson-dev \ + rsync \ tzdata \ wget && \ apt-get clean && \ diff --git a/ci/scripts/r_build.sh b/ci/scripts/r_build.sh index ded395c4ea0..d243602fff5 100755 --- a/ci/scripts/r_build.sh +++ b/ci/scripts/r_build.sh @@ -29,7 +29,7 @@ ${R_BIN} CMD INSTALL ${INSTALL_ARGS} . if [ "${with_docs}" == "true" ]; then ${R_BIN} -e "pkgdown::build_site(install = FALSE)" - rsync -a ${source_dir}/docs/ ${build_dir}/docs/r && + rsync -a ${source_dir}/docs/ ${build_dir}/docs/r fi popd From 95c668e97609ece41f2ac3097e90ce42e46427b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 28 Oct 2021 19:33:41 +0200 Subject: [PATCH 11/24] Build docs in pandas build --- docker-compose.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2fef01964c8..69d7831ebda 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -631,7 +631,7 @@ services: volumes: *conda-volumes command: &python-conda-command ["/arrow/ci/scripts/cpp_build.sh /arrow /build && - /arrow/ci/scripts/python_build.sh /arrow /build true && + /arrow/ci/scripts/python_build.sh /arrow /build && /arrow/ci/scripts/python_test.sh /arrow"] ubuntu-cuda-python: @@ -940,7 +940,10 @@ services: environment: <<: *ccache volumes: *conda-volumes - command: *python-conda-command + command: + ["/arrow/ci/scripts/cpp_build.sh /arrow /build && + /arrow/ci/scripts/python_build.sh /arrow /build true && + /arrow/ci/scripts/python_test.sh /arrow"] conda-python-dask: # Possible $DASK parameters: From 9fda07ceed8fe53d13bdb667c5f5ec3fc29a0b39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 28 Oct 2021 20:27:20 +0200 Subject: [PATCH 12/24] Turn off javadoc in debian-java --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 69d7831ebda..8458d3908f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1416,7 +1416,7 @@ services: - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated command: &java-command > /bin/bash -c " - /arrow/ci/scripts/java_build.sh /arrow /build true && + /arrow/ci/scripts/java_build.sh /arrow /build && /arrow/ci/scripts/java_test.sh /arrow /build" debian-java-jni: From 0b827d2cab0caf169b5c46ca5614fd4dd0b916b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 29 Oct 2021 14:13:55 +0200 Subject: [PATCH 13/24] Increase timeouts --- .github/workflows/docs.yml | 2 +- .github/workflows/ruby.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f844be14799..eb9d4b51214 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -45,7 +45,7 @@ jobs: name: AMD64 Ubuntu 20.10 Complete Documentation runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 60 + timeout-minutes: 120 steps: - name: Checkout Arrow uses: actions/checkout@v2 diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 2afb7532318..5e861ecf2cf 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -58,7 +58,7 @@ jobs: name: AMD64 Ubuntu ${{ matrix.ubuntu }} GLib & Ruby runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 40 + timeout-minutes: 60 strategy: fail-fast: false matrix: From 6413276ce63ac17fd314f989d3a9e6dcdb93bf7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 29 Oct 2021 14:39:53 +0200 Subject: [PATCH 14/24] Ensure directory existst for c_glib docs --- ci/scripts/c_glib_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/c_glib_build.sh b/ci/scripts/c_glib_build.sh index 211cca27e49..2fa92de27dc 100755 --- a/ci/scripts/c_glib_build.sh +++ b/ci/scripts/c_glib_build.sh @@ -48,5 +48,6 @@ ninja install popd if [ "${with_docs}" == "true" ]; then + mkdir -p ${build_root}/docs/c_glib rsync -a ${ARROW_HOME}/share/gtk-doc/html/ ${build_root}/docs/c_glib fi From 6067e1cbc92585dd1d78f5f709a3e8a51b246f67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 29 Oct 2021 23:41:55 +0200 Subject: [PATCH 15/24] Address review comments --- ci/scripts/c_glib_build.sh | 3 +-- ci/scripts/java_build.sh | 1 + ci/scripts/js_build.sh | 2 +- ci/scripts/python_build.sh | 2 +- ci/scripts/r_build.sh | 2 +- docker-compose.yml | 2 -- 6 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ci/scripts/c_glib_build.sh b/ci/scripts/c_glib_build.sh index 2fa92de27dc..5c818f46ac5 100755 --- a/ci/scripts/c_glib_build.sh +++ b/ci/scripts/c_glib_build.sh @@ -24,7 +24,6 @@ build_dir=${2}/c_glib build_root=${2} with_docs=${3:-false} -: ${ARROW_GLIB_GTK_DOC:=${with_docs}} : ${ARROW_GLIB_DEVELOPMENT_MODE:=false} export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig @@ -38,7 +37,7 @@ mkdir -p ${build_dir} meson --prefix=$ARROW_HOME \ --libdir=lib \ -Ddevelopment_mode=${ARROW_GLIB_DEVELOPMENT_MODE} \ - -Dgtk_doc=${ARROW_GLIB_GTK_DOC} \ + -Dgtk_doc=${with_docs} \ ${build_dir} \ ${source_dir} diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh index 2e5a14e1951..870bfac87e0 100755 --- a/ci/scripts/java_build.sh +++ b/ci/scripts/java_build.sh @@ -102,6 +102,7 @@ fi if [ "${with_docs}" == "true" ]; then # HTTP pooling is turned of to avoid download issues https://issues.apache.org/jira/browse/ARROW-11633 + mkdir -p ${build_dir}/docs/java/reference ${mvn} -Dcheckstyle.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false install site rsync -a ${arrow_dir}/java/target/site/apidocs/ ${build_dir}/docs/java/reference fi diff --git a/ci/scripts/js_build.sh b/ci/scripts/js_build.sh index 6519514cf46..74c5700311e 100755 --- a/ci/scripts/js_build.sh +++ b/ci/scripts/js_build.sh @@ -21,7 +21,7 @@ set -ex source_dir=${1}/js build_dir=${2} -with_docs=${2:-false} +with_docs=${3:-false} pushd ${source_dir} diff --git a/ci/scripts/python_build.sh b/ci/scripts/python_build.sh index f8dc2ea2686..0cff0d061a3 100755 --- a/ci/scripts/python_build.sh +++ b/ci/scripts/python_build.sh @@ -20,7 +20,7 @@ set -ex arrow_dir=${1} -source_dir=${1}/python +source_dir=${arrow_dir}/python build_dir=${2}/python with_docs=${3:-false} diff --git a/ci/scripts/r_build.sh b/ci/scripts/r_build.sh index d243602fff5..a50064dbe35 100755 --- a/ci/scripts/r_build.sh +++ b/ci/scripts/r_build.sh @@ -21,7 +21,7 @@ set -ex : ${R_BIN:=R} source_dir=${1}/r build_dir=${2} -with_docs=${2:-false} +with_docs=${3:-false} pushd ${source_dir} diff --git a/docker-compose.yml b/docker-compose.yml index 8458d3908f3..e3ef496e6e8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -513,7 +513,6 @@ services: ulimits: *ulimits environment: <<: *ccache - ARROW_GLIB_GTK_DOC: "true" volumes: *debian-volumes command: &c-glib-command > /bin/bash -c " @@ -541,7 +540,6 @@ services: ulimits: *ulimits environment: <<: *ccache - ARROW_GLIB_GTK_DOC: "true" volumes: *ubuntu-volumes command: *c-glib-command From 92ef80fce49bd96a0010fec0023deb1a2d373c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 3 Nov 2021 17:33:27 +0100 Subject: [PATCH 16/24] Fix JS build --- ci/docker/debian-10-js.dockerfile | 6 ++++++ ci/docker/debian-11-js.dockerfile | 6 ++++++ ci/scripts/js_build.sh | 3 ++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ci/docker/debian-10-js.dockerfile b/ci/docker/debian-10-js.dockerfile index 5bb31f2e32e..f994de4141d 100644 --- a/ci/docker/debian-10-js.dockerfile +++ b/ci/docker/debian-10-js.dockerfile @@ -21,6 +21,12 @@ FROM ${arch}/node:${node} ENV NODE_NO_WARNINGS=1 +# install rsync for copying the generated documentation +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends rsync && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + # TODO(kszucs): # 1. add the files required to install the dependencies to .dockerignore # 2. copy these files to their appropriate path diff --git a/ci/docker/debian-11-js.dockerfile b/ci/docker/debian-11-js.dockerfile index 5bb31f2e32e..f994de4141d 100644 --- a/ci/docker/debian-11-js.dockerfile +++ b/ci/docker/debian-11-js.dockerfile @@ -21,6 +21,12 @@ FROM ${arch}/node:${node} ENV NODE_NO_WARNINGS=1 +# install rsync for copying the generated documentation +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends rsync && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + # TODO(kszucs): # 1. add the files required to install the dependencies to .dockerignore # 2. copy these files to their appropriate path diff --git a/ci/scripts/js_build.sh b/ci/scripts/js_build.sh index 74c5700311e..fe30d0cedff 100755 --- a/ci/scripts/js_build.sh +++ b/ci/scripts/js_build.sh @@ -19,7 +19,8 @@ set -ex -source_dir=${1}/js +arrow_dir=${1} +source_dir=${arrow_dir}/js build_dir=${2} with_docs=${3:-false} From 68116eb3207a8c6e6e6b25656a98721ce48ade0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 3 Nov 2021 17:56:51 +0100 Subject: [PATCH 17/24] Ensure JS doc dir exists --- ci/scripts/js_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/scripts/js_build.sh b/ci/scripts/js_build.sh index fe30d0cedff..26e71321065 100755 --- a/ci/scripts/js_build.sh +++ b/ci/scripts/js_build.sh @@ -41,6 +41,7 @@ if [ "${with_docs}" == "true" ]; then echo "Failed to build docs because the remote is not set correctly. Please set the origin or upstream remote to https://github.com/apache/arrow.git or the apache remote to git@github.com:apache/arrow.git." exit 0 fi + mkdir -p ${build_dir}/docs/js rsync -a ${arrow_dir}/js/doc/ ${build_dir}/docs/js fi From 87db05afa090a4026ef9e55fa744e453d03c09d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 30 Nov 2021 13:32:32 +0100 Subject: [PATCH 18/24] Use environment variables to enable apidoc builds --- .github/workflows/docs.yml | 4 +- ci/scripts/c_glib_build.sh | 7 +- ci/scripts/cpp_build.sh | 187 +++++++++++++++++++------------------ ci/scripts/java_build.sh | 5 +- ci/scripts/js_build.sh | 5 +- ci/scripts/python_build.sh | 5 +- ci/scripts/r_build.sh | 5 +- docker-compose.yml | 34 ++++--- 8 files changed, 136 insertions(+), 116 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index eb9d4b51214..0391cfc60d6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -42,10 +42,12 @@ env: jobs: docker: - name: AMD64 Ubuntu 20.10 Complete Documentation + name: AMD64 Ubuntu 20.04 Complete Documentation runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 120 + env: + UBUNTU: "20.04" steps: - name: Checkout Arrow uses: actions/checkout@v2 diff --git a/ci/scripts/c_glib_build.sh b/ci/scripts/c_glib_build.sh index 5c818f46ac5..e0baf7f3d13 100755 --- a/ci/scripts/c_glib_build.sh +++ b/ci/scripts/c_glib_build.sh @@ -22,9 +22,10 @@ set -ex source_dir=${1}/c_glib build_dir=${2}/c_glib build_root=${2} -with_docs=${3:-false} : ${ARROW_GLIB_DEVELOPMENT_MODE:=false} +: ${BUILD_DOCS_C_GLIB:=OFF} +with_gtk_doc=$([ "${BUILD_DOCS_C_GLIB}" == "ON" ] && echo "true" || echo "false") export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig @@ -37,7 +38,7 @@ mkdir -p ${build_dir} meson --prefix=$ARROW_HOME \ --libdir=lib \ -Ddevelopment_mode=${ARROW_GLIB_DEVELOPMENT_MODE} \ - -Dgtk_doc=${with_docs} \ + -Dgtk_doc=${with_gtk_doc} \ ${build_dir} \ ${source_dir} @@ -46,7 +47,7 @@ ninja ninja install popd -if [ "${with_docs}" == "true" ]; then +if [ "${BUILD_DOCS_C_GLIB}" == "ON" ]; then mkdir -p ${build_root}/docs/c_glib rsync -a ${ARROW_HOME}/share/gtk-doc/html/ ${build_root}/docs/c_glib fi diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh index 0ea9b1b89dc..f791ddd5645 100755 --- a/ci/scripts/cpp_build.sh +++ b/ci/scripts/cpp_build.sh @@ -21,9 +21,9 @@ set -ex source_dir=${1}/cpp build_dir=${2}/cpp -with_docs=${3:-false} : ${ARROW_USE_CCACHE:=OFF} +: ${BUILD_DOCS_CPP:=OFF} # TODO(kszucs): consider to move these to CMake if [ ! -z "${CONDA_PREFIX}" ]; then @@ -49,97 +49,98 @@ fi mkdir -p ${build_dir} pushd ${build_dir} -cmake -G "${CMAKE_GENERATOR:-Ninja}" \ - -DARROW_BOOST_USE_SHARED=${ARROW_BOOST_USE_SHARED:-ON} \ - -DARROW_BUILD_BENCHMARKS=${ARROW_BUILD_BENCHMARKS:-OFF} \ - -DARROW_BUILD_BENCHMARKS_REFERENCE=${ARROW_BUILD_BENCHMARKS:-OFF} \ - -DARROW_BUILD_EXAMPLES=${ARROW_BUILD_EXAMPLES:-OFF} \ - -DARROW_BUILD_INTEGRATION=${ARROW_BUILD_INTEGRATION:-OFF} \ - -DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED:-ON} \ - -DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC:-ON} \ - -DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS:-OFF} \ - -DARROW_BUILD_UTILITIES=${ARROW_BUILD_UTILITIES:-ON} \ - -DARROW_COMPUTE=${ARROW_COMPUTE:-ON} \ - -DARROW_CSV=${ARROW_CSV:-ON} \ - -DARROW_CUDA=${ARROW_CUDA:-OFF} \ - -DARROW_CXXFLAGS=${ARROW_CXXFLAGS:-} \ - -DARROW_DATASET=${ARROW_DATASET:-ON} \ - -DARROW_ENGINE=${ARROW_ENGINE:-ON} \ - -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-AUTO} \ - -DARROW_EXTRA_ERROR_CONTEXT=${ARROW_EXTRA_ERROR_CONTEXT:-OFF} \ - -DARROW_ENABLE_TIMING_TESTS=${ARROW_ENABLE_TIMING_TESTS:-ON} \ - -DARROW_FILESYSTEM=${ARROW_FILESYSTEM:-ON} \ - -DARROW_FLIGHT=${ARROW_FLIGHT:-OFF} \ - -DARROW_FUZZING=${ARROW_FUZZING:-OFF} \ - -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \ - -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \ - -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \ - -DARROW_GCS=${ARROW_GCS:-OFF} \ - -DARROW_HDFS=${ARROW_HDFS:-ON} \ - -DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \ - -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \ - -DARROW_JEMALLOC=${ARROW_JEMALLOC:-ON} \ - -DARROW_JNI=${ARROW_JNI:-OFF} \ - -DARROW_JSON=${ARROW_JSON:-ON} \ - -DARROW_LARGE_MEMORY_TESTS=${ARROW_LARGE_MEMORY_TESTS:-OFF} \ - -DARROW_MIMALLOC=${ARROW_MIMALLOC:-OFF} \ - -DARROW_NO_DEPRECATED_API=${ARROW_NO_DEPRECATED_API:-OFF} \ - -DARROW_ORC=${ARROW_ORC:-OFF} \ - -DARROW_PARQUET=${ARROW_PARQUET:-OFF} \ - -DARROW_PLASMA_JAVA_CLIENT=${ARROW_PLASMA_JAVA_CLIENT:-OFF} \ - -DARROW_PLASMA=${ARROW_PLASMA:-OFF} \ - -DARROW_PYTHON=${ARROW_PYTHON:-OFF} \ - -DARROW_RUNTIME_SIMD_LEVEL=${ARROW_RUNTIME_SIMD_LEVEL:-MAX} \ - -DARROW_S3=${ARROW_S3:-OFF} \ - -DARROW_SKYHOOK=${ARROW_SKYHOOK:-OFF} \ - -DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \ - -DARROW_TEST_MEMCHECK=${ARROW_TEST_MEMCHECK:-OFF} \ - -DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \ - -DARROW_USE_CCACHE=${ARROW_USE_CCACHE:-ON} \ - -DARROW_USE_GLOG=${ARROW_USE_GLOG:-OFF} \ - -DARROW_USE_LD_GOLD=${ARROW_USE_LD_GOLD:-OFF} \ - -DARROW_USE_PRECOMPILED_HEADERS=${ARROW_USE_PRECOMPILED_HEADERS:-OFF} \ - -DARROW_USE_STATIC_CRT=${ARROW_USE_STATIC_CRT:-OFF} \ - -DARROW_USE_TSAN=${ARROW_USE_TSAN:-OFF} \ - -DARROW_USE_UBSAN=${ARROW_USE_UBSAN:-OFF} \ - -DARROW_VERBOSE_THIRDPARTY_BUILD=${ARROW_VERBOSE_THIRDPARTY_BUILD:-OFF} \ - -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-OFF} \ - -DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-OFF} \ - -DARROW_WITH_LZ4=${ARROW_WITH_LZ4:-OFF} \ - -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY:-OFF} \ - -DARROW_WITH_UTF8PROC=${ARROW_WITH_UTF8PROC:-ON} \ - -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-OFF} \ - -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-OFF} \ - -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \ - -Dbenchmark_SOURCE=${benchmark_SOURCE:-} \ - -DBOOST_SOURCE=${BOOST_SOURCE:-} \ - -DBrotli_SOURCE=${Brotli_SOURCE:-} \ - -DBUILD_WARNING_LEVEL=${BUILD_WARNING_LEVEL:-CHECKIN} \ - -Dc-ares_SOURCE=${cares_SOURCE:-} \ - -DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \ - -DCMAKE_C_FLAGS="${CFLAGS:-}" \ - -DCMAKE_CXX_FLAGS="${CXXFLAGS:-}" \ - -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \ - -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \ - -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ - -Dgflags_SOURCE=${gflags_SOURCE:-} \ - -Dgoogle_cloud_cpp_storage_SOURCE=${google_cloud_cpp_storage_SOURCE:-} \ - -DgRPC_SOURCE=${gRPC_SOURCE:-} \ - -DGTest_SOURCE=${GTest_SOURCE:-} \ - -DLz4_SOURCE=${Lz4_SOURCE:-} \ - -DORC_SOURCE=${ORC_SOURCE:-} \ - -DPARQUET_BUILD_EXECUTABLES=${PARQUET_BUILD_EXECUTABLES:-OFF} \ - -DPARQUET_BUILD_EXAMPLES=${PARQUET_BUILD_EXAMPLES:-OFF} \ - -DPARQUET_REQUIRE_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION:-ON} \ - -DProtobuf_SOURCE=${Protobuf_SOURCE:-} \ - -DRapidJSON_SOURCE=${RapidJSON_SOURCE:-} \ - -Dre2_SOURCE=${re2_SOURCE:-} \ - -DSnappy_SOURCE=${Snappy_SOURCE:-} \ - -DThrift_SOURCE=${Thrift_SOURCE:-} \ - -Dutf8proc_SOURCE=${utf8proc_SOURCE:-} \ - -Dzstd_SOURCE=${zstd_SOURCE:-} \ - ${CMAKE_ARGS} \ - ${source_dir} +cmake \ + -DARROW_BOOST_USE_SHARED=${ARROW_BOOST_USE_SHARED:-ON} \ + -DARROW_BUILD_BENCHMARKS_REFERENCE=${ARROW_BUILD_BENCHMARKS:-OFF} \ + -DARROW_BUILD_BENCHMARKS=${ARROW_BUILD_BENCHMARKS:-OFF} \ + -DARROW_BUILD_EXAMPLES=${ARROW_BUILD_EXAMPLES:-OFF} \ + -DARROW_BUILD_INTEGRATION=${ARROW_BUILD_INTEGRATION:-OFF} \ + -DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED:-ON} \ + -DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC:-ON} \ + -DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS:-OFF} \ + -DARROW_BUILD_UTILITIES=${ARROW_BUILD_UTILITIES:-ON} \ + -DARROW_COMPUTE=${ARROW_COMPUTE:-ON} \ + -DARROW_CSV=${ARROW_CSV:-ON} \ + -DARROW_CUDA=${ARROW_CUDA:-OFF} \ + -DARROW_CXXFLAGS=${ARROW_CXXFLAGS:-} \ + -DARROW_DATASET=${ARROW_DATASET:-ON} \ + -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-AUTO} \ + -DARROW_ENABLE_TIMING_TESTS=${ARROW_ENABLE_TIMING_TESTS:-ON} \ + -DARROW_ENGINE=${ARROW_ENGINE:-ON} \ + -DARROW_EXTRA_ERROR_CONTEXT=${ARROW_EXTRA_ERROR_CONTEXT:-OFF} \ + -DARROW_FILESYSTEM=${ARROW_FILESYSTEM:-ON} \ + -DARROW_FLIGHT=${ARROW_FLIGHT:-OFF} \ + -DARROW_FUZZING=${ARROW_FUZZING:-OFF} \ + -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \ + -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \ + -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \ + -DARROW_GCS=${ARROW_GCS:-OFF} \ + -DARROW_HDFS=${ARROW_HDFS:-ON} \ + -DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \ + -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \ + -DARROW_JEMALLOC=${ARROW_JEMALLOC:-ON} \ + -DARROW_JNI=${ARROW_JNI:-OFF} \ + -DARROW_JSON=${ARROW_JSON:-ON} \ + -DARROW_LARGE_MEMORY_TESTS=${ARROW_LARGE_MEMORY_TESTS:-OFF} \ + -DARROW_MIMALLOC=${ARROW_MIMALLOC:-OFF} \ + -DARROW_NO_DEPRECATED_API=${ARROW_NO_DEPRECATED_API:-OFF} \ + -DARROW_ORC=${ARROW_ORC:-OFF} \ + -DARROW_PARQUET=${ARROW_PARQUET:-OFF} \ + -DARROW_PLASMA_JAVA_CLIENT=${ARROW_PLASMA_JAVA_CLIENT:-OFF} \ + -DARROW_PLASMA=${ARROW_PLASMA:-OFF} \ + -DARROW_PYTHON=${ARROW_PYTHON:-OFF} \ + -DARROW_RUNTIME_SIMD_LEVEL=${ARROW_RUNTIME_SIMD_LEVEL:-MAX} \ + -DARROW_S3=${ARROW_S3:-OFF} \ + -DARROW_SKYHOOK=${ARROW_SKYHOOK:-OFF} \ + -DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \ + -DARROW_TEST_MEMCHECK=${ARROW_TEST_MEMCHECK:-OFF} \ + -DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \ + -DARROW_USE_CCACHE=${ARROW_USE_CCACHE:-ON} \ + -DARROW_USE_GLOG=${ARROW_USE_GLOG:-OFF} \ + -DARROW_USE_LD_GOLD=${ARROW_USE_LD_GOLD:-OFF} \ + -DARROW_USE_PRECOMPILED_HEADERS=${ARROW_USE_PRECOMPILED_HEADERS:-OFF} \ + -DARROW_USE_STATIC_CRT=${ARROW_USE_STATIC_CRT:-OFF} \ + -DARROW_USE_TSAN=${ARROW_USE_TSAN:-OFF} \ + -DARROW_USE_UBSAN=${ARROW_USE_UBSAN:-OFF} \ + -DARROW_VERBOSE_THIRDPARTY_BUILD=${ARROW_VERBOSE_THIRDPARTY_BUILD:-OFF} \ + -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-OFF} \ + -DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-OFF} \ + -DARROW_WITH_LZ4=${ARROW_WITH_LZ4:-OFF} \ + -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY:-OFF} \ + -DARROW_WITH_UTF8PROC=${ARROW_WITH_UTF8PROC:-ON} \ + -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-OFF} \ + -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-OFF} \ + -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \ + -Dbenchmark_SOURCE=${benchmark_SOURCE:-} \ + -DBOOST_SOURCE=${BOOST_SOURCE:-} \ + -DBrotli_SOURCE=${Brotli_SOURCE:-} \ + -DBUILD_WARNING_LEVEL=${BUILD_WARNING_LEVEL:-CHECKIN} \ + -Dc-ares_SOURCE=${cares_SOURCE:-} \ + -DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \ + -DCMAKE_C_FLAGS="${CFLAGS:-}" \ + -DCMAKE_CXX_FLAGS="${CXXFLAGS:-}" \ + -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \ + -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \ + -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ + -Dgflags_SOURCE=${gflags_SOURCE:-} \ + -Dgoogle_cloud_cpp_storage_SOURCE=${google_cloud_cpp_storage_SOURCE:-} \ + -DgRPC_SOURCE=${gRPC_SOURCE:-} \ + -DGTest_SOURCE=${GTest_SOURCE:-} \ + -DLz4_SOURCE=${Lz4_SOURCE:-} \ + -DORC_SOURCE=${ORC_SOURCE:-} \ + -DPARQUET_BUILD_EXAMPLES=${PARQUET_BUILD_EXAMPLES:-OFF} \ + -DPARQUET_BUILD_EXECUTABLES=${PARQUET_BUILD_EXECUTABLES:-OFF} \ + -DPARQUET_REQUIRE_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION:-ON} \ + -DProtobuf_SOURCE=${Protobuf_SOURCE:-} \ + -DRapidJSON_SOURCE=${RapidJSON_SOURCE:-} \ + -Dre2_SOURCE=${re2_SOURCE:-} \ + -DSnappy_SOURCE=${Snappy_SOURCE:-} \ + -DThrift_SOURCE=${Thrift_SOURCE:-} \ + -Dutf8proc_SOURCE=${utf8proc_SOURCE:-} \ + -Dzstd_SOURCE=${zstd_SOURCE:-} \ + -G "${CMAKE_GENERATOR:-Ninja}" \ + ${CMAKE_ARGS} \ + ${source_dir} if [ ! -z "${CPP_MAKE_PARALLELISM}" ]; then time cmake --build . --target install -- -j${CPP_MAKE_PARALLELISM} @@ -158,7 +159,7 @@ if [ "${ARROW_USE_CCACHE}" == "ON" ]; then ccache -s fi -if [ "${with_docs}" == "true" ]; then +if [ "${DOCS_BUILD_CPP}" == "ON" ]; then pushd ${source_dir}/apidoc doxygen popd diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh index 870bfac87e0..7693bd883a1 100755 --- a/ci/scripts/java_build.sh +++ b/ci/scripts/java_build.sh @@ -23,7 +23,8 @@ source_dir=${1}/java build_dir=${2} cpp_build_dir=${build_dir}/cpp/${ARROW_BUILD_TYPE:-debug} cdata_dist_dir=${build_dir}/java/c -with_docs=${3:-false} + +: ${BUILD_DOCS_JAVA:=OFF} if [[ "$(uname -s)" == "Linux" ]] && [[ "$(uname -m)" == "s390x" ]]; then # Since some files for s390_64 are not available at maven central, @@ -100,7 +101,7 @@ if [ "${ARROW_PLASMA}" = "ON" ]; then popd fi -if [ "${with_docs}" == "true" ]; then +if [ "${BUILD_DOCS_JAVA}" == "ON" ]; then # HTTP pooling is turned of to avoid download issues https://issues.apache.org/jira/browse/ARROW-11633 mkdir -p ${build_dir}/docs/java/reference ${mvn} -Dcheckstyle.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false install site diff --git a/ci/scripts/js_build.sh b/ci/scripts/js_build.sh index 26e71321065..333c9cf441e 100755 --- a/ci/scripts/js_build.sh +++ b/ci/scripts/js_build.sh @@ -22,7 +22,8 @@ set -ex arrow_dir=${1} source_dir=${arrow_dir}/js build_dir=${2} -with_docs=${3:-false} + +: ${BUILD_DOCS_JS:=OFF} pushd ${source_dir} @@ -30,7 +31,7 @@ yarn --frozen-lockfile yarn lint:ci yarn build -if [ "${with_docs}" == "true" ]; then +if [ "${BUILD_DOCS_JS}" == "ON" ]; then if [ "$(git config --get remote.origin.url)" == "https://github.com/apache/arrow.git" ]; then yarn doc elif [ "$(git config --get remote.upstream.url)" == "https://github.com/apache/arrow.git" ]; then diff --git a/ci/scripts/python_build.sh b/ci/scripts/python_build.sh index 0cff0d061a3..ccf068078d5 100755 --- a/ci/scripts/python_build.sh +++ b/ci/scripts/python_build.sh @@ -22,7 +22,8 @@ set -ex arrow_dir=${1} source_dir=${arrow_dir}/python build_dir=${2}/python -with_docs=${3:-false} + +: ${BUILD_DOCS_PYTHON:=OFF} if [ ! -z "${CONDA_PREFIX}" ]; then echo -e "===\n=== Conda environment for build\n===" @@ -55,7 +56,7 @@ ${PYTHON:-python} \ popd -if [ "${with_docs}" == "true" ]; then +if [ "${BUILD_DOCS_PYTHON}" == "ON" ]; then ncpus=$(python -c "import os; print(os.cpu_count())") sphinx-build -b html -j ${ncpus} ${arrow_dir}/docs/source ${build_dir}/docs fi diff --git a/ci/scripts/r_build.sh b/ci/scripts/r_build.sh index a50064dbe35..a435b04317d 100755 --- a/ci/scripts/r_build.sh +++ b/ci/scripts/r_build.sh @@ -21,13 +21,14 @@ set -ex : ${R_BIN:=R} source_dir=${1}/r build_dir=${2} -with_docs=${3:-false} + +: ${BUILD_DOCS_R:=OFF} pushd ${source_dir} ${R_BIN} CMD INSTALL ${INSTALL_ARGS} . -if [ "${with_docs}" == "true" ]; then +if [ "${BUILD_DOCS_R}" == "ON" ]; then ${R_BIN} -e "pkgdown::build_site(install = FALSE)" rsync -a ${source_dir}/docs/ ${build_dir}/docs/r fi diff --git a/docker-compose.yml b/docker-compose.yml index e3ef496e6e8..9bf5bd0841d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -236,11 +236,12 @@ services: ARROW_MIMALLOC: "ON" ARROW_USE_LD_GOLD: "ON" ARROW_USE_PRECOMPILED_HEADERS: "ON" + BUILD_DOCS_PYTHON: "ON" volumes: &conda-volumes - .:/arrow:delegated - ${DOCKER_VOLUME_PREFIX}conda-ccache:/ccache:delegated command: - ["/arrow/ci/scripts/cpp_build.sh /arrow /build true && + ["/arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/cpp_test.sh /arrow /build"] conda-cpp-valgrind: @@ -513,11 +514,12 @@ services: ulimits: *ulimits environment: <<: *ccache + BUILD_DOCS_C_GLIB: "ON" volumes: *debian-volumes command: &c-glib-command > /bin/bash -c " /arrow/ci/scripts/cpp_build.sh /arrow /build && - /arrow/ci/scripts/c_glib_build.sh /arrow /build true && + /arrow/ci/scripts/c_glib_build.sh /arrow /build && /arrow/ci/scripts/c_glib_test.sh /arrow /build" ubuntu-c-glib: @@ -570,11 +572,12 @@ services: ulimits: *ulimits environment: <<: *ccache + BUILD_DOCS_C_GLIB: "ON" volumes: *debian-volumes command: &ruby-command > /bin/bash -c " /arrow/ci/scripts/cpp_build.sh /arrow /build && - /arrow/ci/scripts/c_glib_build.sh /arrow /build true && + /arrow/ci/scripts/c_glib_build.sh /arrow /build && /arrow/ci/scripts/c_glib_test.sh /arrow /build && /arrow/ci/scripts/ruby_test.sh /arrow /build" @@ -937,10 +940,11 @@ services: shm_size: *shm-size environment: <<: *ccache + BUILD_DOCS_PYTHON: "ON" volumes: *conda-volumes command: ["/arrow/ci/scripts/cpp_build.sh /arrow /build && - /arrow/ci/scripts/python_build.sh /arrow /build true && + /arrow/ci/scripts/python_build.sh /arrow /build && /arrow/ci/scripts/python_test.sh /arrow"] conda-python-dask: @@ -1364,9 +1368,11 @@ services: node: ${NODE} shm_size: *shm-size volumes: *debian-volumes + environment: + BUILD_DOCS_JS: "ON" command: &js-command > /bin/bash -c " - /arrow/ci/scripts/js_build.sh /arrow /build true && + /arrow/ci/scripts/js_build.sh /arrow /build && /arrow/ci/scripts/js_test.sh /arrow" #################################### C# ##################################### @@ -1505,16 +1511,22 @@ services: environment: <<: *ccache ARROW_CUDA: "ON" + BUILD_DOCS_C_GLIB: "ON" + BUILD_DOCS_CPP: "ON" + BUILD_DOCS_JAVA: "ON" + BUILD_DOCS_JS: "ON" + BUILD_DOCS_PYTHON: "ON" + BUILD_DOCS_R: "ON" Protobuf_SOURCE: "BUNDLED" # Need Protobuf >= 3.15 volumes: *ubuntu-volumes command: &docs-command > /bin/bash -c " - /arrow/ci/scripts/cpp_build.sh /arrow /build true && - /arrow/ci/scripts/c_glib_build.sh /arrow /build true && - /arrow/ci/scripts/python_build.sh /arrow /build true && - /arrow/ci/scripts/r_build.sh /arrow /build true && - /arrow/ci/scripts/js_build.sh /arrow /build true && - /arrow/ci/scripts/java_build.sh /arrow /build true" + /arrow/ci/scripts/cpp_build.sh /arrow /build && + /arrow/ci/scripts/c_glib_build.sh /arrow /build && + /arrow/ci/scripts/python_build.sh /arrow /build && + /arrow/ci/scripts/r_build.sh /arrow /build && + /arrow/ci/scripts/js_build.sh /arrow /build && + /arrow/ci/scripts/java_build.sh /arrow /build" ################################# Tools ##################################### From 3e80c179f630d122b4ce8b20023903e86d5aca92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 30 Nov 2021 14:47:10 +0100 Subject: [PATCH 19/24] Reuse conda_env_sphinx.txt in the ubuntu-docs image --- ci/docker/linux-apt-docs.dockerfile | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile index eac59dffb3e..5466e6b3218 100644 --- a/ci/docker/linux-apt-docs.dockerfile +++ b/ci/docker/linux-apt-docs.dockerfile @@ -74,15 +74,8 @@ RUN wget -q -O - https://deb.nodesource.com/setup_${node}.x | bash - && \ rm -rf /var/lib/apt/lists/* && \ npm install -g yarn -# ARROW-13353: breathe >= 4.29.1 tries to parse template arguments, -# but Sphinx can't parse constructs like `typename...`. -RUN pip install \ - meson \ - breathe==4.29.0 \ - ipython \ - pydata-sphinx-theme \ - sphinx \ - sphinx-tabs +COPY ci/conda_env_sphinx.txt /arrow/ci/ +RUN pip install -r arrow/ci/conda_env_sphinx.txt meson COPY c_glib/Gemfile /arrow/c_glib/ RUN gem install --no-document bundler && \ @@ -107,4 +100,4 @@ ENV ARROW_FLIGHT=ON \ ARROW_BUILD_TESTS=OFF \ ARROW_BUILD_UTILITIES=OFF \ ARROW_USE_GLOG=OFF \ - CMAKE_UNITY_BUILD=ON \ + CMAKE_UNITY_BUILD=ON From 896e85685d67978d19d18cbb3e1024cffa8e48ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 30 Nov 2021 19:02:56 +0100 Subject: [PATCH 20/24] Use pip flavoured dependency constraint --- ci/conda_env_sphinx.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/conda_env_sphinx.txt b/ci/conda_env_sphinx.txt index ca82705e73c..90094ddda70 100644 --- a/ci/conda_env_sphinx.txt +++ b/ci/conda_env_sphinx.txt @@ -19,6 +19,6 @@ breathe doxygen ipython -sphinx=4.2 +sphinx==4.2 pydata-sphinx-theme sphinx-tabs From 216b854d4df389246fd6529e749b6f3bddb92034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 1 Dec 2021 22:51:10 +0100 Subject: [PATCH 21/24] Install sphinx-tabs via pip --- ci/conda_env_sphinx.txt | 7 +++++-- ci/docker/conda-python.dockerfile | 4 ++++ ci/docker/linux-apt-docs.dockerfile | 4 +++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/ci/conda_env_sphinx.txt b/ci/conda_env_sphinx.txt index 90094ddda70..9f6c99b132e 100644 --- a/ci/conda_env_sphinx.txt +++ b/ci/conda_env_sphinx.txt @@ -19,6 +19,9 @@ breathe doxygen ipython -sphinx==4.2 +sphinx>=4.2 pydata-sphinx-theme -sphinx-tabs + +# Unable to install sphinx-tabs from conda-forge due to: +# - package sphinx-tabs-1.2.1-py_0 requires sphinx >=2,<4, but none of the providers can be installed +# sphinx-tabs diff --git a/ci/docker/conda-python.dockerfile b/ci/docker/conda-python.dockerfile index 3f041d6f411..ae7abd9c9db 100644 --- a/ci/docker/conda-python.dockerfile +++ b/ci/docker/conda-python.dockerfile @@ -32,6 +32,10 @@ RUN mamba install -q \ nomkl && \ mamba clean --all +# unable to install from conda-forge due to sphinx version pin, see comment in +# arrow/ci/conda_env_sphinx.txt +RUN pip install sphinx-tabs + ENV ARROW_PYTHON=ON \ ARROW_BUILD_STATIC=OFF \ ARROW_BUILD_TESTS=OFF \ diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile index 5466e6b3218..cd2ff6813f8 100644 --- a/ci/docker/linux-apt-docs.dockerfile +++ b/ci/docker/linux-apt-docs.dockerfile @@ -75,7 +75,9 @@ RUN wget -q -O - https://deb.nodesource.com/setup_${node}.x | bash - && \ npm install -g yarn COPY ci/conda_env_sphinx.txt /arrow/ci/ -RUN pip install -r arrow/ci/conda_env_sphinx.txt meson +# unable to install sphinx-tabs from conda-forge due to sphinx version pin, see +# comment in arrow/ci/conda_env_sphinx.txt +RUN pip install -r arrow/ci/conda_env_sphinx.txt meson sphinx-tabs COPY c_glib/Gemfile /arrow/c_glib/ RUN gem install --no-document bundler && \ From bdbf926f555192beb8cc4bc1e9326c305d387ab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 1 Dec 2021 23:01:10 +0100 Subject: [PATCH 22/24] Cannot use conda env file with pip because doxygen is not a python dependency --- ci/docker/linux-apt-docs.dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile index cd2ff6813f8..3cd62a23d7d 100644 --- a/ci/docker/linux-apt-docs.dockerfile +++ b/ci/docker/linux-apt-docs.dockerfile @@ -74,10 +74,13 @@ RUN wget -q -O - https://deb.nodesource.com/setup_${node}.x | bash - && \ rm -rf /var/lib/apt/lists/* && \ npm install -g yarn -COPY ci/conda_env_sphinx.txt /arrow/ci/ -# unable to install sphinx-tabs from conda-forge due to sphinx version pin, see -# comment in arrow/ci/conda_env_sphinx.txt -RUN pip install -r arrow/ci/conda_env_sphinx.txt meson sphinx-tabs +RUN pip install \ + breathe \ + ipython \ + meson \ + pydata-sphinx-theme \ + sphinx-tabs \ + sphinx>=4.2 COPY c_glib/Gemfile /arrow/c_glib/ RUN gem install --no-document bundler && \ From e1f21207296c4fac3692660c23a2da2cb176e74f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 2 Dec 2021 08:25:03 +0900 Subject: [PATCH 23/24] Fix path --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0391cfc60d6..7e15945b02b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -24,7 +24,7 @@ on: - '.github/workflows/docs.yml' - 'ci/docker/linux-apt-docs.dockerfile' - 'ci/docker/linux-apt-python-3.dockerfile' - - 'ci/docker/ubuntu-20.10-cpp.dockerfile' + - 'ci/docker/ubuntu-20.04-cpp.dockerfile' - 'ci/scripts/c_glib_build.sh' - 'ci/scripts/cpp_build.sh' - 'ci/scripts/docs_build.sh' From a4a63891145625ee5fc2d48ed8d2dceedaecde3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 2 Dec 2021 00:31:19 +0100 Subject: [PATCH 24/24] Increase build timeout --- .github/workflows/docs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7e15945b02b..19bebedb46e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -45,7 +45,7 @@ jobs: name: AMD64 Ubuntu 20.04 Complete Documentation runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 120 + timeout-minutes: 150 env: UBUNTU: "20.04" steps: @@ -64,7 +64,7 @@ jobs: key: ubuntu-docs-${{ hashFiles('cpp/**') }} restore-keys: ubuntu-docs- - name: Setup Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: 3.8 - name: Setup Archery