diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000000..19bebedb46e --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Docs + +on: + push: + pull_request: + paths: + - '.github/workflows/docs.yml' + - 'ci/docker/linux-apt-docs.dockerfile' + - 'ci/docker/linux-apt-python-3.dockerfile' + - 'ci/docker/ubuntu-20.04-cpp.dockerfile' + - 'ci/scripts/c_glib_build.sh' + - 'ci/scripts/cpp_build.sh' + - 'ci/scripts/docs_build.sh' + - 'ci/scripts/java_build.sh' + - 'ci/scripts/js_build.sh' + - 'ci/scripts/python_build.sh' + - 'ci/scripts/r_build.sh' + +env: + ARROW_ENABLE_TIMING_TESTS: OFF + DOCKER_VOLUME_PREFIX: ".docker/" + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + +jobs: + + docker: + name: AMD64 Ubuntu 20.04 Complete Documentation + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 150 + env: + UBUNTU: "20.04" + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + run: ci/scripts/util_cleanup.sh + - name: Cache Docker Volumes + uses: actions/cache@v2 + with: + path: .docker + key: ubuntu-docs-${{ hashFiles('cpp/**') }} + restore-keys: ubuntu-docs- + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: archery docker run ubuntu-docs + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + run: archery docker push ubuntu-docs diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 2afb7532318..5e861ecf2cf 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -58,7 +58,7 @@ jobs: name: AMD64 Ubuntu ${{ matrix.ubuntu }} GLib & Ruby runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 40 + timeout-minutes: 60 strategy: fail-fast: false matrix: diff --git a/ci/conda_env_sphinx.txt b/ci/conda_env_sphinx.txt index ca82705e73c..9f6c99b132e 100644 --- a/ci/conda_env_sphinx.txt +++ b/ci/conda_env_sphinx.txt @@ -19,6 +19,9 @@ breathe doxygen ipython -sphinx=4.2 +sphinx>=4.2 pydata-sphinx-theme -sphinx-tabs + +# Unable to install sphinx-tabs from conda-forge due to: +# - package sphinx-tabs-1.2.1-py_0 requires sphinx >=2,<4, but none of the providers can be installed +# sphinx-tabs diff --git a/ci/docker/conda-python.dockerfile b/ci/docker/conda-python.dockerfile index 3749372e4f4..ae7abd9c9db 100644 --- a/ci/docker/conda-python.dockerfile +++ b/ci/docker/conda-python.dockerfile @@ -21,14 +21,21 @@ FROM ${repo}:${arch}-conda-cpp # install python specific packages ARG python=3.6 -COPY ci/conda_env_python.txt /arrow/ci/ +COPY ci/conda_env_python.txt \ + ci/conda_env_sphinx.txt \ + /arrow/ci/ RUN mamba install -q \ --file arrow/ci/conda_env_python.txt \ + --file arrow/ci/conda_env_sphinx.txt \ $([ "$python" == "3.6" -o "$python" == "3.7" ] && echo "pickle5") \ python=${python} \ nomkl && \ mamba clean --all +# unable to install from conda-forge due to sphinx version pin, see comment in +# arrow/ci/conda_env_sphinx.txt +RUN pip install sphinx-tabs + ENV ARROW_PYTHON=ON \ ARROW_BUILD_STATIC=OFF \ ARROW_BUILD_TESTS=OFF \ diff --git a/ci/docker/debian-10-js.dockerfile b/ci/docker/debian-10-js.dockerfile index 5bb31f2e32e..f994de4141d 100644 --- a/ci/docker/debian-10-js.dockerfile +++ b/ci/docker/debian-10-js.dockerfile @@ -21,6 +21,12 @@ FROM ${arch}/node:${node} ENV NODE_NO_WARNINGS=1 +# install rsync for copying the generated documentation +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends rsync && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + # TODO(kszucs): # 1. add the files required to install the dependencies to .dockerignore # 2. copy these files to their appropriate path diff --git a/ci/docker/debian-11-js.dockerfile b/ci/docker/debian-11-js.dockerfile index 5bb31f2e32e..f994de4141d 100644 --- a/ci/docker/debian-11-js.dockerfile +++ b/ci/docker/debian-11-js.dockerfile @@ -21,6 +21,12 @@ FROM ${arch}/node:${node} ENV NODE_NO_WARNINGS=1 +# install rsync for copying the generated documentation +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends rsync && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + # TODO(kszucs): # 1. add the files required to install the dependencies to .dockerignore # 2. copy these files to their appropriate path diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile index eac59dffb3e..3cd62a23d7d 100644 --- a/ci/docker/linux-apt-docs.dockerfile +++ b/ci/docker/linux-apt-docs.dockerfile @@ -74,15 +74,13 @@ RUN wget -q -O - https://deb.nodesource.com/setup_${node}.x | bash - && \ rm -rf /var/lib/apt/lists/* && \ npm install -g yarn -# ARROW-13353: breathe >= 4.29.1 tries to parse template arguments, -# but Sphinx can't parse constructs like `typename...`. RUN pip install \ - meson \ - breathe==4.29.0 \ + breathe \ ipython \ + meson \ pydata-sphinx-theme \ - sphinx \ - sphinx-tabs + sphinx-tabs \ + sphinx>=4.2 COPY c_glib/Gemfile /arrow/c_glib/ RUN gem install --no-document bundler && \ @@ -107,4 +105,4 @@ ENV ARROW_FLIGHT=ON \ ARROW_BUILD_TESTS=OFF \ ARROW_BUILD_UTILITIES=OFF \ ARROW_USE_GLOG=OFF \ - CMAKE_UNITY_BUILD=ON \ + CMAKE_UNITY_BUILD=ON diff --git a/ci/docker/ubuntu-18.04-cpp.dockerfile b/ci/docker/ubuntu-18.04-cpp.dockerfile index 0c05ac4ee6b..d3023f33794 100644 --- a/ci/docker/ubuntu-18.04-cpp.dockerfile +++ b/ci/docker/ubuntu-18.04-cpp.dockerfile @@ -83,6 +83,7 @@ RUN apt-get update -y -q && \ pkg-config \ protobuf-compiler \ rapidjson-dev \ + rsync \ tzdata && \ apt-get clean && \ rm -rf /var/lib/apt/lists* diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index c50f71bc913..55d7e86cd97 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -92,6 +92,7 @@ RUN apt-get update -y -q && \ python3-rados \ rados-objclass-dev \ rapidjson-dev \ + rsync \ tzdata \ wget && \ apt-get clean && \ diff --git a/ci/docker/ubuntu-21.04-cpp.dockerfile b/ci/docker/ubuntu-21.04-cpp.dockerfile index 2017d26b5bd..0fa145f5d97 100644 --- a/ci/docker/ubuntu-21.04-cpp.dockerfile +++ b/ci/docker/ubuntu-21.04-cpp.dockerfile @@ -89,6 +89,7 @@ RUN apt-get update -y -q && \ protobuf-compiler-grpc \ python3-pip \ rapidjson-dev \ + rsync \ tzdata \ wget && \ apt-get clean && \ diff --git a/ci/scripts/c_glib_build.sh b/ci/scripts/c_glib_build.sh index ce3cea18e71..e0baf7f3d13 100755 --- a/ci/scripts/c_glib_build.sh +++ b/ci/scripts/c_glib_build.sh @@ -21,8 +21,11 @@ set -ex source_dir=${1}/c_glib build_dir=${2}/c_glib -: ${ARROW_GLIB_GTK_DOC:=false} +build_root=${2} + : ${ARROW_GLIB_DEVELOPMENT_MODE:=false} +: ${BUILD_DOCS_C_GLIB:=OFF} +with_gtk_doc=$([ "${BUILD_DOCS_C_GLIB}" == "ON" ] && echo "true" || echo "false") export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig @@ -35,7 +38,7 @@ mkdir -p ${build_dir} meson --prefix=$ARROW_HOME \ --libdir=lib \ -Ddevelopment_mode=${ARROW_GLIB_DEVELOPMENT_MODE} \ - -Dgtk_doc=${ARROW_GLIB_GTK_DOC} \ + -Dgtk_doc=${with_gtk_doc} \ ${build_dir} \ ${source_dir} @@ -43,3 +46,8 @@ pushd ${build_dir} ninja ninja install popd + +if [ "${BUILD_DOCS_C_GLIB}" == "ON" ]; then + mkdir -p ${build_root}/docs/c_glib + rsync -a ${ARROW_HOME}/share/gtk-doc/html/ ${build_root}/docs/c_glib +fi diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh index 0ea9b1b89dc..f791ddd5645 100755 --- a/ci/scripts/cpp_build.sh +++ b/ci/scripts/cpp_build.sh @@ -21,9 +21,9 @@ set -ex source_dir=${1}/cpp build_dir=${2}/cpp -with_docs=${3:-false} : ${ARROW_USE_CCACHE:=OFF} +: ${BUILD_DOCS_CPP:=OFF} # TODO(kszucs): consider to move these to CMake if [ ! -z "${CONDA_PREFIX}" ]; then @@ -49,97 +49,98 @@ fi mkdir -p ${build_dir} pushd ${build_dir} -cmake -G "${CMAKE_GENERATOR:-Ninja}" \ - -DARROW_BOOST_USE_SHARED=${ARROW_BOOST_USE_SHARED:-ON} \ - -DARROW_BUILD_BENCHMARKS=${ARROW_BUILD_BENCHMARKS:-OFF} \ - -DARROW_BUILD_BENCHMARKS_REFERENCE=${ARROW_BUILD_BENCHMARKS:-OFF} \ - -DARROW_BUILD_EXAMPLES=${ARROW_BUILD_EXAMPLES:-OFF} \ - -DARROW_BUILD_INTEGRATION=${ARROW_BUILD_INTEGRATION:-OFF} \ - -DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED:-ON} \ - -DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC:-ON} \ - -DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS:-OFF} \ - -DARROW_BUILD_UTILITIES=${ARROW_BUILD_UTILITIES:-ON} \ - -DARROW_COMPUTE=${ARROW_COMPUTE:-ON} \ - -DARROW_CSV=${ARROW_CSV:-ON} \ - -DARROW_CUDA=${ARROW_CUDA:-OFF} \ - -DARROW_CXXFLAGS=${ARROW_CXXFLAGS:-} \ - -DARROW_DATASET=${ARROW_DATASET:-ON} \ - -DARROW_ENGINE=${ARROW_ENGINE:-ON} \ - -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-AUTO} \ - -DARROW_EXTRA_ERROR_CONTEXT=${ARROW_EXTRA_ERROR_CONTEXT:-OFF} \ - -DARROW_ENABLE_TIMING_TESTS=${ARROW_ENABLE_TIMING_TESTS:-ON} \ - -DARROW_FILESYSTEM=${ARROW_FILESYSTEM:-ON} \ - -DARROW_FLIGHT=${ARROW_FLIGHT:-OFF} \ - -DARROW_FUZZING=${ARROW_FUZZING:-OFF} \ - -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \ - -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \ - -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \ - -DARROW_GCS=${ARROW_GCS:-OFF} \ - -DARROW_HDFS=${ARROW_HDFS:-ON} \ - -DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \ - -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \ - -DARROW_JEMALLOC=${ARROW_JEMALLOC:-ON} \ - -DARROW_JNI=${ARROW_JNI:-OFF} \ - -DARROW_JSON=${ARROW_JSON:-ON} \ - -DARROW_LARGE_MEMORY_TESTS=${ARROW_LARGE_MEMORY_TESTS:-OFF} \ - -DARROW_MIMALLOC=${ARROW_MIMALLOC:-OFF} \ - -DARROW_NO_DEPRECATED_API=${ARROW_NO_DEPRECATED_API:-OFF} \ - -DARROW_ORC=${ARROW_ORC:-OFF} \ - -DARROW_PARQUET=${ARROW_PARQUET:-OFF} \ - -DARROW_PLASMA_JAVA_CLIENT=${ARROW_PLASMA_JAVA_CLIENT:-OFF} \ - -DARROW_PLASMA=${ARROW_PLASMA:-OFF} \ - -DARROW_PYTHON=${ARROW_PYTHON:-OFF} \ - -DARROW_RUNTIME_SIMD_LEVEL=${ARROW_RUNTIME_SIMD_LEVEL:-MAX} \ - -DARROW_S3=${ARROW_S3:-OFF} \ - -DARROW_SKYHOOK=${ARROW_SKYHOOK:-OFF} \ - -DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \ - -DARROW_TEST_MEMCHECK=${ARROW_TEST_MEMCHECK:-OFF} \ - -DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \ - -DARROW_USE_CCACHE=${ARROW_USE_CCACHE:-ON} \ - -DARROW_USE_GLOG=${ARROW_USE_GLOG:-OFF} \ - -DARROW_USE_LD_GOLD=${ARROW_USE_LD_GOLD:-OFF} \ - -DARROW_USE_PRECOMPILED_HEADERS=${ARROW_USE_PRECOMPILED_HEADERS:-OFF} \ - -DARROW_USE_STATIC_CRT=${ARROW_USE_STATIC_CRT:-OFF} \ - -DARROW_USE_TSAN=${ARROW_USE_TSAN:-OFF} \ - -DARROW_USE_UBSAN=${ARROW_USE_UBSAN:-OFF} \ - -DARROW_VERBOSE_THIRDPARTY_BUILD=${ARROW_VERBOSE_THIRDPARTY_BUILD:-OFF} \ - -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-OFF} \ - -DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-OFF} \ - -DARROW_WITH_LZ4=${ARROW_WITH_LZ4:-OFF} \ - -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY:-OFF} \ - -DARROW_WITH_UTF8PROC=${ARROW_WITH_UTF8PROC:-ON} \ - -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-OFF} \ - -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-OFF} \ - -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \ - -Dbenchmark_SOURCE=${benchmark_SOURCE:-} \ - -DBOOST_SOURCE=${BOOST_SOURCE:-} \ - -DBrotli_SOURCE=${Brotli_SOURCE:-} \ - -DBUILD_WARNING_LEVEL=${BUILD_WARNING_LEVEL:-CHECKIN} \ - -Dc-ares_SOURCE=${cares_SOURCE:-} \ - -DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \ - -DCMAKE_C_FLAGS="${CFLAGS:-}" \ - -DCMAKE_CXX_FLAGS="${CXXFLAGS:-}" \ - -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \ - -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \ - -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ - -Dgflags_SOURCE=${gflags_SOURCE:-} \ - -Dgoogle_cloud_cpp_storage_SOURCE=${google_cloud_cpp_storage_SOURCE:-} \ - -DgRPC_SOURCE=${gRPC_SOURCE:-} \ - -DGTest_SOURCE=${GTest_SOURCE:-} \ - -DLz4_SOURCE=${Lz4_SOURCE:-} \ - -DORC_SOURCE=${ORC_SOURCE:-} \ - -DPARQUET_BUILD_EXECUTABLES=${PARQUET_BUILD_EXECUTABLES:-OFF} \ - -DPARQUET_BUILD_EXAMPLES=${PARQUET_BUILD_EXAMPLES:-OFF} \ - -DPARQUET_REQUIRE_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION:-ON} \ - -DProtobuf_SOURCE=${Protobuf_SOURCE:-} \ - -DRapidJSON_SOURCE=${RapidJSON_SOURCE:-} \ - -Dre2_SOURCE=${re2_SOURCE:-} \ - -DSnappy_SOURCE=${Snappy_SOURCE:-} \ - -DThrift_SOURCE=${Thrift_SOURCE:-} \ - -Dutf8proc_SOURCE=${utf8proc_SOURCE:-} \ - -Dzstd_SOURCE=${zstd_SOURCE:-} \ - ${CMAKE_ARGS} \ - ${source_dir} +cmake \ + -DARROW_BOOST_USE_SHARED=${ARROW_BOOST_USE_SHARED:-ON} \ + -DARROW_BUILD_BENCHMARKS_REFERENCE=${ARROW_BUILD_BENCHMARKS:-OFF} \ + -DARROW_BUILD_BENCHMARKS=${ARROW_BUILD_BENCHMARKS:-OFF} \ + -DARROW_BUILD_EXAMPLES=${ARROW_BUILD_EXAMPLES:-OFF} \ + -DARROW_BUILD_INTEGRATION=${ARROW_BUILD_INTEGRATION:-OFF} \ + -DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED:-ON} \ + -DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC:-ON} \ + -DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS:-OFF} \ + -DARROW_BUILD_UTILITIES=${ARROW_BUILD_UTILITIES:-ON} \ + -DARROW_COMPUTE=${ARROW_COMPUTE:-ON} \ + -DARROW_CSV=${ARROW_CSV:-ON} \ + -DARROW_CUDA=${ARROW_CUDA:-OFF} \ + -DARROW_CXXFLAGS=${ARROW_CXXFLAGS:-} \ + -DARROW_DATASET=${ARROW_DATASET:-ON} \ + -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-AUTO} \ + -DARROW_ENABLE_TIMING_TESTS=${ARROW_ENABLE_TIMING_TESTS:-ON} \ + -DARROW_ENGINE=${ARROW_ENGINE:-ON} \ + -DARROW_EXTRA_ERROR_CONTEXT=${ARROW_EXTRA_ERROR_CONTEXT:-OFF} \ + -DARROW_FILESYSTEM=${ARROW_FILESYSTEM:-ON} \ + -DARROW_FLIGHT=${ARROW_FLIGHT:-OFF} \ + -DARROW_FUZZING=${ARROW_FUZZING:-OFF} \ + -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \ + -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \ + -DARROW_GANDIVA=${ARROW_GANDIVA:-OFF} \ + -DARROW_GCS=${ARROW_GCS:-OFF} \ + -DARROW_HDFS=${ARROW_HDFS:-ON} \ + -DARROW_HIVESERVER2=${ARROW_HIVESERVER2:-OFF} \ + -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \ + -DARROW_JEMALLOC=${ARROW_JEMALLOC:-ON} \ + -DARROW_JNI=${ARROW_JNI:-OFF} \ + -DARROW_JSON=${ARROW_JSON:-ON} \ + -DARROW_LARGE_MEMORY_TESTS=${ARROW_LARGE_MEMORY_TESTS:-OFF} \ + -DARROW_MIMALLOC=${ARROW_MIMALLOC:-OFF} \ + -DARROW_NO_DEPRECATED_API=${ARROW_NO_DEPRECATED_API:-OFF} \ + -DARROW_ORC=${ARROW_ORC:-OFF} \ + -DARROW_PARQUET=${ARROW_PARQUET:-OFF} \ + -DARROW_PLASMA_JAVA_CLIENT=${ARROW_PLASMA_JAVA_CLIENT:-OFF} \ + -DARROW_PLASMA=${ARROW_PLASMA:-OFF} \ + -DARROW_PYTHON=${ARROW_PYTHON:-OFF} \ + -DARROW_RUNTIME_SIMD_LEVEL=${ARROW_RUNTIME_SIMD_LEVEL:-MAX} \ + -DARROW_S3=${ARROW_S3:-OFF} \ + -DARROW_SKYHOOK=${ARROW_SKYHOOK:-OFF} \ + -DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \ + -DARROW_TEST_MEMCHECK=${ARROW_TEST_MEMCHECK:-OFF} \ + -DARROW_USE_ASAN=${ARROW_USE_ASAN:-OFF} \ + -DARROW_USE_CCACHE=${ARROW_USE_CCACHE:-ON} \ + -DARROW_USE_GLOG=${ARROW_USE_GLOG:-OFF} \ + -DARROW_USE_LD_GOLD=${ARROW_USE_LD_GOLD:-OFF} \ + -DARROW_USE_PRECOMPILED_HEADERS=${ARROW_USE_PRECOMPILED_HEADERS:-OFF} \ + -DARROW_USE_STATIC_CRT=${ARROW_USE_STATIC_CRT:-OFF} \ + -DARROW_USE_TSAN=${ARROW_USE_TSAN:-OFF} \ + -DARROW_USE_UBSAN=${ARROW_USE_UBSAN:-OFF} \ + -DARROW_VERBOSE_THIRDPARTY_BUILD=${ARROW_VERBOSE_THIRDPARTY_BUILD:-OFF} \ + -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI:-OFF} \ + -DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-OFF} \ + -DARROW_WITH_LZ4=${ARROW_WITH_LZ4:-OFF} \ + -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY:-OFF} \ + -DARROW_WITH_UTF8PROC=${ARROW_WITH_UTF8PROC:-ON} \ + -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB:-OFF} \ + -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-OFF} \ + -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \ + -Dbenchmark_SOURCE=${benchmark_SOURCE:-} \ + -DBOOST_SOURCE=${BOOST_SOURCE:-} \ + -DBrotli_SOURCE=${Brotli_SOURCE:-} \ + -DBUILD_WARNING_LEVEL=${BUILD_WARNING_LEVEL:-CHECKIN} \ + -Dc-ares_SOURCE=${cares_SOURCE:-} \ + -DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \ + -DCMAKE_C_FLAGS="${CFLAGS:-}" \ + -DCMAKE_CXX_FLAGS="${CXXFLAGS:-}" \ + -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \ + -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \ + -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ + -Dgflags_SOURCE=${gflags_SOURCE:-} \ + -Dgoogle_cloud_cpp_storage_SOURCE=${google_cloud_cpp_storage_SOURCE:-} \ + -DgRPC_SOURCE=${gRPC_SOURCE:-} \ + -DGTest_SOURCE=${GTest_SOURCE:-} \ + -DLz4_SOURCE=${Lz4_SOURCE:-} \ + -DORC_SOURCE=${ORC_SOURCE:-} \ + -DPARQUET_BUILD_EXAMPLES=${PARQUET_BUILD_EXAMPLES:-OFF} \ + -DPARQUET_BUILD_EXECUTABLES=${PARQUET_BUILD_EXECUTABLES:-OFF} \ + -DPARQUET_REQUIRE_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION:-ON} \ + -DProtobuf_SOURCE=${Protobuf_SOURCE:-} \ + -DRapidJSON_SOURCE=${RapidJSON_SOURCE:-} \ + -Dre2_SOURCE=${re2_SOURCE:-} \ + -DSnappy_SOURCE=${Snappy_SOURCE:-} \ + -DThrift_SOURCE=${Thrift_SOURCE:-} \ + -Dutf8proc_SOURCE=${utf8proc_SOURCE:-} \ + -Dzstd_SOURCE=${zstd_SOURCE:-} \ + -G "${CMAKE_GENERATOR:-Ninja}" \ + ${CMAKE_ARGS} \ + ${source_dir} if [ ! -z "${CPP_MAKE_PARALLELISM}" ]; then time cmake --build . --target install -- -j${CPP_MAKE_PARALLELISM} @@ -158,7 +159,7 @@ if [ "${ARROW_USE_CCACHE}" == "ON" ]; then ccache -s fi -if [ "${with_docs}" == "true" ]; then +if [ "${DOCS_BUILD_CPP}" == "ON" ]; then pushd ${source_dir}/apidoc doxygen popd diff --git a/ci/scripts/docs_build.sh b/ci/scripts/docs_build.sh deleted file mode 100755 index e6ee768ee87..00000000000 --- a/ci/scripts/docs_build.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -ex - -arrow_dir=${1} -build_dir=${2}/docs - -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} -export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig:${PKG_CONFIG_PATH} -export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0 -export CFLAGS="-DARROW_NO_DEPRECATED_API" -export CXXFLAGS="-DARROW_NO_DEPRECATED_API" - -ncpus=$(python3 -c "import os; print(os.cpu_count())") - -# Sphinx docs -sphinx-build -b html -j ${ncpus} ${arrow_dir}/docs/source ${build_dir} - -# C++ - original doxygen -# rsync -a ${arrow_dir}/cpp/apidoc/ ${build_dir}/cpp - -# R -rsync -a ${arrow_dir}/r/docs/ ${build_dir}/r - -# C GLib -rsync -a ${ARROW_HOME}/share/gtk-doc/html/ ${build_dir}/c_glib - -# Java -rsync -a ${arrow_dir}/java/target/site/apidocs/ ${build_dir}/java/reference - -# Javascript -rsync -a ${arrow_dir}/js/doc/ ${build_dir}/js diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh index 1ba37606d3c..7693bd883a1 100755 --- a/ci/scripts/java_build.sh +++ b/ci/scripts/java_build.sh @@ -20,9 +20,11 @@ set -ex arrow_dir=${1} source_dir=${1}/java -cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug} -cdata_dist_dir=${2}/java/c -with_docs=${3:-false} +build_dir=${2} +cpp_build_dir=${build_dir}/cpp/${ARROW_BUILD_TYPE:-debug} +cdata_dist_dir=${build_dir}/java/c + +: ${BUILD_DOCS_JAVA:=OFF} if [[ "$(uname -s)" == "Linux" ]] && [[ "$(uname -m)" == "s390x" ]]; then # Since some files for s390_64 are not available at maven central, @@ -99,9 +101,11 @@ if [ "${ARROW_PLASMA}" = "ON" ]; then popd fi -if [ "${with_docs}" == "true" ]; then +if [ "${BUILD_DOCS_JAVA}" == "ON" ]; then # HTTP pooling is turned of to avoid download issues https://issues.apache.org/jira/browse/ARROW-11633 + mkdir -p ${build_dir}/docs/java/reference ${mvn} -Dcheckstyle.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false install site + rsync -a ${arrow_dir}/java/target/site/apidocs/ ${build_dir}/docs/java/reference fi popd diff --git a/ci/scripts/js_build.sh b/ci/scripts/js_build.sh index 12f58d54bb8..333c9cf441e 100755 --- a/ci/scripts/js_build.sh +++ b/ci/scripts/js_build.sh @@ -19,17 +19,19 @@ set -ex -source_dir=${1}/js -with_docs=${2:-false} +arrow_dir=${1} +source_dir=${arrow_dir}/js +build_dir=${2} + +: ${BUILD_DOCS_JS:=OFF} pushd ${source_dir} yarn --frozen-lockfile -# TODO(kszucs): linting should be moved to archery yarn lint:ci yarn build -if [ "${with_docs}" == "true" ]; then +if [ "${BUILD_DOCS_JS}" == "ON" ]; then if [ "$(git config --get remote.origin.url)" == "https://github.com/apache/arrow.git" ]; then yarn doc elif [ "$(git config --get remote.upstream.url)" == "https://github.com/apache/arrow.git" ]; then @@ -40,6 +42,8 @@ if [ "${with_docs}" == "true" ]; then echo "Failed to build docs because the remote is not set correctly. Please set the origin or upstream remote to https://github.com/apache/arrow.git or the apache remote to git@github.com:apache/arrow.git." exit 0 fi + mkdir -p ${build_dir}/docs/js + rsync -a ${arrow_dir}/js/doc/ ${build_dir}/docs/js fi popd diff --git a/ci/scripts/python_build.sh b/ci/scripts/python_build.sh index ec6d723b2a7..ccf068078d5 100755 --- a/ci/scripts/python_build.sh +++ b/ci/scripts/python_build.sh @@ -19,9 +19,12 @@ set -ex -source_dir=${1}/python +arrow_dir=${1} +source_dir=${arrow_dir}/python build_dir=${2}/python +: ${BUILD_DOCS_PYTHON:=OFF} + if [ ! -z "${CONDA_PREFIX}" ]; then echo -e "===\n=== Conda environment for build\n===" conda list @@ -52,3 +55,8 @@ ${PYTHON:-python} \ --record $relative_build_dir/record.txt popd + +if [ "${BUILD_DOCS_PYTHON}" == "ON" ]; then + ncpus=$(python -c "import os; print(os.cpu_count())") + sphinx-build -b html -j ${ncpus} ${arrow_dir}/docs/source ${build_dir}/docs +fi diff --git a/ci/scripts/r_build.sh b/ci/scripts/r_build.sh index 2a2b9d7d161..a435b04317d 100755 --- a/ci/scripts/r_build.sh +++ b/ci/scripts/r_build.sh @@ -20,14 +20,17 @@ set -ex : ${R_BIN:=R} source_dir=${1}/r -with_docs=${2:-false} +build_dir=${2} + +: ${BUILD_DOCS_R:=OFF} pushd ${source_dir} ${R_BIN} CMD INSTALL ${INSTALL_ARGS} . -if [ "${with_docs}" == "true" ]; then +if [ "${BUILD_DOCS_R}" == "ON" ]; then ${R_BIN} -e "pkgdown::build_site(install = FALSE)" + rsync -a ${source_dir}/docs/ ${build_dir}/docs/r fi -popd \ No newline at end of file +popd diff --git a/docker-compose.yml b/docker-compose.yml index a38210cc5e0..9bf5bd0841d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -236,11 +236,12 @@ services: ARROW_MIMALLOC: "ON" ARROW_USE_LD_GOLD: "ON" ARROW_USE_PRECOMPILED_HEADERS: "ON" + BUILD_DOCS_PYTHON: "ON" volumes: &conda-volumes - .:/arrow:delegated - ${DOCKER_VOLUME_PREFIX}conda-ccache:/ccache:delegated command: - ["/arrow/ci/scripts/cpp_build.sh /arrow /build true && + ["/arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/cpp_test.sh /arrow /build"] conda-cpp-valgrind: @@ -513,7 +514,7 @@ services: ulimits: *ulimits environment: <<: *ccache - ARROW_GLIB_GTK_DOC: "true" + BUILD_DOCS_C_GLIB: "ON" volumes: *debian-volumes command: &c-glib-command > /bin/bash -c " @@ -541,7 +542,6 @@ services: ulimits: *ulimits environment: <<: *ccache - ARROW_GLIB_GTK_DOC: "true" volumes: *ubuntu-volumes command: *c-glib-command @@ -572,6 +572,7 @@ services: ulimits: *ulimits environment: <<: *ccache + BUILD_DOCS_C_GLIB: "ON" volumes: *debian-volumes command: &ruby-command > /bin/bash -c " @@ -939,8 +940,12 @@ services: shm_size: *shm-size environment: <<: *ccache + BUILD_DOCS_PYTHON: "ON" volumes: *conda-volumes - command: *python-conda-command + command: + ["/arrow/ci/scripts/cpp_build.sh /arrow /build && + /arrow/ci/scripts/python_build.sh /arrow /build && + /arrow/ci/scripts/python_test.sh /arrow"] conda-python-dask: # Possible $DASK parameters: @@ -1288,8 +1293,6 @@ services: command: > /bin/bash -c "/arrow/ci/scripts/r_revdepcheck.sh /arrow" - - ################################# Go ######################################## debian-go: @@ -1365,9 +1368,11 @@ services: node: ${NODE} shm_size: *shm-size volumes: *debian-volumes + environment: + BUILD_DOCS_JS: "ON" command: &js-command > /bin/bash -c " - /arrow/ci/scripts/js_build.sh /arrow && + /arrow/ci/scripts/js_build.sh /arrow /build && /arrow/ci/scripts/js_test.sh /arrow" #################################### C# ##################################### @@ -1506,18 +1511,22 @@ services: environment: <<: *ccache ARROW_CUDA: "ON" - ARROW_GLIB_GTK_DOC: "true" + BUILD_DOCS_C_GLIB: "ON" + BUILD_DOCS_CPP: "ON" + BUILD_DOCS_JAVA: "ON" + BUILD_DOCS_JS: "ON" + BUILD_DOCS_PYTHON: "ON" + BUILD_DOCS_R: "ON" Protobuf_SOURCE: "BUNDLED" # Need Protobuf >= 3.15 volumes: *ubuntu-volumes command: &docs-command > /bin/bash -c " - /arrow/ci/scripts/cpp_build.sh /arrow /build true && + /arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/c_glib_build.sh /arrow /build && /arrow/ci/scripts/python_build.sh /arrow /build && - /arrow/ci/scripts/java_build.sh /arrow /build true && - /arrow/ci/scripts/js_build.sh /arrow true && - /arrow/ci/scripts/r_build.sh /arrow true && - /arrow/ci/scripts/docs_build.sh /arrow /build" + /arrow/ci/scripts/r_build.sh /arrow /build && + /arrow/ci/scripts/js_build.sh /arrow /build && + /arrow/ci/scripts/java_build.sh /arrow /build" ################################# Tools #####################################