ARROW-14505: [CI][Docs] Exercise documentation builds more frequently

.github/workflows/docs.yml

@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Docs
+
+on:
+  push:
+  pull_request:
+    paths:
+      - '.github/workflows/docs.yml'
+      - 'ci/docker/linux-apt-docs.dockerfile'
+      - 'ci/docker/linux-apt-python-3.dockerfile'
+      - 'ci/docker/ubuntu-20.04-cpp.dockerfile'
+      - 'ci/scripts/c_glib_build.sh'
+      - 'ci/scripts/cpp_build.sh'
+      - 'ci/scripts/docs_build.sh'
+      - 'ci/scripts/java_build.sh'
+      - 'ci/scripts/js_build.sh'
+      - 'ci/scripts/python_build.sh'
+      - 'ci/scripts/r_build.sh'
+
+env:
+  ARROW_ENABLE_TIMING_TESTS: OFF
+  DOCKER_VOLUME_PREFIX: ".docker/"
+  ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
+  ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
+
+jobs:
+
+  docker:
+    name: AMD64 Ubuntu 20.04 Complete Documentation
+    runs-on: ubuntu-latest
+    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    timeout-minutes: 150
+    env:
+      UBUNTU: "20.04"
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Fetch Submodules and Tags
+        run: ci/scripts/util_checkout.sh
+      - name: Free Up Disk Space
+        run: ci/scripts/util_cleanup.sh
+      - name: Cache Docker Volumes
+        uses: actions/cache@v2
+        with:
+          path: .docker
+          key: ubuntu-docs-${{ hashFiles('cpp/**') }}
+          restore-keys: ubuntu-docs-
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Setup Archery
+        run: pip install -e dev/archery[docker]
+      - name: Execute Docker Build
+        run: archery docker run ubuntu-docs
+      - name: Docker Push
+        if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
+        continue-on-error: true
+        run: archery docker push ubuntu-docs

.github/workflows/ruby.yml

@@ -58,7 +58,7 @@ jobs:
     name: AMD64 Ubuntu ${{ matrix.ubuntu }} GLib & Ruby
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
-    timeout-minutes: 40
+    timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:

ci/conda_env_sphinx.txt

@@ -19,6 +19,9 @@
 breathe
 doxygen
 ipython
-sphinx=4.2
+sphinx>=4.2
 pydata-sphinx-theme
-sphinx-tabs
+
+# Unable to install sphinx-tabs from conda-forge due to:
+# - package sphinx-tabs-1.2.1-py_0 requires sphinx >=2,<4, but none of the providers can be installed
+# sphinx-tabs

ci/docker/conda-python.dockerfile

@@ -21,14 +21,21 @@ FROM ${repo}:${arch}-conda-cpp
 
 # install python specific packages
 ARG python=3.6
-COPY ci/conda_env_python.txt /arrow/ci/
+COPY ci/conda_env_python.txt \
+     ci/conda_env_sphinx.txt \
+     /arrow/ci/
 RUN mamba install -q \
         --file arrow/ci/conda_env_python.txt \
+        --file arrow/ci/conda_env_sphinx.txt \
         $([ "$python" == "3.6" -o "$python" == "3.7" ] && echo "pickle5") \
         python=${python} \
         nomkl && \
     mamba clean --all
 
+# unable to install from conda-forge due to sphinx version pin, see comment in
+# arrow/ci/conda_env_sphinx.txt
+RUN pip install sphinx-tabs
+
 ENV ARROW_PYTHON=ON \
     ARROW_BUILD_STATIC=OFF \
     ARROW_BUILD_TESTS=OFF \

ci/docker/debian-10-js.dockerfile

@@ -21,6 +21,12 @@ FROM ${arch}/node:${node}
 
 ENV NODE_NO_WARNINGS=1
 
+# install rsync for copying the generated documentation
+RUN apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends rsync && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
 # TODO(kszucs):
 # 1. add the files required to install the dependencies to .dockerignore
 # 2. copy these files to their appropriate path

ci/docker/debian-11-js.dockerfile

@@ -21,6 +21,12 @@ FROM ${arch}/node:${node}
 
 ENV NODE_NO_WARNINGS=1
 
+# install rsync for copying the generated documentation
+RUN apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends rsync && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
 # TODO(kszucs):
 # 1. add the files required to install the dependencies to .dockerignore
 # 2. copy these files to their appropriate path

ci/docker/linux-apt-docs.dockerfile

@@ -74,15 +74,13 @@ RUN wget -q -O - https://deb.nodesource.com/setup_${node}.x | bash - && \
     rm -rf /var/lib/apt/lists/* && \
     npm install -g yarn
 
-# ARROW-13353: breathe >= 4.29.1 tries to parse template arguments,
-# but Sphinx can't parse constructs like `typename...`.
 RUN pip install \
-        meson \
-        breathe==4.29.0 \
+        breathe \
         ipython \
+        meson \
         pydata-sphinx-theme \
-        sphinx \
-        sphinx-tabs
+        sphinx-tabs \
+        sphinx>=4.2
 
 COPY c_glib/Gemfile /arrow/c_glib/
 RUN gem install --no-document bundler && \
@@ -107,4 +105,4 @@ ENV ARROW_FLIGHT=ON \
     ARROW_BUILD_TESTS=OFF \
     ARROW_BUILD_UTILITIES=OFF \
     ARROW_USE_GLOG=OFF \
-    CMAKE_UNITY_BUILD=ON \
+    CMAKE_UNITY_BUILD=ON

ci/docker/ubuntu-18.04-cpp.dockerfile

@@ -83,6 +83,7 @@ RUN apt-get update -y -q && \
         pkg-config \
         protobuf-compiler \
         rapidjson-dev \
+        rsync \
         tzdata && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists*

ci/docker/ubuntu-20.04-cpp.dockerfile

@@ -92,6 +92,7 @@ RUN apt-get update -y -q && \
         python3-rados \
         rados-objclass-dev \
         rapidjson-dev \
+        rsync \
         tzdata \
         wget && \
     apt-get clean && \

ci/docker/ubuntu-21.04-cpp.dockerfile

@@ -89,6 +89,7 @@ RUN apt-get update -y -q && \
         protobuf-compiler-grpc \
         python3-pip \
         rapidjson-dev \
+        rsync \
         tzdata \
         wget && \
     apt-get clean && \

ci/scripts/c_glib_build.sh

@@ -21,8 +21,11 @@ set -ex
 
 source_dir=${1}/c_glib
 build_dir=${2}/c_glib
-: ${ARROW_GLIB_GTK_DOC:=false}
+build_root=${2}
+
 : ${ARROW_GLIB_DEVELOPMENT_MODE:=false}
+: ${BUILD_DOCS_C_GLIB:=OFF}
+with_gtk_doc=$([ "${BUILD_DOCS_C_GLIB}" == "ON" ] && echo "true" || echo "false")
 
 export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig
 
@@ -35,11 +38,16 @@ mkdir -p ${build_dir}
 meson --prefix=$ARROW_HOME \
       --libdir=lib \
       -Ddevelopment_mode=${ARROW_GLIB_DEVELOPMENT_MODE} \
-      -Dgtk_doc=${ARROW_GLIB_GTK_DOC} \
+      -Dgtk_doc=${with_gtk_doc} \
       ${build_dir} \
       ${source_dir}
 
 pushd ${build_dir}
 ninja
 ninja install
 popd
+
+if [ "${BUILD_DOCS_C_GLIB}" == "ON" ]; then
+  mkdir -p ${build_root}/docs/c_glib
+  rsync -a ${ARROW_HOME}/share/gtk-doc/html/ ${build_root}/docs/c_glib
+fi