From 8804c56b03eae188b7b1190d97caf3870d743a17 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Tue, 30 Nov 2021 17:34:37 +0000 Subject: [PATCH 1/8] Update docs build to run in 21.04 and update to no longer use apt-key --- ci/docker/linux-apt-docs.dockerfile | 5 ++--- ci/docker/linux-apt-lint.dockerfile | 5 ++--- ci/docker/linux-apt-r.dockerfile | 5 ++--- dev/tasks/tasks.yml | 4 ++-- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile index 12c797f9651..870f57460d4 100644 --- a/ci/docker/linux-apt-docs.dockerfile +++ b/ci/docker/linux-apt-docs.dockerfile @@ -27,9 +27,8 @@ RUN apt-get update -y && \ dirmngr \ apt-transport-https \ software-properties-common && \ - apt-key adv \ - --keyserver keyserver.ubuntu.com \ - --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ + wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | \ + tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc && \ add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran40/' && \ apt-get install -y --no-install-recommends \ autoconf-archive \ diff --git a/ci/docker/linux-apt-lint.dockerfile b/ci/docker/linux-apt-lint.dockerfile index 84de6b05f31..a92aa276f9a 100644 --- a/ci/docker/linux-apt-lint.dockerfile +++ b/ci/docker/linux-apt-lint.dockerfile @@ -41,9 +41,8 @@ RUN apt-get update && \ && rm -rf /var/lib/apt/lists/* ARG r=4.1 -RUN apt-key adv \ - --keyserver keyserver.ubuntu.com \ - --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ +RUN wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | \ + tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc && \ # NOTE: R 3.5 and 3.6 are available in the repos with -cran35 suffix # for trusty, xenial, bionic, and eoan (as of May 2020) # -cran40 has 4.0 versions for bionic and focal diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile index 894f53bc0a3..433bd622c3c 100644 --- a/ci/docker/linux-apt-r.dockerfile +++ b/ci/docker/linux-apt-r.dockerfile @@ -34,9 +34,8 @@ RUN apt-get update -y && \ dirmngr \ apt-transport-https \ software-properties-common && \ - apt-key adv \ - --keyserver keyserver.ubuntu.com \ - --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ + wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | \ + tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc && \ # NOTE: R 3.5 and 3.6 are available in the repos with -cran35 suffix # for trusty, xenial, bionic, and eoan (as of May 2020) # -cran40 has 4.0 versions for bionic and focal diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index a4c3d2c9e4f..751fe36b024 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1256,12 +1256,12 @@ tasks: GO: 1.15 run: debian-go - test-ubuntu-20.10-docs: + test-ubuntu-21.04-docs: ci: azure template: docker-tests/azure.linux.yml params: env: - UBUNTU: "20.10" + UBUNTU: "21.04" run: ubuntu-docs test-ubuntu-default-docs: From 518cb8a7ab777a9be6b2256806b26707226a6f99 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Tue, 30 Nov 2021 17:46:41 +0000 Subject: [PATCH 2/8] Change to 20.04 instead as having clang issues --- dev/tasks/tasks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 751fe36b024..76ef0f4793f 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1256,12 +1256,12 @@ tasks: GO: 1.15 run: debian-go - test-ubuntu-21.04-docs: + test-ubuntu-20.04-docs: ci: azure template: docker-tests/azure.linux.yml params: env: - UBUNTU: "21.04" + UBUNTU: "20.04" run: ubuntu-docs test-ubuntu-default-docs: From 9ab33bec1f0d802f412f48ee08e06104f0a99a6f Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Tue, 30 Nov 2021 20:38:31 +0000 Subject: [PATCH 3/8] Add sphinx-tabs to doc dockerfile --- ci/docker/linux-apt-docs.dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile index 870f57460d4..3d731f4d565 100644 --- a/ci/docker/linux-apt-docs.dockerfile +++ b/ci/docker/linux-apt-docs.dockerfile @@ -81,7 +81,8 @@ RUN pip install \ breathe==4.29.0 \ ipython \ sphinx \ - pydata-sphinx-theme + pydata-sphinx-theme\ + sphinx-tabs COPY c_glib/Gemfile /arrow/c_glib/ RUN gem install --no-document bundler && \ From 1cfdfcf8980eef641cd19ed22697e7a63f60972c Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Tue, 30 Nov 2021 21:11:18 +0000 Subject: [PATCH 4/8] Alphabetise ordering --- ci/docker/linux-apt-docs.dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile index 3d731f4d565..325729ae56e 100644 --- a/ci/docker/linux-apt-docs.dockerfile +++ b/ci/docker/linux-apt-docs.dockerfile @@ -81,8 +81,8 @@ RUN pip install \ breathe==4.29.0 \ ipython \ sphinx \ - pydata-sphinx-theme\ - sphinx-tabs + sphinx-tabs\ + pydata-sphinx-theme COPY c_glib/Gemfile /arrow/c_glib/ RUN gem install --no-document bundler && \ From e73499713aa50c31a4e6d24ede4fc6343ba4f767 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Tue, 30 Nov 2021 21:25:05 +0000 Subject: [PATCH 5/8] Remove redundant job --- dev/tasks/tasks.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 76ef0f4793f..58749e2f3a6 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1256,14 +1256,6 @@ tasks: GO: 1.15 run: debian-go - test-ubuntu-20.04-docs: - ci: azure - template: docker-tests/azure.linux.yml - params: - env: - UBUNTU: "20.04" - run: ubuntu-docs - test-ubuntu-default-docs: ci: azure template: docker-tests/azure.linux.yml From 53e687c17de6eaee8914bea59388f77826105384 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Tue, 30 Nov 2021 21:28:07 +0000 Subject: [PATCH 6/8] alphabetise other list --- ci/docker/linux-apt-docs.dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile index 325729ae56e..0a4f470c9bf 100644 --- a/ci/docker/linux-apt-docs.dockerfile +++ b/ci/docker/linux-apt-docs.dockerfile @@ -80,9 +80,9 @@ RUN pip install \ meson \ breathe==4.29.0 \ ipython \ + pydata-sphinx-theme\ sphinx \ - sphinx-tabs\ - pydata-sphinx-theme + sphinx-tabs COPY c_glib/Gemfile /arrow/c_glib/ RUN gem install --no-document bundler && \ From 4092a64b67a793f98dd9fc4686da84826e8d7e28 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Tue, 30 Nov 2021 21:31:19 +0000 Subject: [PATCH 7/8] Update ci/docker/linux-apt-docs.dockerfile Co-authored-by: Sutou Kouhei --- ci/docker/linux-apt-docs.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile index 0a4f470c9bf..eac59dffb3e 100644 --- a/ci/docker/linux-apt-docs.dockerfile +++ b/ci/docker/linux-apt-docs.dockerfile @@ -80,7 +80,7 @@ RUN pip install \ meson \ breathe==4.29.0 \ ipython \ - pydata-sphinx-theme\ + pydata-sphinx-theme \ sphinx \ sphinx-tabs From 69c79df27bf45af09719767b2f6c1998f1e87241 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Wed, 1 Dec 2021 08:43:10 +0000 Subject: [PATCH 8/8] Remove redundant dockerfile and note to build ubuntu-docs on UBUNTU 20.10 in release notes --- ci/docker/ubuntu-20.10-cpp.dockerfile | 140 -------------------------- dev/release/post-09-docs.sh | 2 +- 2 files changed, 1 insertion(+), 141 deletions(-) delete mode 100644 ci/docker/ubuntu-20.10-cpp.dockerfile diff --git a/ci/docker/ubuntu-20.10-cpp.dockerfile b/ci/docker/ubuntu-20.10-cpp.dockerfile deleted file mode 100644 index 59f5fa4c886..00000000000 --- a/ci/docker/ubuntu-20.10-cpp.dockerfile +++ /dev/null @@ -1,140 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG base=amd64/ubuntu:20.10 -FROM ${base} -ARG arch - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN echo "debconf debconf/frontend select Noninteractive" | \ - debconf-set-selections - -# Installs LLVM toolchain, for Gandiva and testing other compilers -# -# Note that this is installed before the base packages to improve iteration -# while debugging package list with docker build. -ARG clang_tools -ARG llvm -RUN if [ "${llvm}" -gt "10" ]; then \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - gnupg \ - wget && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb https://apt.llvm.org/groovy/ llvm-toolchain-groovy-${llvm} main" > \ - /etc/apt/sources.list.d/llvm.list && \ - if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \ - echo "deb https://apt.llvm.org/groovy/ llvm-toolchain-groovy-${clang_tools} main" > \ - /etc/apt/sources.list.d/clang-tools.list; \ - fi \ - fi && \ - apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - clang-${clang_tools} \ - clang-${llvm} \ - clang-format-${clang_tools} \ - clang-tidy-${clang_tools} \ - llvm-${llvm}-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -# Installs C++ toolchain and dependencies -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - autoconf \ - ca-certificates \ - ccache \ - cmake \ - g++ \ - gcc \ - gdb \ - git \ - libbenchmark-dev \ - libboost-filesystem-dev \ - libboost-system-dev \ - libbrotli-dev \ - libbz2-dev \ - libc-ares-dev \ - libcurl4-openssl-dev \ - libgflags-dev \ - libgoogle-glog-dev \ - libgrpc++-dev \ - liblz4-dev \ - libprotobuf-dev \ - libprotoc-dev \ - libre2-dev \ - libsnappy-dev \ - libssl-dev \ - libthrift-dev \ - libutf8proc-dev \ - libzstd-dev \ - make \ - ninja-build \ - pkg-config \ - protobuf-compiler \ - protobuf-compiler-grpc \ - python3-pip \ - rapidjson-dev \ - tzdata \ - wget && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists* - -COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local -COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_gcs_testbench.sh ${arch} default - -# Prioritize system packages and local installation -# The following dependencies will be downloaded due to missing/invalid packages -# provided by the distribution: -# - libc-ares-dev does not install CMake config files -# - flatbuffer is not packaged -# - libgtest-dev only provide sources -# - libprotobuf-dev only provide sources -ENV ARROW_BUILD_TESTS=ON \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_DATASET=ON \ - ARROW_FLIGHT=OFF \ - ARROW_GANDIVA=ON \ - ARROW_HDFS=ON \ - ARROW_HOME=/usr/local \ - ARROW_INSTALL_NAME_RPATH=OFF \ - ARROW_NO_DEPRECATED_API=ON \ - ARROW_ORC=ON \ - ARROW_PARQUET=ON \ - ARROW_PLASMA=ON \ - ARROW_S3=ON \ - ARROW_USE_ASAN=OFF \ - ARROW_USE_CCACHE=ON \ - ARROW_USE_UBSAN=OFF \ - ARROW_WITH_BROTLI=ON \ - ARROW_WITH_BZ2=ON \ - ARROW_WITH_LZ4=ON \ - ARROW_WITH_SNAPPY=ON \ - ARROW_WITH_ZLIB=ON \ - ARROW_WITH_ZSTD=ON \ - AWSSDK_SOURCE=BUNDLED \ - GTest_SOURCE=BUNDLED \ - ORC_SOURCE=BUNDLED \ - PARQUET_BUILD_EXAMPLES=ON \ - PARQUET_BUILD_EXECUTABLES=ON \ - PATH=/usr/lib/ccache/:$PATH \ - PYTHON=python3 diff --git a/dev/release/post-09-docs.sh b/dev/release/post-09-docs.sh index 9c0b77bb51c..35c4be8745b 100755 --- a/dev/release/post-09-docs.sh +++ b/dev/release/post-09-docs.sh @@ -43,7 +43,7 @@ popd pushd "${ARROW_DIR}" git checkout "${release_tag}" -UBUNTU=20.10 archery docker run \ +archery docker run \ -v "${ARROW_SITE_DIR}/docs:/build/docs" \ -e ARROW_DOCS_VERSION="${version}" \ ubuntu-docs