From d9c6e6e4e89a55ba53a5c6d2998fffb77d4facbc Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 3 Jun 2022 13:56:48 +0900 Subject: [PATCH 01/14] ARROW-16745: [Packaging][RPM] Add support for AlmaLinux 9 --- c_glib/meson.build | 2 +- dev/release/binary-task.rb | 1 + .../yum/almalinux-9/Dockerfile | 26 ++++++++ .../apache-arrow/yum/almalinux-8/Dockerfile | 3 +- .../apache-arrow/yum/almalinux-9-aarch64/from | 18 +++++ .../apache-arrow/yum/almalinux-9/Dockerfile | 66 +++++++++++++++++++ .../yum/amazon-linux-2/Dockerfile | 3 +- .../apache-arrow/yum/arrow.spec.in | 63 +++++++++++------- .../apache-arrow/yum/centos-7/Dockerfile | 3 +- .../yum/centos-8-stream/Dockerfile | 3 +- dev/tasks/linux-packages/package-task.rb | 2 + dev/tasks/tasks.yml | 33 +++++----- 12 files changed, 180 insertions(+), 43 deletions(-) create mode 100644 dev/tasks/linux-packages/apache-arrow-release/yum/almalinux-9/Dockerfile create mode 100644 dev/tasks/linux-packages/apache-arrow/yum/almalinux-9-aarch64/from create mode 100644 dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile diff --git a/c_glib/meson.build b/c_glib/meson.build index 81352463aeb..e9582cf93b6 100644 --- a/c_glib/meson.build +++ b/c_glib/meson.build @@ -97,7 +97,7 @@ else int main(void) { - arrow::adapters::orc::ORCFileReader::Open(nullptr, nullptr, nullptr); + arrow::adapters::orc::ORCFileWriter::Open(nullptr); return 0; } ''' diff --git a/dev/release/binary-task.rb b/dev/release/binary-task.rb index 8f7fc89e191..751f3d186d2 100644 --- a/dev/release/binary-task.rb +++ b/dev/release/binary-task.rb @@ -1456,6 +1456,7 @@ def yum_release_repositories_dir def available_yum_targets [ + ["almalinux", "9"], ["almalinux", "8"], ["amazon-linux", "2"], ["centos", "8-stream"], diff --git a/dev/tasks/linux-packages/apache-arrow-release/yum/almalinux-9/Dockerfile b/dev/tasks/linux-packages/apache-arrow-release/yum/almalinux-9/Dockerfile new file mode 100644 index 00000000000..a2383d55a39 --- /dev/null +++ b/dev/tasks/linux-packages/apache-arrow-release/yum/almalinux-9/Dockerfile @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM almalinux:9 + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ + dnf install -y ${quiet} \ + rpmdevtools && \ + dnf clean ${quiet} all diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile index df338a2c25c..a75fd022bda 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile @@ -60,7 +60,8 @@ RUN \ rpmdevtools \ snappy-devel \ tar \ + thrift-devel \ # utf8proc-devel \ - vala-devel \ + vala \ zlib-devel && \ dnf clean ${quiet} all diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9-aarch64/from b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9-aarch64/from new file mode 100644 index 00000000000..5fab1520f4e --- /dev/null +++ b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9-aarch64/from @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arm64v8/almalinux:9 diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile new file mode 100644 index 00000000000..df63f7a1a7e --- /dev/null +++ b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG FROM=almalinux:9 +FROM ${FROM} + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ + dnf install -y ${quiet} epel-release && \ + dnf install --enablerepo=crb -y ${quiet} \ + bison \ + boost-devel \ + brotli-devel \ + bzip2-devel \ + c-ares-devel \ + ccache \ + clang \ + cmake \ + curl-devel \ + flex \ + gcc-c++ \ + gflags-devel \ + git \ + gobject-introspection-devel \ + gtk-doc \ + json-devel \ + libarchive \ + libzstd-devel \ + llvm-devel \ + llvm-static \ + lz4-devel \ + make \ + ncurses-devel \ + ninja-build \ + openssl-devel \ + pkg-config \ + python3 \ + python3-devel \ + python3-numpy \ + python3-pip \ + re2-devel \ + rapidjson-devel \ + rpmdevtools \ + snappy-devel \ + tar \ + thrift-devel \ + utf8proc-devel \ + vala \ + zlib-devel && \ + dnf clean ${quiet} all diff --git a/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile index 4b885a2c411..9409b25c3c5 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile @@ -48,8 +48,9 @@ RUN \ rpmdevtools \ snappy-devel \ tar \ + thrift-devel \ utf8proc-devel \ - vala-devel \ + vala \ zlib-devel && \ # Install ninja-build dependencies in amzn2-core yum install -y ${quiet} ninja-build && \ diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index a8c8ccf2c49..fa0520a35d3 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -17,6 +17,19 @@ # specific language governing permissions and limitations # under the License. +%if !%{defined cmake} +%define cmake %{cmake3} +%endif +%if !%{defined cmake_build} +%define cmake_build %{cmake3_build} +%endif +%if !%{defined cmake_install} +%define cmake_install %{cmake3_install} +%endif +%if !%{defined __cmake_builddir} +%define __cmake_builddir %{__cmake3_builddir} +%endif + %define _amzn %{?amzn:%{amzn}}%{!?amzn:0} %define is_amazon_linux (%{_amzn} != 0) @@ -30,6 +43,12 @@ if [ %{rhel} -lt 8 ]; then \ echo 3; \ fi) +%define python_version %( \ + if [ %{rhel} -lt 9 ]; then \ + echo 39; \ + else \ + echo 3; \ + fi) %define lz4_requirement %( \ if [ %{_amzn} -eq 0 ]; then \ @@ -38,9 +57,10 @@ %define use_boost (!%{is_amazon_linux}) %define use_flight (%{rhel} >= 8) -%define use_gandiva (%{rhel} >= 8 && %{_arch} != "aarch64") +%define use_gandiva (%{rhel} >= 8 && "%{_arch}" != "aarch64") %define use_gcs (%{rhel} >= 8 || %{is_amazon_linux}) %define use_gflags (!%{is_amazon_linux}) +%define use_glog (%{rhel} <= 8) %define use_mimalloc (%{rhel} >= 8) %define use_python (%{rhel} >= 8) # TODO: Enable this. This works on local but is fragile on GitHub Actions and @@ -48,11 +68,9 @@ # %define use_s3 (%{rhel} >= 8) %define use_s3 0 -%define have_rapidjson (%{rhel} == 7) +%define have_rapidjson (%{rhel} != 8) %define have_re2 (%{rhel} >= 8) -# EPEL ships utf8proc but it's old. -# %define have_utf8proc (%{rhel} == 7) -%define have_utf8proc 0 +%define have_utf8proc (%{rhel} >= 9) %define have_zstd (!%{is_amazon_linux}) Name: @PACKAGE@ @@ -81,7 +99,9 @@ BuildRequires: gcc-c++ BuildRequires: gflags-devel %endif BuildRequires: git +%if %{use_glog} BuildRequires: glog-devel +%endif %if %{use_gcs} BuildRequires: json-devel %endif @@ -93,8 +113,8 @@ BuildRequires: ninja-build BuildRequires: openssl-devel BuildRequires: pkgconfig %if %{use_python} -BuildRequires: python39-devel -BuildRequires: python39-numpy +BuildRequires: python%{python_version}-devel +BuildRequires: python%{python_version}-numpy %endif %if %{have_rapidjson} BuildRequires: rapidjson-devel @@ -103,6 +123,7 @@ BuildRequires: rapidjson-devel BuildRequires: re2-devel %endif BuildRequires: snappy-devel +BuildRequires: thrift-devel %if %{have_utf8proc} BuildRequires: utf8proc-devel %endif @@ -115,7 +136,7 @@ BuildRequires: ncurses-devel BuildRequires: gobject-introspection-devel BuildRequires: gtk-doc -BuildRequires: vala-devel +BuildRequires: vala %description Apache Arrow is a data processing library for analysis. @@ -125,9 +146,8 @@ Apache Arrow is a data processing library for analysis. %build cpp_build_type=release -mkdir cpp/build -cd cpp/build -%cmake3 .. \ +cd cpp +%cmake \ -DARROW_CSV=ON \ -DARROW_DATASET=ON \ %if %{use_flight} @@ -168,8 +188,7 @@ cd cpp/build -DPythonInterp_FIND_VERSION_MAJOR=3 \ %endif -GNinja - -ninja %{?_smp_mflags} +%cmake_build cd - cd c_glib @@ -178,11 +197,11 @@ meson setup build \ --default-library=both \ --libdir=%{_libdir} \ --prefix=%{_prefix} \ - -Darrow_cpp_build_dir=../cpp/build \ + -Darrow_cpp_build_dir=../cpp/%{__cmake_builddir} \ -Darrow_cpp_build_type=$cpp_build_type \ -Dgtk_doc=true \ -Dvala=true -LD_LIBRARY_PATH=$PWD/../cpp/build/$cpp_build_type \ +LD_LIBRARY_PATH=$PWD/../cpp/%{__cmake_builddir}/$cpp_build_type \ ninja -C build %{?_smp_mflags} cd - @@ -190,13 +209,11 @@ cd - cpp_build_type=release cd c_glib -DESTDIR=$RPM_BUILD_ROOT ninja -C build install -ninja -C build clean +DESTDIR=%{buildroot} ninja -C build install cd - -cd cpp/build -DESTDIR=$RPM_BUILD_ROOT ninja install -ninja clean +cd cpp +%cmake_install cd - %package -n %{name}%{major_version}-libs @@ -206,7 +223,9 @@ Requires: brotli %if %{use_gflags} Requires: gflags %endif +%if %{use_glog} Requires: glog +%endif %if %{have_zstd} Requires: libzstd %endif @@ -414,7 +433,7 @@ Libraries and header files for Gandiva. Summary: Python integration library for Apache Arrow License: Apache-2.0 Requires: %{name}%{major_version}-libs = %{version}-%{release} -Requires: python39-numpy +Requires: python%{python_version}-numpy %description -n %{name}%{major_version}-python-libs This package contains the Python integration library for Apache Arrow. @@ -429,7 +448,7 @@ Summary: Libraries and header files for Python integration library for Apache Ar License: Apache-2.0 Requires: %{name}%{major_version}-python-libs = %{version}-%{release} Requires: %{name}-devel = %{version}-%{release} -Requires: python39-devel +Requires: python%{python_version}-devel %description python-devel Libraries and header files for Python integration library for Apache Arrow. diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile index 2cbef786213..9c93e2f2407 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile @@ -53,7 +53,8 @@ RUN \ rpmdevtools \ snappy-devel \ tar \ - vala-devel \ + thrift-devel \ + vala \ zlib-devel && \ yum clean ${quiet} all diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile index 14608bc5e6a..b29cc4565bd 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile @@ -60,7 +60,8 @@ RUN \ rpmdevtools \ snappy-devel \ tar \ + thrift-devel \ utf8proc-devel \ - vala-devel \ + vala \ zlib-devel && \ dnf clean ${quiet} all diff --git a/dev/tasks/linux-packages/package-task.rb b/dev/tasks/linux-packages/package-task.rb index 985ba8badd0..a1650b0cbd0 100644 --- a/dev/tasks/linux-packages/package-task.rb +++ b/dev/tasks/linux-packages/package-task.rb @@ -430,6 +430,8 @@ def yum_targets_default [ "almalinux-8", # "almalinux-8-arch64", + "almalinux-9", + # "almalinux-9-arch64", "amazon-linux-2", # "amazon-linux-2-arch64", "centos-8-stream", diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 11675e8bba3..c7d3cc1c597 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -652,13 +652,14 @@ tasks: {% endfor %} {% endfor %} -{% for target in ["almalinux-8", +{% for target in ["almalinux-9", + "almalinux-8", "amazon-linux-2", "centos-8-stream", "centos-7"] %} - {% set is_rhel8_based = (target in ["almalinux-8", "centos-8-stream"]) %} + {% set is_rhel7_based = (target in ["amazon-linux-2", "centos-7"]) %} {% for architecture in ["amd64", "arm64"] %} - {% if not (target in ["amazon-linux-2", "centos-7"] and architecture == "arm64") %} + {% if not (is_rhel7_based and architecture == "arm64") %} {{ target }}-{{ architecture }}: {% if architecture == "amd64" %} ci: github @@ -684,20 +685,20 @@ tasks: - arrow-dataset-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow-dataset-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow-dataset-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if is_rhel8_based %} + {% if not is_rhel7_based %} - arrow[0-9]+-dataset-glib-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} - arrow[0-9]+-dataset-glib-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if is_rhel8_based %} + {% if not is_rhel7_based %} - arrow[0-9]+-dataset-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} - arrow[0-9]+-dataset-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if is_rhel8_based %} + {% if not is_rhel7_based %} - arrow-debugsource-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} - arrow-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if is_rhel8_based %} + {% if not is_rhel7_based %} - arrow-flight-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow-flight-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow-flight-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm @@ -708,17 +709,17 @@ tasks: {% endif %} - arrow-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if is_rhel8_based %} + {% if not is_rhel7_based %} - arrow[0-9]+-glib-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} - arrow[0-9]+-glib-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if is_rhel8_based %} + {% if not is_rhel7_based %} - arrow[0-9]+-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} - arrow[0-9]+-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% if target != "amazon-linux-2" %} - arrow-python-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if is_rhel8_based %} + {% if not is_rhel7_based %} - arrow-python-flight-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow[0-9]+-python-flight-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - arrow[0-9]+-python-flight-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm @@ -729,7 +730,7 @@ tasks: {% if architecture == "amd64" %} - arrow-{no_rc_version}-1.[a-z0-9]+.src.rpm {% endif %} - {% if is_rhel8_based and architecture == "amd64" %} + {% if not is_rhel7_based and architecture == "amd64" %} - gandiva-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - gandiva-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - gandiva-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm @@ -741,26 +742,26 @@ tasks: - parquet-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - parquet-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - parquet-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if is_rhel8_based %} + {% if not is_rhel7_based %} - parquet[0-9]+-glib-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} - parquet[0-9]+-glib-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if is_rhel8_based %} + {% if not is_rhel7_based %} - parquet[0-9]+-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} - parquet[0-9]+-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - plasma-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - plasma-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - plasma-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if is_rhel8_based %} + {% if not is_rhel7_based %} - plasma[0-9]+-glib-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} - plasma[0-9]+-glib-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if is_rhel8_based %} + {% if not is_rhel7_based %} - plasma[0-9]+-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} - plasma[0-9]+-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm - {% if is_rhel8_based %} + {% if not is_rhel7_based %} - plasma-store-server-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm {% endif %} - plasma-store-server-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm From 09c6c7a51b1849630554c1fe68cb4a7e68a1d9bd Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 3 Jun 2022 17:53:34 +0900 Subject: [PATCH 02/14] Define shim --- .../apache-arrow/yum/arrow.spec.in | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index fa0520a35d3..bdf12061cb4 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -17,17 +17,36 @@ # specific language governing permissions and limitations # under the License. +# RHEL <= 7 and Amazon Linux 2 %if !%{defined cmake} -%define cmake %{cmake3} +%define cmake %cmake3 %endif -%if !%{defined cmake_build} -%define cmake_build %{cmake3_build} + +# RHEL <= 7 +%if !%{defined cmake_build} && %{defined cmake3_build} +%define cmake_build cmake3_build +%endif +# Amazon Linux 2 +%if !%{defined cmake_build} && !%{defined cmake3_build} +%define cmake_build %__cmake3 --build . +%endif + +# RHEL <= 7 +%if !%{defined cmake_install} && %{defined cmake3_install} +%define cmake_install %cmake3_install +%endif +# Amazon Linux 2 +%if !%{defined cmake_install} && !%{defined cmake3_install} +%define cmake_install DESTDIR="%{buildroot}" %__cmake3 --install . %endif -%if !%{defined cmake_install} -%define cmake_install %{cmake3_install} + +# RHEL <= 7 +%if !%{defined __cmake_builddir} && %{defined __cmake3_builddir} +%define __cmake_builddir %__cmake3_builddir %endif -%if !%{defined __cmake_builddir} -%define __cmake_builddir %{__cmake3_builddir} +# Amazon Linux 2 +%if !%{defined __cmake_builddir} && !%{defined __cmake3_builddir} +%define __cmake_builddir . %endif %define _amzn %{?amzn:%{amzn}}%{!?amzn:0} @@ -65,7 +84,7 @@ %define use_python (%{rhel} >= 8) # TODO: Enable this. This works on local but is fragile on GitHub Actions and # Travis CI. -# %define use_s3 (%{rhel} >= 8) +# %%define use_s3 (%{rhel} >= 8) %define use_s3 0 %define have_rapidjson (%{rhel} != 8) From 82ee34054219451917486696a6becbcad48f0385 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 3 Jun 2022 17:56:51 +0900 Subject: [PATCH 03/14] Add support for verifying AlmaLinux 9 --- dev/release/verify-yum.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh index caed5e37b0f..74a007b1d39 100755 --- a/dev/release/verify-yum.sh +++ b/dev/release/verify-yum.sh @@ -51,11 +51,15 @@ have_gandiva=yes have_glib=yes have_parquet=yes have_python=yes -install_command="dnf install -y --enablerepo=powertools" +install_command="dnf install -y --enablerepo=crb" echo "::group::Prepare repository" case "${distribution}-${distribution_version}" in + almalinux-8) + distribution_prefix="almalinux" + install_command="dnf install -y --enablerepo=powertools" + ;; almalinux-*) distribution_prefix="almalinux" ;; @@ -78,6 +82,11 @@ case "${distribution}-${distribution_version}" in have_python=no install_command="yum install -y" ;; + centos-8-stream) + distribution_prefix="centos" + repository_version+="-stream" + install_command="dnf install -y --enablerepo=powertools" + ;; centos-*) distribution_prefix="centos" repository_version+="-stream" From 4db5ba3aac742e36ab5d7c95b85f56966900bf3f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 3 Jun 2022 18:00:42 +0900 Subject: [PATCH 04/14] Fix build dir --- dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index bdf12061cb4..2bb53fff09d 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -46,7 +46,7 @@ %endif # Amazon Linux 2 %if !%{defined __cmake_builddir} && !%{defined __cmake3_builddir} -%define __cmake_builddir . +%define __cmake_builddir build %endif %define _amzn %{?amzn:%{amzn}}%{!?amzn:0} From c85c76b09b8d1e62fd00c73afedcbfef2b034b98 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 4 Jun 2022 10:49:26 +0900 Subject: [PATCH 05/14] Use Ubuntu 22.04 --- .../linux-packages/github.linux.amd64.yml | 37 +------------------ 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/dev/tasks/linux-packages/github.linux.amd64.yml b/dev/tasks/linux-packages/github.linux.amd64.yml index b7619e3ca32..e11388c9e5d 100644 --- a/dev/tasks/linux-packages/github.linux.amd64.yml +++ b/dev/tasks/linux-packages/github.linux.amd64.yml @@ -22,7 +22,7 @@ jobs: package: name: Package - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: {{ macros.github_checkout_arrow()|indent }} {{ macros.github_login_dockerhub()|indent }} @@ -66,10 +66,9 @@ jobs: run: | set -e sudo apt update - # We can install createrepo_c by package with Ubuntu 22.04. - # createrepo_c \ sudo apt install -y \ apt-utils \ + createrepo_c \ devscripts \ gpg \ rpm @@ -92,38 +91,6 @@ jobs: ;; esac gpg --export --armor test@example.com >> arrow/dev/tasks/linux-packages/KEYS - # We can install createrepo_c by package with Ubuntu 22.04. - # This is workaround: - - name: Install createrepo_c - run: | - sudo apt install -y \ - cmake \ - libbz2-dev \ - libcurl4-openssl-dev \ - libglib2.0-dev \ - liblzma-dev \ - libmagic-dev \ - librpm-dev \ - libsqlite3-dev \ - libssl-dev \ - libxml2-dev \ - libzstd-dev \ - pkg-config \ - zlib1g-dev - git clone --depth 1 https://github.com/rpm-software-management/createrepo_c.git - pushd createrepo_c - /usr/bin/cmake \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DENABLE_BASHCOMP=OFF \ - -DENABLE_DRPM=OFF \ - -DENABLE_PYTHON=OFF \ - -DWITH_LIBMODULEMD=OFF \ - -DWITH_ZCHUNK=OFF \ - . - make -j$(nproc) - sudo make install - popd - rm -rf createrepo_c - name: Test run: | set -e From 87477da762bf3d55975cd4c85b8ca80ad5303f94 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 4 Jun 2022 12:05:58 +0900 Subject: [PATCH 06/14] Fix shim --- .../apache-arrow/yum/arrow.spec.in | 57 +++++++------------ 1 file changed, 20 insertions(+), 37 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 2bb53fff09d..7afc9f564c2 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -17,38 +17,6 @@ # specific language governing permissions and limitations # under the License. -# RHEL <= 7 and Amazon Linux 2 -%if !%{defined cmake} -%define cmake %cmake3 -%endif - -# RHEL <= 7 -%if !%{defined cmake_build} && %{defined cmake3_build} -%define cmake_build cmake3_build -%endif -# Amazon Linux 2 -%if !%{defined cmake_build} && !%{defined cmake3_build} -%define cmake_build %__cmake3 --build . -%endif - -# RHEL <= 7 -%if !%{defined cmake_install} && %{defined cmake3_install} -%define cmake_install %cmake3_install -%endif -# Amazon Linux 2 -%if !%{defined cmake_install} && !%{defined cmake3_install} -%define cmake_install DESTDIR="%{buildroot}" %__cmake3 --install . -%endif - -# RHEL <= 7 -%if !%{defined __cmake_builddir} && %{defined __cmake3_builddir} -%define __cmake_builddir %__cmake3_builddir -%endif -# Amazon Linux 2 -%if !%{defined __cmake_builddir} && !%{defined __cmake3_builddir} -%define __cmake_builddir build -%endif - %define _amzn %{?amzn:%{amzn}}%{!?amzn:0} %define is_amazon_linux (%{_amzn} != 0) @@ -74,6 +42,21 @@ echo ">= 1.8.0"; \ fi) +# RHEL <= 8 and Amazon Linux 2 don't provide cmake* macros. They provide +# cmake3* macros. +%define arrow_cmake_builddir \ + %{?__cmake_builddir:%__cmake_builddir} \ + %{!?__cmake_builddir:build} +%define arrow_cmake \ + %{?cmake:%cmake} \ + {%!?cmake:%cmake3 -S . -B %{arrow_cmake_builddir}} +%define arrow_cmake_build \ + %{?cmake_build:%cmake_build} \ + {%!?cmake_build:ninja -C %{arrow_cmake_builddir} --verbose} +%define arrow_cmake_install \ + %{?cmake_install:%cmake_install} \ + %{!?cmake_install:DESTDIR="%{buildroot}" ninja -C %{arrow_cmake_builddir} install} + %define use_boost (!%{is_amazon_linux}) %define use_flight (%{rhel} >= 8) %define use_gandiva (%{rhel} >= 8 && "%{_arch}" != "aarch64") @@ -166,7 +149,7 @@ Apache Arrow is a data processing library for analysis. %build cpp_build_type=release cd cpp -%cmake \ +%arrow_cmake \ -DARROW_CSV=ON \ -DARROW_DATASET=ON \ %if %{use_flight} @@ -207,7 +190,7 @@ cd cpp -DPythonInterp_FIND_VERSION_MAJOR=3 \ %endif -GNinja -%cmake_build +%arrow_cmake_build cd - cd c_glib @@ -216,11 +199,11 @@ meson setup build \ --default-library=both \ --libdir=%{_libdir} \ --prefix=%{_prefix} \ - -Darrow_cpp_build_dir=../cpp/%{__cmake_builddir} \ + -Darrow_cpp_build_dir=../cpp/%{arrow_cmake_builddir} \ -Darrow_cpp_build_type=$cpp_build_type \ -Dgtk_doc=true \ -Dvala=true -LD_LIBRARY_PATH=$PWD/../cpp/%{__cmake_builddir}/$cpp_build_type \ +LD_LIBRARY_PATH=$PWD/../cpp/%{arrow_cmake_builddir}/$cpp_build_type \ ninja -C build %{?_smp_mflags} cd - @@ -232,7 +215,7 @@ DESTDIR=%{buildroot} ninja -C build install cd - cd cpp -%cmake_install +%arrow_cmake_install cd - %package -n %{name}%{major_version}-libs From 18f8f95106c573c828a98ed4d354679cd5e1bac4 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 4 Jun 2022 12:12:32 +0900 Subject: [PATCH 07/14] Fix shim --- dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 7afc9f564c2..4e61fdd5c4c 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -49,13 +49,13 @@ %{!?__cmake_builddir:build} %define arrow_cmake \ %{?cmake:%cmake} \ - {%!?cmake:%cmake3 -S . -B %{arrow_cmake_builddir}} + %{!?cmake:%cmake3 -S . -B build} %define arrow_cmake_build \ %{?cmake_build:%cmake_build} \ - {%!?cmake_build:ninja -C %{arrow_cmake_builddir} --verbose} + %{!?cmake_build:ninja -C build --verbose} %define arrow_cmake_install \ %{?cmake_install:%cmake_install} \ - %{!?cmake_install:DESTDIR="%{buildroot}" ninja -C %{arrow_cmake_builddir} install} + %{!?cmake_install:DESTDIR="%{buildroot}" ninja -C build install} %define use_boost (!%{is_amazon_linux}) %define use_flight (%{rhel} >= 8) From c16bbb6d74a3176918ec7c98ff90832b563744c5 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 4 Jun 2022 12:18:10 +0900 Subject: [PATCH 08/14] Fix shim --- .../apache-arrow/yum/arrow.spec.in | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 4e61fdd5c4c..c2626bab6f7 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -44,18 +44,10 @@ # RHEL <= 8 and Amazon Linux 2 don't provide cmake* macros. They provide # cmake3* macros. -%define arrow_cmake_builddir \ - %{?__cmake_builddir:%__cmake_builddir} \ - %{!?__cmake_builddir:build} -%define arrow_cmake \ - %{?cmake:%cmake} \ - %{!?cmake:%cmake3 -S . -B build} -%define arrow_cmake_build \ - %{?cmake_build:%cmake_build} \ - %{!?cmake_build:ninja -C build --verbose} -%define arrow_cmake_install \ - %{?cmake_install:%cmake_install} \ - %{!?cmake_install:DESTDIR="%{buildroot}" ninja -C build install} +%define arrow_cmake_builddir %{?__cmake_builddir:%__cmake_builddir}%{!?__cmake_builddir:build} +%define arrow_cmake %{?cmake:%cmake}%{!?cmake:%cmake3 -S . -B %{arrow_cmake_builddir}} +%define arrow_cmake_build %{?cmake_build:%cmake_build}%{!?cmake_build:ninja -C %{arrow_cmake_builddir} --verbose} +%define arrow_cmake_install %{?cmake_install:%cmake_install}%{!?cmake_install:DESTDIR="%{buildroot}" ninja -C %{arrow_cmake_builddir} install} %define use_boost (!%{is_amazon_linux}) %define use_flight (%{rhel} >= 8) From f0099d205dd0723c6a5c2e6ca75e6970197fde3c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 4 Jun 2022 12:27:23 +0900 Subject: [PATCH 09/14] Use Ruby 3.1 --- dev/tasks/linux-packages/github.linux.amd64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/linux-packages/github.linux.amd64.yml b/dev/tasks/linux-packages/github.linux.amd64.yml index e11388c9e5d..3151bacd928 100644 --- a/dev/tasks/linux-packages/github.linux.amd64.yml +++ b/dev/tasks/linux-packages/github.linux.amd64.yml @@ -30,7 +30,7 @@ jobs: - name: Set up Ruby uses: ruby/setup-ruby@v1 with: - ruby-version: '3.0' + ruby-version: '3.1' - name: Cache ccache uses: actions/cache@v2 with: From 7b35a54fb489d837ae0dcb82c7f5d8d92958ad5b Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 5 Jun 2022 21:19:57 +0900 Subject: [PATCH 10/14] Fix package name --- dev/tasks/linux-packages/github.linux.amd64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/linux-packages/github.linux.amd64.yml b/dev/tasks/linux-packages/github.linux.amd64.yml index 3151bacd928..f252a081d67 100644 --- a/dev/tasks/linux-packages/github.linux.amd64.yml +++ b/dev/tasks/linux-packages/github.linux.amd64.yml @@ -68,7 +68,7 @@ jobs: sudo apt update sudo apt install -y \ apt-utils \ - createrepo_c \ + createrepo-c \ devscripts \ gpg \ rpm From f13deda865d4735a632e5896f57e8633dfff7d68 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 5 Jun 2022 22:02:10 +0900 Subject: [PATCH 11/14] Fix shim --- .../apache-arrow/yum/arrow.spec.in | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index c2626bab6f7..8000a2b253f 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -42,12 +42,17 @@ echo ">= 1.8.0"; \ fi) -# RHEL <= 8 and Amazon Linux 2 don't provide cmake* macros. They provide -# cmake3* macros. -%define arrow_cmake_builddir %{?__cmake_builddir:%__cmake_builddir}%{!?__cmake_builddir:build} -%define arrow_cmake %{?cmake:%cmake}%{!?cmake:%cmake3 -S . -B %{arrow_cmake_builddir}} -%define arrow_cmake_build %{?cmake_build:%cmake_build}%{!?cmake_build:ninja -C %{arrow_cmake_builddir} --verbose} -%define arrow_cmake_install %{?cmake_install:%cmake_install}%{!?cmake_install:DESTDIR="%{buildroot}" ninja -C %{arrow_cmake_builddir} install} +%if %{rhel} >= 9 +%define arrow_cmake_builddir %{__cmake_builddir} +%define arrow_cmake %cmake +%define arrow_cmake_build %cmake_build +%define arrow_cmake_install %cmake_install +%else +%define arrow_cmake_builddir build +%define arrow_cmake %cmake3 -S . -B %{arrow_cmake_builddir} +%define arrow_cmake_build ninja -C %{arrow_cmake_builddir} -v +%define arrow_cmake_install DESTDIR="%{buildroot}" ninja -C %{arrow_cmake_builddir} install +%endif %define use_boost (!%{is_amazon_linux}) %define use_flight (%{rhel} >= 8) From f2903d868a01c403de49acfa3f18005beec3e6ef Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 5 Jun 2022 23:25:33 +0900 Subject: [PATCH 12/14] Fix ID --- dev/release/verify-yum.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh index 74a007b1d39..27427fbf49f 100755 --- a/dev/release/verify-yum.sh +++ b/dev/release/verify-yum.sh @@ -82,7 +82,7 @@ case "${distribution}-${distribution_version}" in have_python=no install_command="yum install -y" ;; - centos-8-stream) + centos-8) distribution_prefix="centos" repository_version+="-stream" install_command="dnf install -y --enablerepo=powertools" From e94f7ecde6285576b98c47991f54a50d8237bc0d Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 6 Jun 2022 01:20:06 +0900 Subject: [PATCH 13/14] Add workaround for AlmaLinux 9 --- .../apache-arrow-release/Rakefile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/dev/tasks/linux-packages/apache-arrow-release/Rakefile b/dev/tasks/linux-packages/apache-arrow-release/Rakefile index b85cd90f30e..2c9ddfc87ba 100644 --- a/dev/tasks/linux-packages/apache-arrow-release/Rakefile +++ b/dev/tasks/linux-packages/apache-arrow-release/Rakefile @@ -46,18 +46,35 @@ class ApacheArrowReleasePackageTask < PackageTask keys = File.read(keys_path) File.open(keys_path, "w") do |keys_file| is_ed25519_key = false + deny_lists = [ + # "rpmkeys --import" reports error for this key. + # It seems that a subkey of this key may be related. (Is SHA1 bad?) + "8CAAD602", + ] + is_denyed_key = false keys.each_line do |line| case line.chomp when /\Apub\s+ed25519\s/ is_ed25519_key = true next + when /\Apub\s+[^\/]+\/([\h]+)\s/ + short_finger_print = $1 + if deny_lists.include?(short_finger_print) + is_denyed_key = true + next + end when "-----END PGP PUBLIC KEY BLOCK-----" if is_ed25519_key is_ed25519_key = false next end + if is_denyed_key + is_denyed_key = false + next + end else next if is_ed25519_key + next if is_denyed_key end keys_file.print(line) end From 0b076e5cdc5eb14961af11977c0b607c188b674c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 6 Jun 2022 11:17:19 +0900 Subject: [PATCH 14/14] Fix order --- dev/tasks/linux-packages/package-task.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/linux-packages/package-task.rb b/dev/tasks/linux-packages/package-task.rb index a1650b0cbd0..752ddc3e26e 100644 --- a/dev/tasks/linux-packages/package-task.rb +++ b/dev/tasks/linux-packages/package-task.rb @@ -428,10 +428,10 @@ def yum_targets_default # Disable aarch64 targets by default for now # because they require some setups on host. [ - "almalinux-8", - # "almalinux-8-arch64", "almalinux-9", # "almalinux-9-arch64", + "almalinux-8", + # "almalinux-8-arch64", "amazon-linux-2", # "amazon-linux-2-arch64", "centos-8-stream",