GH-34914: [Packaging][Linux] Add support for Acero

cpp/src/arrow/dataset/CMakeLists.txt

@@ -31,10 +31,9 @@ set(ARROW_DATASET_SRCS
     scanner.cc
     scan_node.cc)
 
+set(ARROW_DATASET_PKG_CONFIG_REQUIRES arrow_acero)
 if(ARROW_PARQUET)
-  set(ARROW_DATASET_PKG_CONFIG_REQUIRES parquet)
-else()
-  set(ARROW_DATASET_PKG_CONFIG_REQUIRES arrow arrow_acero)
+  list(APPEND ARROW_DATASET_PKG_CONFIG_REQUIRES parquet)
 endif()
 
 set(ARROW_DATASET_STATIC_LINK_LIBS)

dev/release/post-11-bump-versions-test.rb

@@ -308,13 +308,6 @@ def test_deb_package_names
       }
     end
     expected_changes = [
-      {
-        sampled_diff: [
-          "-dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib#{@so_version}.install",
-          "+dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib#{@next_so_version}.install",
-        ],
-        path: "dev/release/rat_exclude_files.txt",
-      },
       {
         sampled_diff: [
           "-Package: libarrow#{@so_version}",
@@ -324,8 +317,8 @@ def test_deb_package_names
       },
       {
         sampled_diff: [
-          "-      - libarrow-dataset-glib#{@so_version}-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb",
-          "+      - libarrow-dataset-glib#{@next_so_version}-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb",
+          "-      - libarrow-acero#{@so_version}-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb",
+          "+      - libarrow-acero#{@next_so_version}-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb",
         ],
         path: "dev/tasks/tasks.yml",
       },

dev/release/rat_exclude_files.txt

@@ -47,61 +47,11 @@ dev/tasks/linux-packages/apache-arrow-apt-source/debian/compat
 dev/tasks/linux-packages/apache-arrow-apt-source/debian/control
 dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules
 dev/tasks/linux-packages/apache-arrow-apt-source/debian/source/format
+dev/tasks/linux-packages/apache-arrow/debian/*.doc-base
+dev/tasks/linux-packages/apache-arrow/debian/*.install
+dev/tasks/linux-packages/apache-arrow/debian/*.links
 dev/tasks/linux-packages/apache-arrow/debian/compat
 dev/tasks/linux-packages/apache-arrow/debian/control.in
-dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-1.0.install
-dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-cuda-1.0.install
-dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-dataset-1.0.install
-dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-flight-1.0.install
-dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-flight-sql-1.0.install
-dev/tasks/linux-packages/apache-arrow/debian/gir1.2-gandiva-1.0.install
-dev/tasks/linux-packages/apache-arrow/debian/gir1.2-parquet-1.0.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib1200.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda1200.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.doc-base
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib1200.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset1200.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-doc.doc-base
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-doc.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib1200.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-sql-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-sql-glib-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-sql-glib-doc.doc-base
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-sql-glib-doc.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-sql-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-sql-glib1200.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-sql1200.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight1200.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.doc-base
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib1200.install
-dev/tasks/linux-packages/apache-arrow/debian/libarrow1200.install
-dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.doc-base
-dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.install
-dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib1200.install
-dev/tasks/linux-packages/apache-arrow/debian/libgandiva1200.install
-dev/tasks/linux-packages/apache-arrow/debian/libparquet-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-dev.install
-dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.doc-base
-dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.install
-dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.links
-dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib1200.install
-dev/tasks/linux-packages/apache-arrow/debian/libparquet1200.install
 dev/tasks/linux-packages/apache-arrow/debian/patches/series
 dev/tasks/linux-packages/apache-arrow/debian/rules
 dev/tasks/linux-packages/apache-arrow/debian/shlibs.local

dev/release/verify-yum.sh

@@ -57,6 +57,7 @@ have_glib=yes
 have_parquet=yes
 have_python=yes
 have_ruby=yes
+have_vala=yes
 ruby_devel_packages=(ruby-devel)
 install_command="dnf install -y --enablerepo=crb"
 uninstall_command="dnf remove -y"
@@ -105,6 +106,7 @@ case "${distribution}-${distribution_version}" in
     have_gandiva=no
     have_python=no
     have_ruby=no
+    have_vala=no
     install_command="yum install -y"
     uninstall_command="yum remove -y"
     clean_command="yum clean"
@@ -234,12 +236,14 @@ if [ "${have_glib}" = "yes" ]; then
   ${install_command} --enablerepo=epel arrow-glib-devel-${package_version}
   ${install_command} --enablerepo=epel arrow-glib-doc-${package_version}
 
-  ${install_command} vala
-  cp -a "${TOP_SOURCE_DIR}/c_glib/example/vala" build/
-  pushd build/vala
-  valac --pkg arrow-glib --pkg posix build.vala
-  ./build
-  popd
+  if [ "${have_vala}" = "yes" ]; then
+    ${install_command} vala
+    cp -a "${TOP_SOURCE_DIR}/c_glib/example/vala" build/
+    pushd build/vala
+    valac --pkg arrow-glib --pkg posix build.vala
+    ./build
+    popd
+  fi
 
   if [ "${have_ruby}" = "yes" ]; then
     ${install_command} "${ruby_devel_packages[@]}"

dev/tasks/linux-packages/apache-arrow/debian/arrow-tools.install

@@ -0,0 +1 @@
+usr/bin/arrow-*

dev/tasks/linux-packages/apache-arrow/debian/control.in

@@ -79,6 +79,19 @@ Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ library files for CUDA support.
 
+Package: libarrow-acero1200
+Section: libs
+Architecture: any
+Multi-Arch: same
+Pre-Depends: ${misc:Pre-Depends}
+Depends:
+  ${misc:Depends},
+  ${shlibs:Depends},
+  libarrow1200 (= ${binary:Version})
+Description: Apache Arrow is a data processing library for analysis
+ .
+ This package provides C++ library files for Acero module.
+
 Package: libarrow-dataset1200
 Section: libs
 Architecture: any
@@ -87,7 +100,7 @@ Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
   ${shlibs:Depends},
-  libarrow1200 (= ${binary:Version}),
+  libarrow-acero1200 (= ${binary:Version}),
   libparquet1200 (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis
  .
@@ -159,13 +172,25 @@ Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ header files for CUDA support.
 
+Package: libarrow-acero-dev
+Section: libdevel
+Architecture: any
+Multi-Arch: same
+Depends:
+  ${misc:Depends},
+  libarrow-acero1200 (= ${binary:Version}),
+  libparquet-dev (= ${binary:Version})
+Description: Apache Arrow is a data processing library for analysis
+ .
+ This package provides C++ header files for Acero module.
+
 Package: libarrow-dataset-dev
 Section: libdevel
 Architecture: any
 Multi-Arch: same
 Depends:
   ${misc:Depends},
-  libarrow-dev (= ${binary:Version}),
+  libarrow-acero-dev (= ${binary:Version}),
   libarrow-dataset1200 (= ${binary:Version}),
   libparquet-dev (= ${binary:Version})
 Description: Apache Arrow is a data processing library for analysis

dev/tasks/linux-packages/apache-arrow/debian/libarrow-acero-dev.install

@@ -0,0 +1,4 @@
+usr/lib/*/cmake/ArrowAcero/
+usr/lib/*/libarrow_acero.a
+usr/lib/*/libarrow_acero.so
+usr/lib/*/pkgconfig/arrow-acero.pc

dev/tasks/linux-packages/apache-arrow/debian/libarrow-acero1200.install

@@ -0,0 +1 @@
+usr/lib/*/libarrow_acero.so.*

dev/tasks/linux-packages/apache-arrow/debian/parquet-tools.install

@@ -0,0 +1 @@
+usr/bin/parquet-*

dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in

@@ -77,6 +77,7 @@
 # Travis CI.
 # %%define use_s3 (%%{rhel} >= 8)
 %define use_s3 0
+%define use_vala (%{rhel} >= 8 || %{is_amazon_linux})
 
 %define have_lz4_libs (%{rhel} >= 8)
 %define have_rapidjson (%{rhel} != 8)
@@ -146,7 +147,9 @@ BuildRequires:	ncurses-devel
 
 BuildRequires:	gobject-introspection-devel
 BuildRequires:	gtk-doc
+%if %{use_vala}
 BuildRequires:	vala
+%endif
 
 %description
 Apache Arrow is a data processing library for analysis.
@@ -212,7 +215,9 @@ meson setup build \
   -Darrow_cpp_build_dir=../cpp/%{arrow_cmake_builddir} \
   -Darrow_cpp_build_type=$cpp_build_type \
   -Dgtk_doc=true \
+%if %{use_vala}
   -Dvapi=true
+%endif
 
 LD_LIBRARY_PATH=$PWD/../cpp/%{arrow_cmake_builddir}/$cpp_build_type \
   meson compile -C build %{?_smp_mflags}
@@ -320,6 +325,7 @@ Libraries and header files for Apache Arrow C++.
 %{_datadir}/arrow/gdb/
 %{_datadir}/gdb/auto-load/
 %{_includedir}/arrow/
+%exclude %{_includedir}/arrow/acero/
 %exclude %{_includedir}/arrow/dataset/
 %if %{use_flight}
 %exclude %{_includedir}/arrow/flight/
@@ -336,10 +342,44 @@ Libraries and header files for Apache Arrow C++.
 %{_libdir}/pkgconfig/arrow-orc.pc
 %{_libdir}/pkgconfig/arrow.pc
 
+%package -n %{name}%{major_version}-acero-libs
+Summary:	C++ library to execute a query in streaming
+License:	Apache-2.0
+Requires:	%{name}%{major_version}-libs = %{version}-%{release}
+
+%description -n %{name}%{major_version}-acero-libs
+This package contains the libraries for Apache Arrow Acero.
+
+%files -n %{name}%{major_version}-acero-libs
+%defattr(-,root,root,-)
+%doc README.md
+%license LICENSE.txt NOTICE.txt
+%{_libdir}/libarrow_acero.so.*
+
+%package acero-devel
+Summary:	Libraries and header files for Apache Arrow Acero
+License:	Apache-2.0
+Requires:	%{name}%{major_version}-acero-libs = %{version}-%{release}
+Requires:	%{name}-devel = %{version}-%{release}
+
+%description acero-devel
+Libraries and header files for Apache Arrow Acero
+
+%files acero-devel
+%defattr(-,root,root,-)
+%doc README.md
+%license LICENSE.txt NOTICE.txt
+%{_includedir}/arrow/acero/
+%{_libdir}/cmake/ArrowAcero/
+%{_libdir}/libarrow_acero.a
+%{_libdir}/libarrow_acero.so
+%{_libdir}/pkgconfig/arrow-acero.pc
+
 %package -n %{name}%{major_version}-dataset-libs
 Summary:	C++ library to read and write semantic datasets stored in different locations and formats
 License:	Apache-2.0
-Requires:	%{name}%{major_version}-libs = %{version}-%{release}
+Requires:	%{name}%{major_version}-acero-libs = %{version}-%{release}
+Requires:	parquet%{major_version}-libs = %{version}-%{release}
 
 %description -n %{name}%{major_version}-dataset-libs
 This package contains the libraries for Apache Arrow dataset.
@@ -354,7 +394,8 @@ This package contains the libraries for Apache Arrow dataset.
 Summary:	Libraries and header files for Apache Arrow dataset.
 License:	Apache-2.0
 Requires:	%{name}%{major_version}-dataset-libs = %{version}-%{release}
-Requires:	%{name}-devel = %{version}-%{release}
+Requires:	%{name}-acero-devel = %{version}-%{release}
+Requires:	parquet-devel = %{version}-%{release}
 
 %description dataset-devel
 Libraries and header files for Apache Arrow dataset.
@@ -559,7 +600,9 @@ Libraries and header files for Apache Arrow GLib.
 %license LICENSE.txt NOTICE.txt
 %{_datadir}/arrow-glib/example/
 %{_datadir}/gir-1.0/Arrow-*.gir
+%if %{use_vala}
 %{_datadir}/vala/vapi/arrow-glib.*
+%endif
 %{_includedir}/arrow-glib/
 %{_libdir}/libarrow-glib.a
 %{_libdir}/libarrow-glib.so
@@ -611,7 +654,9 @@ Libraries and header files for Apache Arrow Dataset GLib.
 %doc README.md
 %license LICENSE.txt NOTICE.txt
 %{_datadir}/gir-1.0/ArrowDataset-*.gir
+%if %{use_vala}
 %{_datadir}/vala/vapi/arrow-dataset-glib.*
+%endif
 %{_includedir}/arrow-dataset-glib/
 %{_libdir}/libarrow-dataset-glib.a
 %{_libdir}/libarrow-dataset-glib.so
@@ -662,7 +707,9 @@ Libraries and header files for Apache Arrow Flight GLib.
 %doc README.md
 %license LICENSE.txt NOTICE.txt
 %{_datadir}/gir-1.0/ArrowFlight-*.gir
+%if %{use_vala}
 %{_datadir}/vala/vapi/arrow-flight-glib.*
+%endif
 %{_includedir}/arrow-flight-glib/
 %{_libdir}/libarrow-flight-glib.a
 %{_libdir}/libarrow-flight-glib.so
@@ -712,7 +759,9 @@ Libraries and header files for Apache Arrow Flight SQL GLib.
 %doc README.md
 %license LICENSE.txt NOTICE.txt
 %{_datadir}/gir-1.0/ArrowFlightSQL-*.gir
+%if %{use_vala}
 %{_datadir}/vala/vapi/arrow-flight-sql-glib.*
+%endif
 %{_includedir}/arrow-flight-sql-glib/
 %{_libdir}/libarrow-flight-sql-glib.a
 %{_libdir}/libarrow-flight-sql-glib.so
@@ -764,7 +813,9 @@ Libraries and header files for Gandiva GLib.
 %doc README.md
 %license LICENSE.txt NOTICE.txt
 %{_datadir}/gir-1.0/Gandiva-*.gir
+%if %{use_vala}
 %{_datadir}/vala/vapi/gandiva-glib.*
+%endif
 %{_includedir}/gandiva-glib/
 %{_libdir}/libgandiva-glib.a
 %{_libdir}/libgandiva-glib.so
@@ -815,7 +866,9 @@ Libraries and header files for Apache Parquet GLib.
 %doc README.md
 %license LICENSE.txt NOTICE.txt
 %{_datadir}/gir-1.0/Parquet-*.gir
+%if %{use_vala}
 %{_datadir}/vala/vapi/parquet-glib.*
+%endif
 %{_includedir}/parquet-glib/
 %{_libdir}/libparquet-glib.a
 %{_libdir}/libparquet-glib.so

dev/tasks/tasks.yml

@@ -576,6 +576,9 @@ tasks:
       - gir1.2-arrow-flight-sql-1.0_{no_rc_version}-1_[a-z0-9]+.deb
       - gir1.2-gandiva-1.0_{no_rc_version}-1_[a-z0-9]+.deb
       - gir1.2-parquet-1.0_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-acero-dev_{no_rc_version}-1_[a-z0-9]+.deb
+      - libarrow-acero1200-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
+      - libarrow-acero1200_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
@@ -659,6 +662,12 @@ tasks:
       - apache-arrow-release-{no_rc_version}-1.[a-z0-9]+.noarch.rpm
       - apache-arrow-release-{no_rc_version}-1.[a-z0-9]+.src.rpm
     {% endif %}
+      - arrow[0-9]+-acero-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
+      - arrow-acero-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
+    {% if not is_rhel7_based %}
+      - arrow[0-9]+-acero-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
+    {% endif %}
+      - arrow[0-9]+-dataset-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
       - arrow-dataset-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
       - arrow-dataset-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
       - arrow-dataset-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm