From bf8d7321230b995495096e175a18e45b0a332832 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 12 Jun 2022 06:59:30 +0900 Subject: [PATCH 1/8] ARROW-16815: [Packaging][RPM] Don't use too many parallel jobs This is a follow-up of ARROW-16745/#13307. If we use 3 or more jobs to build RPM, memory on Travis CI is exhausted. It causes the "No output has been received in the last 10m0s, this potentially indicates a stalled build or something wrong with the build itself." error. --- dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index c671ce8aede..453b44ef74a 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -45,12 +45,12 @@ %if %{rhel} >= 9 %define arrow_cmake_builddir %{__cmake_builddir} %define arrow_cmake %cmake -%define arrow_cmake_build %cmake_build +%define arrow_cmake_build ninja -C %{arrow_cmake_builddir} %{?_smp_mflags} -v %define arrow_cmake_install %cmake_install %else %define arrow_cmake_builddir build %define arrow_cmake %cmake3 -S . -B %{arrow_cmake_builddir} -%define arrow_cmake_build ninja -C %{arrow_cmake_builddir} -v +%define arrow_cmake_build ninja -C %{arrow_cmake_builddir} %{?_smp_mflags} -v %define arrow_cmake_install DESTDIR="%{buildroot}" ninja -C %{arrow_cmake_builddir} install %endif From 199611fe7fb4334cc040d60d1472ed660c2a83c2 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 12 Jun 2022 23:02:19 +0900 Subject: [PATCH 2/8] Don't verbose --- dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 453b44ef74a..7df3bc4aec4 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -45,12 +45,12 @@ %if %{rhel} >= 9 %define arrow_cmake_builddir %{__cmake_builddir} %define arrow_cmake %cmake -%define arrow_cmake_build ninja -C %{arrow_cmake_builddir} %{?_smp_mflags} -v +%define arrow_cmake_build ninja -C %{arrow_cmake_builddir} %{?_smp_mflags} %define arrow_cmake_install %cmake_install %else %define arrow_cmake_builddir build %define arrow_cmake %cmake3 -S . -B %{arrow_cmake_builddir} -%define arrow_cmake_build ninja -C %{arrow_cmake_builddir} %{?_smp_mflags} -v +%define arrow_cmake_build ninja -C %{arrow_cmake_builddir} %{?_smp_mflags} %define arrow_cmake_install DESTDIR="%{buildroot}" ninja -C %{arrow_cmake_builddir} install %endif From 911ccf8a530db6ce4f0f5e9b718734ddfe6d3545 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 13 Jun 2022 11:16:42 +0900 Subject: [PATCH 3/8] Increase wait time --- dev/tasks/linux-packages/travis.linux.arm64.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dev/tasks/linux-packages/travis.linux.arm64.yml b/dev/tasks/linux-packages/travis.linux.arm64.yml index ef292cd6b4e..aacac9e3ecb 100644 --- a/dev/tasks/linux-packages/travis.linux.arm64.yml +++ b/dev/tasks/linux-packages/travis.linux.arm64.yml @@ -114,12 +114,13 @@ script: done - popd - | - rake \ - --trace \ - {{ task_namespace }}:build \ - BUILD_DIR=build \ - DEB_BUILD_OPTIONS=parallel=2 \ - RPM_BUILD_NCPUS=2 + travis_wait 30 \ + rake \ + --trace \ + {{ task_namespace }}:build \ + BUILD_DIR=build \ + DEB_BUILD_OPTIONS=parallel=2 \ + RPM_BUILD_NCPUS=2 - sudo rm -rf */*/build - popd # Push Docker image From 745b88dc67e4cd3c5ac399c733b8bbe10a72657c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 13 Jun 2022 12:34:03 +0900 Subject: [PATCH 4/8] Fix timeout --- dev/tasks/linux-packages/travis.linux.arm64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/linux-packages/travis.linux.arm64.yml b/dev/tasks/linux-packages/travis.linux.arm64.yml index aacac9e3ecb..8ae1b611ec3 100644 --- a/dev/tasks/linux-packages/travis.linux.arm64.yml +++ b/dev/tasks/linux-packages/travis.linux.arm64.yml @@ -114,7 +114,7 @@ script: done - popd - | - travis_wait 30 \ + travis_wait 55 \ rake \ --trace \ {{ task_namespace }}:build \ From 080c85d5fa525ad61e3945bc76ad63c0597924bd Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 13 Jun 2022 14:31:32 +0900 Subject: [PATCH 5/8] Don't parallel --- dev/tasks/linux-packages/travis.linux.arm64.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/dev/tasks/linux-packages/travis.linux.arm64.yml b/dev/tasks/linux-packages/travis.linux.arm64.yml index 8ae1b611ec3..85c6fa64ed0 100644 --- a/dev/tasks/linux-packages/travis.linux.arm64.yml +++ b/dev/tasks/linux-packages/travis.linux.arm64.yml @@ -114,13 +114,12 @@ script: done - popd - | - travis_wait 55 \ - rake \ - --trace \ - {{ task_namespace }}:build \ - BUILD_DIR=build \ - DEB_BUILD_OPTIONS=parallel=2 \ - RPM_BUILD_NCPUS=2 + rake \ + --trace \ + {{ task_namespace }}:build \ + BUILD_DIR=build \ + DEB_BUILD_OPTIONS=parallel=2 \ + RPM_BUILD_NCPUS=1 - sudo rm -rf */*/build - popd # Push Docker image From 4a4cc1c89e7e2ef51f5f17db3cd18aefb9a8b801 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 13 Jun 2022 17:46:09 +0900 Subject: [PATCH 6/8] Use Makefile to avoid parallel build in external project --- dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 7df3bc4aec4..61bf2e9f11c 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -45,13 +45,13 @@ %if %{rhel} >= 9 %define arrow_cmake_builddir %{__cmake_builddir} %define arrow_cmake %cmake -%define arrow_cmake_build ninja -C %{arrow_cmake_builddir} %{?_smp_mflags} +%define arrow_cmake_build make -C %{arrow_cmake_builddir} %{?_smp_mflags} %define arrow_cmake_install %cmake_install %else %define arrow_cmake_builddir build %define arrow_cmake %cmake3 -S . -B %{arrow_cmake_builddir} -%define arrow_cmake_build ninja -C %{arrow_cmake_builddir} %{?_smp_mflags} -%define arrow_cmake_install DESTDIR="%{buildroot}" ninja -C %{arrow_cmake_builddir} install +%define arrow_cmake_build make -C %{arrow_cmake_builddir} %{?_smp_mflags} +%define arrow_cmake_install DESTDIR="%{buildroot}" make -C %{arrow_cmake_builddir} install %endif %define use_boost (!%{is_amazon_linux}) @@ -189,7 +189,7 @@ cd cpp -DPythonInterp_FIND_VERSION=ON \ -DPythonInterp_FIND_VERSION_MAJOR=3 \ %endif - -GNinja + -G"Unix Makefiles" %arrow_cmake_build cd - From e2dedd560fec0e74100fd33aa4bbc387ae12048b Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 14 Jun 2022 09:09:03 +0900 Subject: [PATCH 7/8] Use 2 CPUs --- dev/tasks/linux-packages/travis.linux.arm64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/linux-packages/travis.linux.arm64.yml b/dev/tasks/linux-packages/travis.linux.arm64.yml index 85c6fa64ed0..ef292cd6b4e 100644 --- a/dev/tasks/linux-packages/travis.linux.arm64.yml +++ b/dev/tasks/linux-packages/travis.linux.arm64.yml @@ -119,7 +119,7 @@ script: {{ task_namespace }}:build \ BUILD_DIR=build \ DEB_BUILD_OPTIONS=parallel=2 \ - RPM_BUILD_NCPUS=1 + RPM_BUILD_NCPUS=2 - sudo rm -rf */*/build - popd # Push Docker image From d5aa1d781334242606f564e266e8de0c6029eaa9 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 14 Jun 2022 14:20:54 +0900 Subject: [PATCH 8/8] Disable Flight on aarch64 --- dev/release/verify-yum.sh | 1 + dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh index 27427fbf49f..68b89379ec4 100755 --- a/dev/release/verify-yum.sh +++ b/dev/release/verify-yum.sh @@ -93,6 +93,7 @@ case "${distribution}-${distribution_version}" in ;; esac if [ "$(arch)" = "aarch64" ]; then + have_flight=no have_gandiva=no fi diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 61bf2e9f11c..d5bbb933841 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -55,7 +55,8 @@ %endif %define use_boost (!%{is_amazon_linux}) -%define use_flight (%{rhel} >= 8) +# TODO: Enable this on aarch64 too. This causes timeout on Travis CI. +%define use_flight (%{rhel} >= 8 && "%{_arch}" != "aarch64") %define use_gandiva (%{rhel} >= 8 && "%{_arch}" != "aarch64") %define use_gcs (%{rhel} >= 8 || %{is_amazon_linux}) %define use_gflags (!%{is_amazon_linux})