From 56098ae872edc68e583095b59fecef59e4f3310e Mon Sep 17 00:00:00 2001 From: Yosuke Shiro Date: Sat, 29 Jun 2019 16:30:40 +0900 Subject: [PATCH 1/9] [Release][GLib] Replace c_glib/ by c_glib/ after running autogen.sh --- dev/release/02-source.sh | 10 +++++----- dev/release/source/build.sh | 30 ++++-------------------------- 2 files changed, 9 insertions(+), 31 deletions(-) diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh index edaeec9ebe7..55d14b23820 100755 --- a/dev/release/02-source.sh +++ b/dev/release/02-source.sh @@ -61,13 +61,13 @@ rm -rf ${tag} git archive ${release_hash} --prefix ${tag}/) | \ tar xf - -# replace c_glib/ by tar.gz generated by "make dist" +# Replace c_glib/ after running c_glib/autogen.sh to create c_gilb/ source archive containing the configure script if [ ${SOURCE_GLIB} -gt 0 ]; then archive_name=tmp-apache-arrow (cd "${SOURCE_TOP_DIR}" && \ git archive ${release_hash} --prefix ${archive_name}/) \ > ${archive_name}.tar - dist_c_glib_tar_gz=c_glib.tar.gz + c_glib_including_configure_tar_gz=c_glib.tar.gz docker_image_name=apache-arrow/release-source DEBUG=yes docker build -t ${docker_image_name} "${SOURCE_DIR}/source" docker \ @@ -76,11 +76,11 @@ if [ ${SOURCE_GLIB} -gt 0 ]; then --interactive \ --volume "$PWD":/host \ ${docker_image_name} \ - /build.sh ${archive_name} ${dist_c_glib_tar_gz} + /build.sh ${archive_name} ${c_glib_including_configure_tar_gz} rm -f ${archive_name}.tar rm -rf ${tag}/c_glib - tar xf ${dist_c_glib_tar_gz} -C ${tag} - rm -f ${dist_c_glib_tar_gz} + tar xf ${c_glib_including_configure_tar_gz} -C ${tag} + rm -f ${c_glib_including_configure_tar_gz} fi # Resolve all hard and symbolic links diff --git a/dev/release/source/build.sh b/dev/release/source/build.sh index 20c972466b4..18f0bf6d459 100755 --- a/dev/release/source/build.sh +++ b/dev/release/source/build.sh @@ -20,35 +20,13 @@ set -e archive_name=$1 -dist_c_glib_tar_gz=$2 +c_glib_including_configure_tar_gz=$2 tar xf /host/${archive_name}.tar -# build Apache Arrow C++ before building Apache Arrow GLib because -# Apache Arrow GLib requires Apache Arrow C++. -mkdir -p ${archive_name}/cpp/build -cpp_install_dir=${PWD}/${archive_name}/cpp/install -cd ${archive_name}/cpp/build -cmake .. \ - -DCMAKE_INSTALL_PREFIX=${cpp_install_dir} \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DARROW_PLASMA=yes \ - -DARROW_GANDIVA=yes \ - -DARROW_PARQUET=yes -make -j8 -make install -cd - - -# build source archive for Apache Arrow GLib by "make dist". +# Run autogen.sh to create c_glib/ source archive containing the configure script cd ${archive_name}/c_glib ./autogen.sh -./configure \ - PKG_CONFIG_PATH=${cpp_install_dir}/lib/pkgconfig \ - --enable-gtk-doc -LD_LIBRARY_PATH=${cpp_install_dir}/lib make -j8 -make dist -tar xzf *.tar.gz -rm *.tar.gz cd - -mv ${archive_name}/c_glib/apache-arrow-glib-* c_glib/ -tar czf /host/${dist_c_glib_tar_gz} c_glib +mv ${archive_name}/c_glib/ c_glib/ +tar czf /host/${c_glib_including_configure_tar_gz} c_glib/ From e04276e33601498ed86abaafb7bcf2ae9d9f8a09 Mon Sep 17 00:00:00 2001 From: Yosuke Shiro Date: Sat, 29 Jun 2019 16:57:33 +0900 Subject: [PATCH 2/9] Remove libraries for C++ build --- dev/release/source/Dockerfile | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/dev/release/source/Dockerfile b/dev/release/source/Dockerfile index 9085cef3327..207f128fa6b 100644 --- a/dev/release/source/Dockerfile +++ b/dev/release/source/Dockerfile @@ -19,33 +19,14 @@ FROM ubuntu:18.04 ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends wget software-properties-common gpg-agent && \ - wget --quiet -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - apt-add-repository -y "deb http://apt.llvm.org/bionic llvm-toolchain-bionic-7 main" && \ - apt-get -y install clang-7 - RUN apt update && \ apt install -y -V \ autoconf-archive \ - bison \ - cmake \ - flex \ - g++ \ - gcc \ gtk-doc-tools \ - libboost-filesystem-dev \ - libboost-regex-dev \ - libboost-system-dev \ libgirepository1.0-dev \ libglib2.0-doc \ - libprotobuf-dev \ - libprotoc-dev \ libtool \ - lsb-release \ - make \ - pkg-config \ - protobuf-compiler && \ + pkg-config && \ apt clean && \ rm -rf /var/lib/apt/lists/* From aa786804d8d425724488655fbbe1b2d56dde314e Mon Sep 17 00:00:00 2001 From: Yosuke Shiro Date: Sat, 29 Jun 2019 17:02:32 +0900 Subject: [PATCH 3/9] Enable test test_glib_configure on Travis CI --- ci/travis_release_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/travis_release_test.sh b/ci/travis_release_test.sh index 4b541525fc4..dec8528fd8b 100755 --- a/ci/travis_release_test.sh +++ b/ci/travis_release_test.sh @@ -29,4 +29,4 @@ sudo apt-get install apt-transport-https sudo apt-get update sudo apt-get install dotnet-sdk-2.2 -dev/release/run-test.rb +ARROW_TEST_SOURCE_GLIB=yes dev/release/run-test.rb From 70cb4a762d1c190141c80c97c328e8dc82f2b983 Mon Sep 17 00:00:00 2001 From: Yosuke Shiro Date: Sat, 29 Jun 2019 18:35:16 +0900 Subject: [PATCH 4/9] Remove an unnecessary diff --- dev/release/source/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/source/build.sh b/dev/release/source/build.sh index 18f0bf6d459..2eb0c3dce1f 100755 --- a/dev/release/source/build.sh +++ b/dev/release/source/build.sh @@ -29,4 +29,4 @@ cd ${archive_name}/c_glib ./autogen.sh cd - mv ${archive_name}/c_glib/ c_glib/ -tar czf /host/${c_glib_including_configure_tar_gz} c_glib/ +tar czf /host/${c_glib_including_configure_tar_gz} c_glib From e357a88b787836e9eb2f617b59ff8e1ab9adc7ff Mon Sep 17 00:00:00 2001 From: Yosuke Shiro Date: Sat, 29 Jun 2019 19:37:03 +0900 Subject: [PATCH 5/9] Exclude c_glib/autom4te.cache/* from RAT check --- dev/release/rat_exclude_files.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 79a1c10483b..037492bcdf5 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -157,6 +157,7 @@ MANIFEST.in __init__.pxd __init__.py requirements.txt +c_glib/autom4te.cache/* c_glib/configure c_glib/version c_glib/m4/*.m4 From 46a4f89951cba296d7b92cf71cb7f8d604f04a5b Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 3 Jul 2019 15:31:44 +0900 Subject: [PATCH 6/9] Use docker-compose --- dev/release/02-source.sh | 22 +++++++++------------- dev/release/source/Dockerfile | 4 +--- dev/release/source/build.sh | 4 ++-- docker-compose.yml | 10 ++++++++++ 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/dev/release/02-source.sh b/dev/release/02-source.sh index 55d14b23820..ca192a8be9a 100755 --- a/dev/release/02-source.sh +++ b/dev/release/02-source.sh @@ -66,21 +66,17 @@ if [ ${SOURCE_GLIB} -gt 0 ]; then archive_name=tmp-apache-arrow (cd "${SOURCE_TOP_DIR}" && \ git archive ${release_hash} --prefix ${archive_name}/) \ - > ${archive_name}.tar + > "${SOURCE_TOP_DIR}/${archive_name}.tar" c_glib_including_configure_tar_gz=c_glib.tar.gz - docker_image_name=apache-arrow/release-source - DEBUG=yes docker build -t ${docker_image_name} "${SOURCE_DIR}/source" - docker \ - run \ - --rm \ - --interactive \ - --volume "$PWD":/host \ - ${docker_image_name} \ - /build.sh ${archive_name} ${c_glib_including_configure_tar_gz} - rm -f ${archive_name}.tar + "${SOURCE_TOP_DIR}/dev/run_docker_compose.sh" \ + release-source \ + /arrow/dev/release/source/build.sh \ + ${archive_name} \ + ${c_glib_including_configure_tar_gz} + rm -f "${SOURCE_TOP_DIR}/${archive_name}.tar" rm -rf ${tag}/c_glib - tar xf ${c_glib_including_configure_tar_gz} -C ${tag} - rm -f ${c_glib_including_configure_tar_gz} + tar xf "${SOURCE_TOP_DIR}/${c_glib_including_configure_tar_gz}" -C ${tag} + rm -f "${SOURCE_TOP_DIR}/${c_glib_including_configure_tar_gz}" fi # Resolve all hard and symbolic links diff --git a/dev/release/source/Dockerfile b/dev/release/source/Dockerfile index 207f128fa6b..7d5453b80c4 100644 --- a/dev/release/source/Dockerfile +++ b/dev/release/source/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:18.04 +FROM debian:buster ENV DEBIAN_FRONTEND noninteractive @@ -29,5 +29,3 @@ RUN apt update && \ pkg-config && \ apt clean && \ rm -rf /var/lib/apt/lists/* - -COPY build.sh /build.sh diff --git a/dev/release/source/build.sh b/dev/release/source/build.sh index 2eb0c3dce1f..53bcfe14034 100755 --- a/dev/release/source/build.sh +++ b/dev/release/source/build.sh @@ -22,11 +22,11 @@ set -e archive_name=$1 c_glib_including_configure_tar_gz=$2 -tar xf /host/${archive_name}.tar +tar xf /arrow/${archive_name}.tar # Run autogen.sh to create c_glib/ source archive containing the configure script cd ${archive_name}/c_glib ./autogen.sh cd - mv ${archive_name}/c_glib/ c_glib/ -tar czf /host/${c_glib_including_configure_tar_gz} c_glib +tar czf /arrow/${c_glib_including_configure_tar_gz} c_glib diff --git a/docker-compose.yml b/docker-compose.yml index 94171483c9a..768c4e6eb72 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -609,6 +609,16 @@ services: # TODO(kszucs): hive-integration + ################################# Release ################################### + + release-source: + image: arrow:release-source + build: + context: . + dockerfile: dev/release/source/Dockerfile + volumes: + - .:/arrow:delegated + ######################## Verification Containers ############################ debian-stretch: From 501a2dd97a0f974e826377bfa46d0f6c82b54d5d Mon Sep 17 00:00:00 2001 From: Yosuke Shiro Date: Wed, 3 Jul 2019 23:01:41 +0900 Subject: [PATCH 7/9] Remove autom4te.cache after running autogen.sh --- dev/release/rat_exclude_files.txt | 1 - dev/release/source/build.sh | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 037492bcdf5..79a1c10483b 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -157,7 +157,6 @@ MANIFEST.in __init__.pxd __init__.py requirements.txt -c_glib/autom4te.cache/* c_glib/configure c_glib/version c_glib/m4/*.m4 diff --git a/dev/release/source/build.sh b/dev/release/source/build.sh index 53bcfe14034..558600e1fb7 100755 --- a/dev/release/source/build.sh +++ b/dev/release/source/build.sh @@ -27,6 +27,7 @@ tar xf /arrow/${archive_name}.tar # Run autogen.sh to create c_glib/ source archive containing the configure script cd ${archive_name}/c_glib ./autogen.sh +rm -rf autom4te.cache cd - mv ${archive_name}/c_glib/ c_glib/ tar czf /arrow/${c_glib_including_configure_tar_gz} c_glib From 3a2550fbd244bee4d69666a766f0f33103c40d60 Mon Sep 17 00:00:00 2001 From: Yosuke Shiro Date: Wed, 3 Jul 2019 23:05:56 +0900 Subject: [PATCH 8/9] Remove omit from 02-source-test.rb --- dev/release/02-source-test.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dev/release/02-source-test.rb b/dev/release/02-source-test.rb index e777c7561c4..cf3c887a1c8 100644 --- a/dev/release/02-source-test.rb +++ b/dev/release/02-source-test.rb @@ -55,10 +55,6 @@ def test_symbolic_links end def test_glib_configure - unless ENV["ARROW_TEST_SOURCE_GLIB"] == "yes" - omit("This takes a long time. " + - "Set ARROW_TEST_SOURCE_GLIB=yes environment variable to test this.") - end source("GLIB") Dir.chdir("#{@tag_name}/c_glib") do assert_equal([ From 9a69f8edcaed369c42c61f137510cb9c7ce6f16f Mon Sep 17 00:00:00 2001 From: Yosuke Shiro Date: Fri, 5 Jul 2019 08:50:05 +0900 Subject: [PATCH 9/9] Remove an unnecessary environment variable --- ci/travis_release_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/travis_release_test.sh b/ci/travis_release_test.sh index dec8528fd8b..4b541525fc4 100755 --- a/ci/travis_release_test.sh +++ b/ci/travis_release_test.sh @@ -29,4 +29,4 @@ sudo apt-get install apt-transport-https sudo apt-get update sudo apt-get install dotnet-sdk-2.2 -ARROW_TEST_SOURCE_GLIB=yes dev/release/run-test.rb +dev/release/run-test.rb