From 5f72fd115289844fd1a349328df13cdf30bf47d1 Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Wed, 13 Sep 2017 09:09:38 -0400 Subject: [PATCH 1/5] ARROW-1529: [GLib] Use Xcode 8.3 on Travis CI Author: Kouhei Sutou Closes #1092 from kou/glib-travis-macos and squashes the following commits: 291808b2 [Kouhei Sutou] [GLib] Use Xcode 8.3 on Travis CI --- .travis.yml | 2 +- ci/travis_before_script_c_glib.sh | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index c4a8f4120c7..f6f64003d27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -128,7 +128,7 @@ matrix: script: - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh - compiler: clang - osx_image: xcode6.4 + osx_image: xcode8.3 os: osx cache: addons: diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh index 6799ae4ce41..533c892712d 100755 --- a/ci/travis_before_script_c_glib.sh +++ b/ci/travis_before_script_c_glib.sh @@ -23,7 +23,9 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh if [ $TRAVIS_OS_NAME == "osx" ]; then brew install gtk-doc autoconf-archive gobject-introspection - brew upgrade git cmake wget libtool + brew upgrade git cmake + brew outdated || brew upgrade wget + brew outdated || brew upgrade libtool export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/libffi/lib/pkgconfig else @@ -36,6 +38,7 @@ gem install test-unit gobject-introspection if [ $TRAVIS_OS_NAME == "osx" ]; then brew install lua + sudo env PKG_CONFIG_PATH=$PKG_CONFIG_PATH luarocks install lgi else git clone \ --quiet \ @@ -47,8 +50,8 @@ else echo "yes" | ./install.sh > /dev/null . ~/torch/install/bin/torch-activate popd + luarocks install lgi fi -luarocks install lgi go get github.com/linuxdeepin/go-gir-generator || : pushd $GOPATH/src/github.com/linuxdeepin/go-gir-generator @@ -74,6 +77,12 @@ pushd $ARROW_C_GLIB_DIR export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib +if [ $TRAVIS_OS_NAME == "osx" ]; then + install_name_tool \ + -id $ARROW_CPP_INSTALL/lib/libarrow.dylib \ + $ARROW_CPP_INSTALL/lib/libarrow.dylib +fi + CONFIGURE_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL" if [ $TRAVIS_OS_NAME != "osx" ]; then CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS --enable-gtk-doc" From 46ee940d6fad1a6b7b8d2afc7f7a0dac810c58fd Mon Sep 17 00:00:00 2001 From: Kouhei Sutou Date: Fri, 15 Sep 2017 10:47:04 -0400 Subject: [PATCH 2/5] ARROW-1537: [C++] Support building with full path install_name on macOS If you use `@rpath` for install_name (default), you can use the DYLD_LIBRARY_PATH environment variable to find libarrow.dylib. But the DYLD_LIBRARY_PATH environment variable isn't inherited to sub process by System Integration Protection (SIP). It's difficult to use libarrow.dylib. You can use full path install_name by -DARROW_INSTALL_NAME_RPATH=OFF CMake option. If you use it, you can find libarrow.dylib without DYLD_LIBRARY_PATH environment variable. Author: Kouhei Sutou Closes #1100 from kou/cpp-macos-support-install-name and squashes the following commits: 8207ace [Kouhei Sutou] [C++] Support building with full path install_name on macOS --- ci/travis_before_script_c_glib.sh | 6 ------ ci/travis_before_script_cpp.sh | 3 ++- cpp/CMakeLists.txt | 4 ++++ cpp/cmake_modules/BuildUtils.cmake | 9 +++++++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh index 533c892712d..52bfe87eba7 100755 --- a/ci/travis_before_script_c_glib.sh +++ b/ci/travis_before_script_c_glib.sh @@ -77,12 +77,6 @@ pushd $ARROW_C_GLIB_DIR export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib -if [ $TRAVIS_OS_NAME == "osx" ]; then - install_name_tool \ - -id $ARROW_CPP_INSTALL/lib/libarrow.dylib \ - $ARROW_CPP_INSTALL/lib/libarrow.dylib -fi - CONFIGURE_OPTIONS="--prefix=$ARROW_C_GLIB_INSTALL" if [ $TRAVIS_OS_NAME != "osx" ]; then CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS --enable-gtk-doc" diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh index a613957ba0d..d46fa2ff943 100755 --- a/ci/travis_before_script_cpp.sh +++ b/ci/travis_before_script_cpp.sh @@ -70,7 +70,8 @@ if [ $only_library_mode == "yes" ]; then CMAKE_COMMON_FLAGS="\ $CMAKE_COMMON_FLAGS \ -DARROW_BUILD_TESTS=OFF \ --DARROW_BUILD_UTILITIES=OFF" +-DARROW_BUILD_UTILITIES=OFF \ +-DARROW_INSTALL_NAME_RPATH=OFF" fi # Use Ninja for faster builds when using toolchain diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 577a4bbea83..972132f29e9 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -150,6 +150,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") "Build Arrow libraries with RATH set to \$ORIGIN" OFF) + option(ARROW_INSTALL_NAME_RPATH + "Build Arrow libraries with install_name set to @rpath" + ON) + option(ARROW_PLASMA "Build the plasma object store along with Arrow" OFF) diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index 6b2be41d24a..8f92d73baea 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -176,10 +176,15 @@ function(ADD_ARROW_LIB LIB_NAME) endif() if (APPLE) - set_target_properties(${LIB_NAME}_shared + if (ARROW_INSTALL_NAME_RPATH) + set(_lib_install_name "@rpath") + else() + set(_lib_install_name "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") + endif() + set_target_properties(${LIB_NAME}_shared PROPERTIES BUILD_WITH_INSTALL_RPATH ON - INSTALL_NAME_DIR "@rpath") + INSTALL_NAME_DIR "${_lib_install_name}") endif() endfunction() From c762764065e602980ad99951003ca9593e790c7f Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 15 Sep 2017 10:48:22 -0400 Subject: [PATCH 3/5] ARROW-1542: [C++] Install packages in temporary directory in MSVC build verification script I found that the script did not work due to the remnants of the last time I ran it. Author: Wes McKinney Closes #1101 from wesm/ARROW-1542 and squashes the following commits: 0718370 [Wes McKinney] Install packages in temporary directory in MSVC build verification script --- dev/release/verify-release-candidate.bat | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat index 503f1fd40b6..27157da6687 100644 --- a/dev/release/verify-release-candidate.bat +++ b/dev/release/verify-release-candidate.bat @@ -45,15 +45,18 @@ tar xvf %1.tar.gz -C "C:/tmp/" set GENERATOR=Visual Studio 14 2015 Win64 set CONFIGURATION=release set ARROW_SOURCE=C:\tmp\%1 +set INSTALL_DIR=C:\tmp\%1\install pushd %ARROW_SOURCE% call activate arrow-verify-release set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library -set ARROW_HOME=%CONDA_PREFIX%\Library set PARQUET_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library -set PARQUET_HOME=%CONDA_PREFIX%\Library + +set ARROW_HOME=%INSTALL_DIR% +set PARQUET_HOME=%INSTALL_DIR% +set PATH=%INSTALL_DIR%\bin;%PATH% @rem Build and test Arrow C++ libraries mkdir cpp\build From 40e72f5fb916ae7833624fad9aae1746de8f0ab8 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 15 Sep 2017 12:29:10 -0400 Subject: [PATCH 4/5] ARROW-559: Add release verification script for Linux Since we're accumulating a bunch of components, I started this script which we can refine to make verifying releases easier for others. I bootstrapped some pieces off https://github.com/apache/parquet-cpp/blob/master/dev/release/verify-release-candidate, very helpful! This script: * Checks GPG signature, checksums * Installs temporary Python install for the duration of these tests * Builds/install C++ and runs tests (with Python and Plasma) * Builds parquet-cpp against the Arrow RC * Python (with Parquet and Plasma extensions) * C GLib (requires Ruby in PATH and the gems indicated in README) * Integration tests * JavaScript (requires NodeJS >= 6.0.0) There are some potentially snowflake-y aspects to my environment: * BOOST_ROOT is set to a Boost install location containing libraries built with `-fPIC`. I'm not sure what to do about this one. One maybe better option is to use system level boost and shared libraries * Maven 3.3.9 is in PATH * NodeJS 6.11.3 is in PATH There are probably some other things that Linux users will run into as they run this script. I had to compile GLib libraries in this since the ones at system level (Ubuntu 14.04) are too old. cc @kou @xhochy Author: Wes McKinney Closes #1102 from wesm/ARROW-559 and squashes the following commits: 8fd6530 [Wes McKinney] Use Boost shared libraries 3531927 [Wes McKinney] Add note to dev/README.md 079b5e4 [Wes McKinney] Fix comments 17f7ac0 [Wes McKinney] More fixes, finally works adb3146 [Wes McKinney] More work on release verification script 86ef171 [Wes McKinney] Start Linux release verification script --- dev/README.md | 18 ++ dev/release/verify-release-candidate.sh | 244 ++++++++++++++++++++++++ 2 files changed, 262 insertions(+) create mode 100755 dev/release/verify-release-candidate.sh diff --git a/dev/README.md b/dev/README.md index e986abef191..91d688dfee7 100644 --- a/dev/README.md +++ b/dev/README.md @@ -92,3 +92,21 @@ Merge hash: 485658a5 Would you like to pick 485658a5 into another branch? (y/n): ``` For now just say n as we have 1 branch + +## Verifying Release Candidates + +We have provided a script to assist with verifying release candidates: + +```shell +bash dev/release/verify-release-candidate.sh 0.7.0 0 +``` + +Currently this only works on Linux (patches to expand to macOS welcome!). Read +the script for information about system dependencies. + +On Windows, we have a script that verifies C++ and Python (requires Visual +Studio 2015): + +``` +dev/release/verify-release-candidate.bat apache-arrow-0.7.0.tar.gz +``` \ No newline at end of file diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh new file mode 100755 index 00000000000..38680f429ed --- /dev/null +++ b/dev/release/verify-release-candidate.sh @@ -0,0 +1,244 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Requirements +# - Ruby 2.x +# - Plus gem dependencies, see c_glib/README +# - Maven >= 3.3.9 +# - JDK >=7 +# - gcc >= 4.8 +# - nodejs >= 6.0.0 (best way is to use nvm) +# +# If using a non-system Boost, set BOOST_ROOT and add Boost libraries to +# LD_LIBRARY_PATH + +case $# in + 2) VERSION="$1" + RC_NUMBER="$2" + ;; + + *) echo "Usage: $0 X.Y.Z RC_NUMBER" + exit 1 + ;; +esac + +set -ex + +HERE=$(cd `dirname "${BASH_SOURCE[0]:-$0}"` && pwd) + +ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow' + +download_dist_file() { + curl -f -O $ARROW_DIST_URL/$1 +} + +download_rc_file() { + download_dist_file apache-arrow-${VERSION}-rc${RC_NUMBER}/$1 +} + +import_gpg_keys() { + download_dist_file KEYS + gpg --import KEYS +} + +fetch_archive() { + local dist_name=$1 + download_rc_file ${dist_name}.tar.gz + download_rc_file ${dist_name}.tar.gz.asc + download_rc_file ${dist_name}.tar.gz.md5 + download_rc_file ${dist_name}.tar.gz.sha512 + gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz + gpg --print-md MD5 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.md5 + sha512sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512 +} + +setup_tempdir() { + cleanup() { + rm -fr "$TMPDIR" + } + trap cleanup EXIT + TMPDIR=$(mktemp -d -t "$1.XXXXX") +} + + +setup_miniconda() { + # Setup short-lived miniconda for Python and integration tests + MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh + + MINICONDA=`pwd`/test-miniconda + + wget -O miniconda.sh $MINICONDA_URL + bash miniconda.sh -b -p $MINICONDA + rm -f miniconda.sh + + export PATH=$MINICONDA/bin:$PATH + + conda create -n arrow-test -y -q python=3.6 \ + nomkl numpy pandas six cython + source activate arrow-test +} + +# Build and test C++ + +test_and_install_cpp() { + mkdir cpp/build + pushd cpp/build + + cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ + -DARROW_PLASMA=on \ + -DARROW_PYTHON=on \ + -DARROW_BOOST_USE_SHARED=on \ + -DCMAKE_BUILD_TYPE=release \ + -DARROW_BUILD_BENCHMARKS=on \ + .. + + make -j$NPROC + make install + + ctest -L unittest + popd +} + +# Build and install Parquet master so we can test the Python bindings + +install_parquet_cpp() { + git clone git@github.com:apache/parquet-cpp.git + + mkdir parquet-cpp/build + pushd parquet-cpp/build + + cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ + -DCMAKE_BUILD_TYPE=release \ + -DPARQUET_BOOST_USE_SHARED=on \ + -DPARQUET_BUILD_TESTS=off \ + .. + + make -j$NPROC + make install + + popd +} + +# Build and test Python + +test_python() { + pushd python + + pip install -r requirements.txt + + python setup.py build_ext --inplace --with-parquet --with-plasma + py.test pyarrow -v --pdb + + popd +} + + +test_glib() { + # Build and test GLib, requires GLib >= 2.44 , so install that + # here + GLIB_VERSION=glib-2.53.7 + GLIB_URL=https://gensho.ftp.acc.umu.se/pub/gnome/sources/glib/2.53/$GLIB_VERSION.tar.xz + curl -f -O $GLIB_URL + tar xf $GLIB_VERSION.tar.xz + pushd $GLIB_VERSION + + ./configure --disable-libelf --enable-libmount=no --prefix=$ARROW_HOME + make -j$NPROC + make install + + popd + + pushd c_glib + + ./configure --prefix=$ARROW_HOME + make -j$NPROC + make install + + NO_MAKE=yes test/run-test.sh + + popd +} + +test_js() { + pushd js + npm install + npm run validate + popd +} + +# Build and test Java (Requires newer Maven -- I used 3.3.9) + +test_package_java() { + pushd java + + mvn test + mvn package + + popd +} + +# Run integration tests +test_integration() { + JAVA_DIR=`pwd`/java + CPP_BUILD_DIR=`pwd`/cpp/build + + export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar + export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/release + + pushd integration + + python integration_test.py + + popd +} + +setup_tempdir "arrow-$VERSION" +echo "Working in sandbox $TMPDIR" +cd $TMPDIR + +export ARROW_HOME=$TMPDIR/install +export PARQUET_HOME=$TMPDIR/install +export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH +export PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig:$PKG_CONFIG_PATH + +NPROC=$(nproc) +VERSION=$1 +RC_NUMBER=$2 + +TARBALL=apache-arrow-$1.tar.gz + +import_gpg_keys + +DIST_NAME="apache-arrow-${VERSION}" +fetch_archive $DIST_NAME +tar xvzf ${DIST_NAME}.tar.gz +cd ${DIST_NAME} + +setup_miniconda +test_and_install_cpp +install_parquet_cpp +test_python +test_glib +test_package_java +test_integration +test_js + +echo 'Release candidate looks good!' +exit 0 From 9c6c128c587b41e1bb22d3f58273be4cf3592b3d Mon Sep 17 00:00:00 2001 From: rvernica Date: Sun, 17 Sep 2017 09:11:04 -0700 Subject: [PATCH 5/5] ARROW-1545: Remove deprecared args of builder --- cpp/apidoc/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/apidoc/index.md b/cpp/apidoc/index.md index 293241c0dc7..25be1f284fc 100644 --- a/cpp/apidoc/index.md +++ b/cpp/apidoc/index.md @@ -55,7 +55,7 @@ build these objects. To build an array of `int64_t` elements, we can use the `arrow::Int64Builder`. In the following example, we build an array of the range 1 to 8 where the element that should hold the number 4 is nulled. - Int64Builder builder(arrow::default_memory_pool(), arrow::int64()); + Int64Builder builder; builder.Append(1); builder.Append(2); builder.Append(3);