diff --git a/.travis.yml b/.travis.yml index c4a8f4120c7..f6f64003d27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -128,7 +128,7 @@ matrix: script: - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh - compiler: clang - osx_image: xcode6.4 + osx_image: xcode8.3 os: osx cache: addons: diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh index 6799ae4ce41..52bfe87eba7 100755 --- a/ci/travis_before_script_c_glib.sh +++ b/ci/travis_before_script_c_glib.sh @@ -23,7 +23,9 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh if [ $TRAVIS_OS_NAME == "osx" ]; then brew install gtk-doc autoconf-archive gobject-introspection - brew upgrade git cmake wget libtool + brew upgrade git cmake + brew outdated || brew upgrade wget + brew outdated || brew upgrade libtool export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/libffi/lib/pkgconfig else @@ -36,6 +38,7 @@ gem install test-unit gobject-introspection if [ $TRAVIS_OS_NAME == "osx" ]; then brew install lua + sudo env PKG_CONFIG_PATH=$PKG_CONFIG_PATH luarocks install lgi else git clone \ --quiet \ @@ -47,8 +50,8 @@ else echo "yes" | ./install.sh > /dev/null . ~/torch/install/bin/torch-activate popd + luarocks install lgi fi -luarocks install lgi go get github.com/linuxdeepin/go-gir-generator || : pushd $GOPATH/src/github.com/linuxdeepin/go-gir-generator diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh index a613957ba0d..d46fa2ff943 100755 --- a/ci/travis_before_script_cpp.sh +++ b/ci/travis_before_script_cpp.sh @@ -70,7 +70,8 @@ if [ $only_library_mode == "yes" ]; then CMAKE_COMMON_FLAGS="\ $CMAKE_COMMON_FLAGS \ -DARROW_BUILD_TESTS=OFF \ --DARROW_BUILD_UTILITIES=OFF" +-DARROW_BUILD_UTILITIES=OFF \ +-DARROW_INSTALL_NAME_RPATH=OFF" fi # Use Ninja for faster builds when using toolchain diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 577a4bbea83..972132f29e9 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -150,6 +150,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") "Build Arrow libraries with RATH set to \$ORIGIN" OFF) + option(ARROW_INSTALL_NAME_RPATH + "Build Arrow libraries with install_name set to @rpath" + ON) + option(ARROW_PLASMA "Build the plasma object store along with Arrow" OFF) diff --git a/cpp/apidoc/index.md b/cpp/apidoc/index.md index 293241c0dc7..25be1f284fc 100644 --- a/cpp/apidoc/index.md +++ b/cpp/apidoc/index.md @@ -55,7 +55,7 @@ build these objects. To build an array of `int64_t` elements, we can use the `arrow::Int64Builder`. In the following example, we build an array of the range 1 to 8 where the element that should hold the number 4 is nulled. - Int64Builder builder(arrow::default_memory_pool(), arrow::int64()); + Int64Builder builder; builder.Append(1); builder.Append(2); builder.Append(3); diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index 6b2be41d24a..8f92d73baea 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -176,10 +176,15 @@ function(ADD_ARROW_LIB LIB_NAME) endif() if (APPLE) - set_target_properties(${LIB_NAME}_shared + if (ARROW_INSTALL_NAME_RPATH) + set(_lib_install_name "@rpath") + else() + set(_lib_install_name "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") + endif() + set_target_properties(${LIB_NAME}_shared PROPERTIES BUILD_WITH_INSTALL_RPATH ON - INSTALL_NAME_DIR "@rpath") + INSTALL_NAME_DIR "${_lib_install_name}") endif() endfunction() diff --git a/dev/README.md b/dev/README.md index e986abef191..91d688dfee7 100644 --- a/dev/README.md +++ b/dev/README.md @@ -92,3 +92,21 @@ Merge hash: 485658a5 Would you like to pick 485658a5 into another branch? (y/n): ``` For now just say n as we have 1 branch + +## Verifying Release Candidates + +We have provided a script to assist with verifying release candidates: + +```shell +bash dev/release/verify-release-candidate.sh 0.7.0 0 +``` + +Currently this only works on Linux (patches to expand to macOS welcome!). Read +the script for information about system dependencies. + +On Windows, we have a script that verifies C++ and Python (requires Visual +Studio 2015): + +``` +dev/release/verify-release-candidate.bat apache-arrow-0.7.0.tar.gz +``` \ No newline at end of file diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat index 503f1fd40b6..27157da6687 100644 --- a/dev/release/verify-release-candidate.bat +++ b/dev/release/verify-release-candidate.bat @@ -45,15 +45,18 @@ tar xvf %1.tar.gz -C "C:/tmp/" set GENERATOR=Visual Studio 14 2015 Win64 set CONFIGURATION=release set ARROW_SOURCE=C:\tmp\%1 +set INSTALL_DIR=C:\tmp\%1\install pushd %ARROW_SOURCE% call activate arrow-verify-release set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library -set ARROW_HOME=%CONDA_PREFIX%\Library set PARQUET_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library -set PARQUET_HOME=%CONDA_PREFIX%\Library + +set ARROW_HOME=%INSTALL_DIR% +set PARQUET_HOME=%INSTALL_DIR% +set PATH=%INSTALL_DIR%\bin;%PATH% @rem Build and test Arrow C++ libraries mkdir cpp\build diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh new file mode 100755 index 00000000000..38680f429ed --- /dev/null +++ b/dev/release/verify-release-candidate.sh @@ -0,0 +1,244 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Requirements +# - Ruby 2.x +# - Plus gem dependencies, see c_glib/README +# - Maven >= 3.3.9 +# - JDK >=7 +# - gcc >= 4.8 +# - nodejs >= 6.0.0 (best way is to use nvm) +# +# If using a non-system Boost, set BOOST_ROOT and add Boost libraries to +# LD_LIBRARY_PATH + +case $# in + 2) VERSION="$1" + RC_NUMBER="$2" + ;; + + *) echo "Usage: $0 X.Y.Z RC_NUMBER" + exit 1 + ;; +esac + +set -ex + +HERE=$(cd `dirname "${BASH_SOURCE[0]:-$0}"` && pwd) + +ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow' + +download_dist_file() { + curl -f -O $ARROW_DIST_URL/$1 +} + +download_rc_file() { + download_dist_file apache-arrow-${VERSION}-rc${RC_NUMBER}/$1 +} + +import_gpg_keys() { + download_dist_file KEYS + gpg --import KEYS +} + +fetch_archive() { + local dist_name=$1 + download_rc_file ${dist_name}.tar.gz + download_rc_file ${dist_name}.tar.gz.asc + download_rc_file ${dist_name}.tar.gz.md5 + download_rc_file ${dist_name}.tar.gz.sha512 + gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz + gpg --print-md MD5 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.md5 + sha512sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512 +} + +setup_tempdir() { + cleanup() { + rm -fr "$TMPDIR" + } + trap cleanup EXIT + TMPDIR=$(mktemp -d -t "$1.XXXXX") +} + + +setup_miniconda() { + # Setup short-lived miniconda for Python and integration tests + MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh + + MINICONDA=`pwd`/test-miniconda + + wget -O miniconda.sh $MINICONDA_URL + bash miniconda.sh -b -p $MINICONDA + rm -f miniconda.sh + + export PATH=$MINICONDA/bin:$PATH + + conda create -n arrow-test -y -q python=3.6 \ + nomkl numpy pandas six cython + source activate arrow-test +} + +# Build and test C++ + +test_and_install_cpp() { + mkdir cpp/build + pushd cpp/build + + cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ + -DARROW_PLASMA=on \ + -DARROW_PYTHON=on \ + -DARROW_BOOST_USE_SHARED=on \ + -DCMAKE_BUILD_TYPE=release \ + -DARROW_BUILD_BENCHMARKS=on \ + .. + + make -j$NPROC + make install + + ctest -L unittest + popd +} + +# Build and install Parquet master so we can test the Python bindings + +install_parquet_cpp() { + git clone git@github.com:apache/parquet-cpp.git + + mkdir parquet-cpp/build + pushd parquet-cpp/build + + cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ + -DCMAKE_BUILD_TYPE=release \ + -DPARQUET_BOOST_USE_SHARED=on \ + -DPARQUET_BUILD_TESTS=off \ + .. + + make -j$NPROC + make install + + popd +} + +# Build and test Python + +test_python() { + pushd python + + pip install -r requirements.txt + + python setup.py build_ext --inplace --with-parquet --with-plasma + py.test pyarrow -v --pdb + + popd +} + + +test_glib() { + # Build and test GLib, requires GLib >= 2.44 , so install that + # here + GLIB_VERSION=glib-2.53.7 + GLIB_URL=https://gensho.ftp.acc.umu.se/pub/gnome/sources/glib/2.53/$GLIB_VERSION.tar.xz + curl -f -O $GLIB_URL + tar xf $GLIB_VERSION.tar.xz + pushd $GLIB_VERSION + + ./configure --disable-libelf --enable-libmount=no --prefix=$ARROW_HOME + make -j$NPROC + make install + + popd + + pushd c_glib + + ./configure --prefix=$ARROW_HOME + make -j$NPROC + make install + + NO_MAKE=yes test/run-test.sh + + popd +} + +test_js() { + pushd js + npm install + npm run validate + popd +} + +# Build and test Java (Requires newer Maven -- I used 3.3.9) + +test_package_java() { + pushd java + + mvn test + mvn package + + popd +} + +# Run integration tests +test_integration() { + JAVA_DIR=`pwd`/java + CPP_BUILD_DIR=`pwd`/cpp/build + + export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar + export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/release + + pushd integration + + python integration_test.py + + popd +} + +setup_tempdir "arrow-$VERSION" +echo "Working in sandbox $TMPDIR" +cd $TMPDIR + +export ARROW_HOME=$TMPDIR/install +export PARQUET_HOME=$TMPDIR/install +export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH +export PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig:$PKG_CONFIG_PATH + +NPROC=$(nproc) +VERSION=$1 +RC_NUMBER=$2 + +TARBALL=apache-arrow-$1.tar.gz + +import_gpg_keys + +DIST_NAME="apache-arrow-${VERSION}" +fetch_archive $DIST_NAME +tar xvzf ${DIST_NAME}.tar.gz +cd ${DIST_NAME} + +setup_miniconda +test_and_install_cpp +install_parquet_cpp +test_python +test_glib +test_package_java +test_integration +test_js + +echo 'Release candidate looks good!' +exit 0