From 86ef171efd210a084d28869cf2504533b4d969ec Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 14 Sep 2017 22:53:50 -0400
Subject: [PATCH 1/6] Start Linux release verification script

Change-Id: Ida0d5a74765e28f8f88306edfef12f6bf723c859
---
 dev/release/verify-release-candidate.sh | 235 ++++++++++++++++++++++++
 1 file changed, 235 insertions(+)
 create mode 100755 dev/release/verify-release-candidate.sh

diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
new file mode 100755
index 00000000000..13db6286d39
--- /dev/null
+++ b/dev/release/verify-release-candidate.sh
@@ -0,0 +1,235 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Requirements
+# - Ruby 2.x
+# - Maven >= 3.3.9
+# - JDK >=7
+# - gcc >= 4.8
+
+case $# in
+  2) VERSION="$1"
+     RC_NUMBER="$2"
+     ;;
+
+  *) echo "Usage: $0 X.Y.Z RC_NUMBER"
+     exit 1
+     ;;
+esac
+
+set -ex
+
+HERE=$(cd `dirname "${BASH_SOURCE[0]:-$0}"` && pwd)
+
+ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow'
+
+download_dist_file() {
+  curl -f -O $ARROW_DIST_URL/$1
+}
+
+download_rc_file() {
+  download_dist_file apache-arrow-${VERSION}-rc${RC_NUMBER}/$1
+}
+
+import_gpg_keys() {
+  download_dist_file KEYS
+  gpg --import KEYS
+}
+
+fetch_archive() {
+  local dist_name=$1
+  download_rc_file ${dist_name}.tar.gz
+  download_rc_file ${dist_name}.tar.gz.asc
+  download_rc_file ${dist_name}.tar.gz.md5
+  download_rc_file ${dist_name}.tar.gz.sha512
+  gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
+  gpg --print-md MD5 ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.md5
+  sha512sum ${dist_name}.tar.gz | diff - ${dist_name}.tar.gz.sha512
+}
+
+setup_tempdir() {
+  cleanup() {
+    rm -fr "$TMPDIR"
+  }
+  trap cleanup EXIT
+  TMPDIR=$(mktemp -d -t "$1.XXXXX")
+}
+
+
+setup_miniconda() {
+  # Setup short-lived miniconda for Python and integration tests
+  MINICONDA_URL=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+
+  MINICONDA=`pwd`/test-miniconda
+
+  wget -O miniconda.sh $MINICONDA_URL
+  bash miniconda.sh -b -p $MINICONDA
+  rm -f miniconda.sh
+
+  export PATH=$MINICONDA/bin:$PATH
+
+  conda create -n arrow-test -y -q python=3.6 \
+        nomkl numpy pandas six cython
+  source activate arrow-test
+}
+
+# Build and test C++
+
+test_and_install_cpp() {
+  mkdir cpp/build
+  pushd cpp/build
+
+  cmake .. -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+        -DARROW_PLASMA=on \
+        -DARROW_PYTHON=on \
+        -DCMAKE_BUILD_TYPE=release \
+        -DARROW_BUILD_BENCHMARKS=on \
+        ..
+
+  make -j$NPROC
+  make install
+
+  ctest -L unittest
+  popd
+}
+
+# Build and install Parquet master so we can test the Python bindings
+
+install_parquet_cpp() {
+  git clone git@github.com:apache/parquet-cpp.git
+
+  mkdir parquet-cpp/build
+  pushd parquet-cpp/build
+
+  cmake .. -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+        -DCMAKE_BUILD_TYPE=release \
+        -DPARQUET_BUILD_TESTS=off \
+        ..
+
+  make -j$NPROC
+  make install
+
+  popd
+}
+
+# Build and test Python
+
+test_python() {
+  pushd python
+
+  pip install -r requirements.txt
+
+  python setup.py build_ext --inplace --with-parquet --with-plasma
+  py.test pyarrow -v --pdb
+
+  popd
+}
+
+
+test_glib() {
+  # Build and test GLib, requires newer GLib (I used 2.52.3), so install that
+  # here
+  GLIB_VERSION=glib-2.53.7
+  GLIB_URL=https://gensho.ftp.acc.umu.se/pub/gnome/sources/glib/2.53/$GLIB_VERSION.tar.xz
+  curl -f -O $GLIB_URL
+  tar xf $GLIB_VERSION.tar.xz
+  pushd $GLIB_VERSION
+
+  ./configure --prefix=$ARROW_HOME
+  make -j$NPROC
+  make install
+
+  popd
+
+  pushd c_glib
+
+  ./configure --prefix=$ARROW_HOME
+  make -j$NPROC
+  make install
+
+  NO_MAKE=yes test/run-test.sh
+
+  popd
+}
+
+# Build and test Java (Requires newer Maven -- I used 3.3.9)
+
+test_package_java() {
+  pushd java
+
+  mvn test
+  mvn package
+
+  popd
+}
+
+# Run integration tests
+test_integration() {
+  pushd integration
+
+  JAVA_DIR=$SOURCE_DIR/java
+  CPP_BUILD_DIR=$SOURCE_DIR/cpp/build
+
+  export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar
+  export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/release
+
+  python integration_test.py
+
+  popd
+}
+
+setup_tempdir "arrow-$VERSION"
+echo "Working in sandbox $TMPDIR"
+cd $TMPDIR
+
+export SOURCE_DIR=`pwd`
+export ARROW_HOME=$TMPDIR/install
+export PARQUET_HOME=$TMPDIR/install
+export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH
+export PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig:$PKG_CONFIG_PATH
+
+NPROC=$(nproc)
+VERSION=$1
+RC_NUMBER=$2
+
+TARBALL=apache-arrow-$1.tar.gz
+
+import_gpg_keys
+
+DIST_NAME="apache-arrow-${VERSION}"
+fetch_archive $DIST_NAME
+tar xvzf ${DIST_NAME}.tar.gz
+cd ${DIST_NAME}
+
+setup_miniconda
+
+test_and_install_cpp
+
+# install_parquet_cpp
+# test_python
+
+test_glib
+
+test_package_java
+
+test_integration
+
+echo 'Release candidate looks good!'
+exit 0

From adb3146e8932baa8007d03680693536fdf1091a7 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Thu, 14 Sep 2017 23:33:05 -0400
Subject: [PATCH 2/6] More work on release verification script

Change-Id: I827922474a9180396079e1b377f52df683fc4696
---
 dev/release/verify-release-candidate.sh | 27 ++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 13db6286d39..2b1391e5f83 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -20,9 +20,13 @@
 
 # Requirements
 # - Ruby 2.x
+#   - Plus gem dependencies, see c_glib/README
 # - Maven >= 3.3.9
 # - JDK >=7
 # - gcc >= 4.8
+# - nodejs >= 6.0.0 (best way is to use nvm)
+#
+# BOOST_ROOT set to a Boost install that permits static linking
 
 case $# in
   2) VERSION="$1"
@@ -96,9 +100,10 @@ test_and_install_cpp() {
   mkdir cpp/build
   pushd cpp/build
 
-  cmake .. -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+  cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
         -DARROW_PLASMA=on \
         -DARROW_PYTHON=on \
+        -DARROW_BOOST_USE_SHARED=off \
         -DCMAKE_BUILD_TYPE=release \
         -DARROW_BUILD_BENCHMARKS=on \
         ..
@@ -120,6 +125,7 @@ install_parquet_cpp() {
 
   cmake .. -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
         -DCMAKE_BUILD_TYPE=release \
+        -DPARQUET_BOOST_USE_SHARED=off \
         -DPARQUET_BUILD_TESTS=off \
         ..
 
@@ -152,7 +158,7 @@ test_glib() {
   tar xf $GLIB_VERSION.tar.xz
   pushd $GLIB_VERSION
 
-  ./configure --prefix=$ARROW_HOME
+  ./configure --disable-libelf --enable-libmount=no --prefix=$ARROW_HOME
   make -j$NPROC
   make install
 
@@ -169,6 +175,13 @@ test_glib() {
   popd
 }
 
+test_js() {
+  pushd js
+  npm install
+  npm run validate
+  popd
+}
+
 # Build and test Java (Requires newer Maven -- I used 3.3.9)
 
 test_package_java() {
@@ -219,17 +232,13 @@ tar xvzf ${DIST_NAME}.tar.gz
 cd ${DIST_NAME}
 
 setup_miniconda
-
 test_and_install_cpp
-
-# install_parquet_cpp
-# test_python
-
+install_parquet_cpp
+test_python
 test_glib
-
 test_package_java
-
 test_integration
+test_js
 
 echo 'Release candidate looks good!'
 exit 0

From 17f7ac0f4a8bafc092818f55284ac4510645bba4 Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 15 Sep 2017 00:04:06 -0400
Subject: [PATCH 3/6] More fixes, finally works

Change-Id: I0d26139a7fdf7ebc62ff2937509209fec2e9d5d3
---
 dev/release/verify-release-candidate.sh | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index 2b1391e5f83..bda78c74beb 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -195,14 +195,14 @@ test_package_java() {
 
 # Run integration tests
 test_integration() {
-  pushd integration
-
-  JAVA_DIR=$SOURCE_DIR/java
-  CPP_BUILD_DIR=$SOURCE_DIR/cpp/build
+  JAVA_DIR=`pwd`/java
+  CPP_BUILD_DIR=`pwd`/cpp/build
 
   export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar
   export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/release
 
+  pushd integration
+
   python integration_test.py
 
   popd
@@ -212,7 +212,6 @@ setup_tempdir "arrow-$VERSION"
 echo "Working in sandbox $TMPDIR"
 cd $TMPDIR
 
-export SOURCE_DIR=`pwd`
 export ARROW_HOME=$TMPDIR/install
 export PARQUET_HOME=$TMPDIR/install
 export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH

From 079b5e47cc4ff915b263ed117150d022174773df Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 15 Sep 2017 09:14:15 -0400
Subject: [PATCH 4/6] Fix comments

Change-Id: Id7278ede0ebb0da91060ed769309bc4addd469e7
---
 dev/release/verify-release-candidate.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index bda78c74beb..b2e98ebba2d 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -123,7 +123,7 @@ install_parquet_cpp() {
   mkdir parquet-cpp/build
   pushd parquet-cpp/build
 
-  cmake .. -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+  cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
         -DCMAKE_BUILD_TYPE=release \
         -DPARQUET_BOOST_USE_SHARED=off \
         -DPARQUET_BUILD_TESTS=off \
@@ -150,7 +150,7 @@ test_python() {
 
 
 test_glib() {
-  # Build and test GLib, requires newer GLib (I used 2.52.3), so install that
+  # Build and test GLib, requires GLib >= 2.44 , so install that
   # here
   GLIB_VERSION=glib-2.53.7
   GLIB_URL=https://gensho.ftp.acc.umu.se/pub/gnome/sources/glib/2.53/$GLIB_VERSION.tar.xz

From 35319275a86f904f3bc9b978f8df1ff3eeafe2bf Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 15 Sep 2017 09:20:46 -0400
Subject: [PATCH 5/6] Add note to dev/README.md

Change-Id: I81be820d1c345ba364c2e5fb9a4ed2f24f1f15a0
---
 dev/README.md | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/dev/README.md b/dev/README.md
index e986abef191..91d688dfee7 100644
--- a/dev/README.md
+++ b/dev/README.md
@@ -92,3 +92,21 @@ Merge hash: 485658a5
 Would you like to pick 485658a5 into another branch? (y/n):
 ```
 For now just say n as we have 1 branch
+
+## Verifying Release Candidates
+
+We have provided a script to assist with verifying release candidates:
+
+```shell
+bash dev/release/verify-release-candidate.sh 0.7.0 0
+```
+
+Currently this only works on Linux (patches to expand to macOS welcome!). Read
+the script for information about system dependencies.
+
+On Windows, we have a script that verifies C++ and Python (requires Visual
+Studio 2015):
+
+```
+dev/release/verify-release-candidate.bat apache-arrow-0.7.0.tar.gz
+```
\ No newline at end of file

From 8fd6530898b74ed3657a3d8283ec77332e2caffc Mon Sep 17 00:00:00 2001
From: Wes McKinney <wes.mckinney@twosigma.com>
Date: Fri, 15 Sep 2017 10:10:34 -0400
Subject: [PATCH 6/6] Use Boost shared libraries

Change-Id: Id840b188db8d6b9d96ef1feacb3918302574af7c
---
 dev/release/verify-release-candidate.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index b2e98ebba2d..38680f429ed 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -26,7 +26,8 @@
 # - gcc >= 4.8
 # - nodejs >= 6.0.0 (best way is to use nvm)
 #
-# BOOST_ROOT set to a Boost install that permits static linking
+# If using a non-system Boost, set BOOST_ROOT and add Boost libraries to
+# LD_LIBRARY_PATH
 
 case $# in
   2) VERSION="$1"
@@ -103,7 +104,7 @@ test_and_install_cpp() {
   cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
         -DARROW_PLASMA=on \
         -DARROW_PYTHON=on \
-        -DARROW_BOOST_USE_SHARED=off \
+        -DARROW_BOOST_USE_SHARED=on \
         -DCMAKE_BUILD_TYPE=release \
         -DARROW_BUILD_BENCHMARKS=on \
         ..
@@ -125,7 +126,7 @@ install_parquet_cpp() {
 
   cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
         -DCMAKE_BUILD_TYPE=release \
-        -DPARQUET_BOOST_USE_SHARED=off \
+        -DPARQUET_BOOST_USE_SHARED=on \
         -DPARQUET_BUILD_TESTS=off \
         ..