ARROW-4649: [C++/CI/R] Add nightly job that tests the homebrew formula

LICENSE.txt

@@ -1838,9 +1838,39 @@ for PyArrow. Ibis is released under the Apache License, Version 2.0.
 
 This project includes code from the autobrew project.
 
-* r/tools/autobrew and r/tools/apache-arrow.rb are based on code
-  from the autobrew project.
+* r/tools/autobrew and dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
+  are based on code from the autobrew project.
 
 Copyright: Copyright (c) 2017 - 2019, Jeroen Ooms.
 All rights reserved.
 Homepage: https://github.com/jeroen/autobrew
+
+--------------------------------------------------------------------------------
+
+dev/tasks/homebrew-formulae/apache-arrow.rb has the following license:
+
+BSD 2-Clause License
+
+Copyright (c) 2009-present, Homebrew contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

dev/release/00-prepare-test.rb

@@ -106,6 +106,20 @@ def test_update_version_pre_tag
                         "+    <Version>#{@release_version}</Version>"],
                      ],
                    },
+                   {
+                     path: "dev/tasks/homebrew-formulae/apache-arrow.rb",
+                     hunks: [
+                       ["-  url \"https://www.apache.org/dyn/closer.cgi?path=arrow/arrow-#{@snapshot_version}/apache-arrow-#{@snapshot_version}.tar.gz\"",
+                        "+  url \"https://www.apache.org/dyn/closer.cgi?path=arrow/arrow-#{@release_version}/apache-arrow-#{@release_version}.tar.gz\""],
+                     ],
+                   },
+                   {
+                     path: "dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb",
+                     hunks: [
+                       ["-  url \"https://www.apache.org/dyn/closer.cgi?path=arrow/arrow-#{@previous_version}.9000/apache-arrow-#{@previous_version}.9000.tar.gz\"",
+                        "+  url \"https://www.apache.org/dyn/closer.cgi?path=arrow/arrow-#{@release_version}/apache-arrow-#{@release_version}.tar.gz\""],
+                     ],
+                   },
                    {
                      path: "js/package.json",
                      hunks: [
@@ -262,6 +276,20 @@ def test_update_version_post_tag
                         "+    <Version>#{@next_snapshot_version}</Version>"],
                      ],
                    },
+                   {
+                     path: "dev/tasks/homebrew-formulae/apache-arrow.rb",
+                     hunks: [
+                       ["-  url \"https://www.apache.org/dyn/closer.cgi?path=arrow/arrow-#{@release_version}/apache-arrow-#{@release_version}.tar.gz\"",
+                        "+  url \"https://www.apache.org/dyn/closer.cgi?path=arrow/arrow-#{@next_snapshot_version}/apache-arrow-#{@next_snapshot_version}.tar.gz\""],
+                     ],
+                   },
+                   {
+                     path: "dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb",
+                     hunks: [
+                       ["-  url \"https://www.apache.org/dyn/closer.cgi?path=arrow/arrow-#{@release_version}/apache-arrow-#{@release_version}.tar.gz\"",
+                        "+  url \"https://www.apache.org/dyn/closer.cgi?path=arrow/arrow-#{@release_version}.9000/apache-arrow-#{@release_version}.9000.tar.gz\""],
+                     ],
+                   },
                    {
                      path: "js/package.json",
                      hunks: [

dev/release/00-prepare.sh

@@ -42,14 +42,6 @@ update_versions() {
       ;;
   esac
 
-  cd "${SOURCE_DIR}/../../cpp"
-  sed -i.bak -E -e \
-    "s/^set\(ARROW_VERSION \".+\"\)/set(ARROW_VERSION \"${version}\")/" \
-    CMakeLists.txt
-  rm -f CMakeLists.txt.bak
-  git add CMakeLists.txt
-  cd -
-
   cd "${SOURCE_DIR}/../../c_glib"
   sed -i.bak -E -e \
     "s/^m4_define\(\[arrow_glib_version\], .+\)/m4_define([arrow_glib_version], ${version})/" \
@@ -61,6 +53,22 @@ update_versions() {
   git add configure.ac meson.build
   cd -
 
+  cd "${SOURCE_DIR}/../../ci"
+  sed -i.bak -E -e \
+    "s/^pkgver=.+/pkgver=${r_version}/" \
+    PKGBUILD
+  rm -f PKGBUILD.bak
+  git add PKGBUILD
+  cd -
+
+  cd "${SOURCE_DIR}/../../cpp"
+  sed -i.bak -E -e \
+    "s/^set\(ARROW_VERSION \".+\"\)/set(ARROW_VERSION \"${version}\")/" \
+    CMakeLists.txt
+  rm -f CMakeLists.txt.bak
+  git add CMakeLists.txt
+  cd -
+
   cd "${SOURCE_DIR}/../../csharp"
   sed -i.bak -E -e \
     "s/^    <Version>.+<\/Version>/    <Version>${version}<\/Version>/" \
@@ -69,6 +77,19 @@ update_versions() {
   git add Directory.Build.props
   cd -
 
+  cd "${SOURCE_DIR}/../../dev/tasks/homebrew-formulae"
+  sed -i.bak -E -e \
+    "s/arrow-[0-9.]+[0-9]+/arrow-${r_version}/g" \
+    autobrew/apache-arrow.rb
+  rm -f autobrew/apache-arrow.rb.bak
+  git add autobrew/apache-arrow.rb
+  sed -i.bak -E -e \
+    "s/arrow-[0-9.\-]+[0-9SNAPHOT]+/arrow-${version}/g" \
+    apache-arrow.rb
+  rm -f apache-arrow.rb.bak
+  git add apache-arrow.rb
+  cd -
+
   cd "${SOURCE_DIR}/../../js"
   sed -i.bak -E -e \
     "s/^  \"version\": \".+\"/  \"version\": \"${version}\"/" \
@@ -99,17 +120,6 @@ update_versions() {
     DESCRIPTION
   rm -f DESCRIPTION.bak
   git add DESCRIPTION
-  cd -
-
-  cd "${SOURCE_DIR}/../../ci"
-  sed -i.bak -E -e \
-    "s/^pkgver=.+/pkgver=${r_version}/" \
-    PKGBUILD
-  rm -f PKGBUILD.bak
-  git add PKGBUILD
-  cd -
-
-  cd "${SOURCE_DIR}/../../r"
   if [ ${type} = "snapshot" ]; then
     # Add a news entry for the new dev version
     echo "dev"

dev/release/rat_exclude_files.txt

@@ -32,6 +32,7 @@ cpp/src/plasma/thirdparty/ae/config.h
 cpp/src/plasma/thirdparty/ae/zmalloc.h
 cpp/src/plasma/thirdparty/dlmalloc.c
 dev/release/rat_exclude_files.txt
+dev/tasks/homebrew-formulae/apache-arrow.rb
 dev/tasks/linux-packages/debian.ubuntu-xenial/compat
 dev/tasks/linux-packages/debian.ubuntu-xenial/control
 dev/tasks/linux-packages/debian.ubuntu-xenial/gir1.2-arrow-1.0.install

dev/tasks/homebrew-formulae/apache-arrow.rb

@@ -0,0 +1,57 @@
+class ApacheArrow < Formula
+  desc "Columnar in-memory analytics layer designed to accelerate big data"
+  homepage "https://arrow.apache.org/"
+  url "https://www.apache.org/dyn/closer.cgi?path=arrow/arrow-0.15.0-SNAPSHOT/apache-arrow-0.15.0-SNAPSHOT.tar.gz"
+  sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28"
+  head "https://github.com/apache/arrow.git"
+
+  depends_on "cmake" => :build
+  depends_on "boost"
+  depends_on "brotli"
+  depends_on "double-conversion"
+  depends_on "flatbuffers"
+  depends_on "glog"
+  depends_on "grpc"
+  depends_on "lz4"
+  depends_on "numpy"
+  depends_on "openssl"
+  depends_on "protobuf"
+  depends_on "python"
+  depends_on "rapidjson"
+  depends_on "snappy"
+  depends_on "thrift"
+  depends_on "zstd"
+
+  def install
+    ENV.cxx11
+    args = %W[
+      -DARROW_FLIGHT=ON
+      -DARROW_ORC=ON
+      -DARROW_PARQUET=ON
+      -DARROW_PLASMA=ON
+      -DARROW_PROTOBUF_USE_SHARED=ON
+      -DARROW_PYTHON=ON
+      -DARROW_INSTALL_NAME_RPATH=OFF
+      -DPYTHON_EXECUTABLE=#{Formula["python"].bin/"python3"}
+    ]
+
+    mkdir "build"
+    cd "build" do
+      system "cmake", "../cpp", *std_cmake_args, *args
+      system "make"
+      system "make", "install"
+    end
+  end
+
+  test do
+    (testpath/"test.cpp").write <<~EOS
+      #include "arrow/api.h"
+      int main(void) {
+        arrow::int64();
+        return 0;
+      }
+    EOS
+    system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", "-larrow", "-o", "test"
+    system "./test"
+  end
+end

r/tools/apache-arrow.rb → dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb

@@ -19,7 +19,7 @@
 class ApacheArrow < Formula
   desc "Columnar in-memory analytics layer designed to accelerate big data"
   homepage "https://arrow.apache.org/"
-  url "https://archive.apache.org/dist/arrow/arrow-0.14.1/apache-arrow-0.14.1.tar.gz"
+  url "https://www.apache.org/dyn/closer.cgi?path=arrow/arrow-0.14.1.9000/apache-arrow-0.14.1.9000.tar.gz"
   sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28"
   head "https://github.com/apache/arrow.git"
 
@@ -31,11 +31,11 @@ class ApacheArrow < Formula
 
   depends_on "cmake" => :build
   depends_on "flatbuffers" => :build
-  depends_on "double-conversion"
   depends_on "boost"
+  depends_on "double-conversion"
   depends_on "lz4"
-  depends_on "thrift"
   depends_on "snappy"
+  depends_on "thrift"
 
   def install
     ENV.cxx11

dev/tasks/homebrew-formulae/travis.osx.yml

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+os: osx
+language: ruby
+cache: bundler
+
+# don't build twice
+if: tag IS blank
+
+env:
+  global:
+    - TRAVIS_TAG={{ task.tag }}
+    - ARROW_FORMULA=./arrow/dev/tasks/homebrew-formulae/{{ formula }}
+
+before_script:
+- git clone --no-checkout {{ arrow.remote }} arrow
+- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+- if [ $CROSSBOW_USE_COMMIT_ID == true ]; then git -C arrow checkout {{ arrow.head }}; else git -C arrow checkout FETCH_HEAD; fi
+- brew update
+- brew audit $ARROW_FORMULA
+script:
+- brew install -v --build-from-source --HEAD $ARROW_FORMULA
+- brew test $ARROW_FORMULA
+
+notifications:
+  email:
+    - {{ job.email }}

dev/tasks/tasks.yml

@@ -161,6 +161,8 @@ groups:
     - ubuntu-disco-arm64
     - centos-6
     - centos-7
+    - homebrew-cpp
+    - homebrew-cpp-autobrew
     - gandiva-jar-trusty
     - gandiva-jar-osx
     - docker-r
@@ -1085,6 +1087,22 @@ tasks:
       - plasma-glib-libs-{no_rc_version}-1.el7.x86_64.rpm
       - plasma-libs-{no_rc_version}-1.el7.x86_64.rpm
 
+  ############################## Homebrew Tasks ################################
+
+  homebrew-cpp:
+    ci: travis
+    platform: osx
+    template: homebrew-formulae/travis.osx.yml
+    params:
+      formula: apache-arrow.rb
+
+  homebrew-cpp-autobrew:
+    ci: travis
+    platform: osx
+    template: homebrew-formulae/travis.osx.yml
+    params:
+      formula: autobrew/apache-arrow.rb
+
   ############################## Gandiva Tasks ################################
 
   gandiva-jar-trusty:

r/configure

@@ -40,10 +40,12 @@ if [ "$ARROW_R_DEV" == "TRUE" ]; then
 fi
 
 if [ "$LOCAL_AUTOBREW" == "TRUE" ]; then
-  # LOCAL_AUTOBREW means use the formula in tools/
+  # LOCAL_AUTOBREW means use the script in tools/
+  # If you want to use a local apache-arrow.rb formula, do:
+  # $ cp ../dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb apache-arrow.rb
+  # (assuming a local checkout of the apache/arrow repository)
   # FORCE_AUTOBREW without LOCAL_AUTOBREW means to pull from jeroen.github.io
   cp tools/autobrew .
-  cp tools/apache-arrow.rb .
   FORCE_AUTOBREW="TRUE"
 fi