From 14bce9cbc579819db608d645dfd71916a634996c Mon Sep 17 00:00:00 2001 From: Javier Luraschi Date: Sat, 16 Mar 2019 00:20:34 +0000 Subject: [PATCH 01/17] [R] Progress towards completing windows support --- r/.gitignore | 2 +- r/src/Makevars.win | 20 ++++++++++++++++++++ r/tools/winlibs.R | 9 +++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 r/src/Makevars.win create mode 100644 r/tools/winlibs.R diff --git a/r/.gitignore b/r/.gitignore index 0f405f57136..9532537ff0d 100644 --- a/r/.gitignore +++ b/r/.gitignore @@ -10,4 +10,4 @@ inst/doc .Rproj.user .Rhistory src/Makevars - +windows/ diff --git a/r/src/Makevars.win b/r/src/Makevars.win new file mode 100644 index 00000000000..c8a927d83f9 --- /dev/null +++ b/r/src/Makevars.win @@ -0,0 +1,20 @@ +VERSION = 0.12.0.9000 +RWINLIB = ../windows/arrow-$(VERSION) +PKG_CPPFLAGS = -I$(RWINLIB)/include -DARROW_STATIC -DPARQUET_STATIC +CXX_STD = CXX11 + +PKG_LIBS = \ + -L$(RWINLIB)/lib$(subst gcc,,$(COMPILED_BY))$(R_ARCH) \ + -L$(RWINLIB)/lib$(R_ARCH) \ + -lparquet -larrow -lthrift -lboost_regex-mt-s -ldouble-conversion -lWs2_32 + +#all: clean +all: $(SHLIB) + +$(OBJECTS): winlibs + +winlibs: + "${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" "../tools/winlibs.R" $(VERSION) + +clean: + rm -f $(SHLIB) $(OBJECTS) diff --git a/r/tools/winlibs.R b/r/tools/winlibs.R new file mode 100644 index 00000000000..e9bf112acf8 --- /dev/null +++ b/r/tools/winlibs.R @@ -0,0 +1,9 @@ +# Download static arrow from rwinlib +VERSION <- commandArgs(TRUE) +if(!file.exists(sprintf("../windows/arrow-%s/include/arrow/api.h", VERSION))){ + if(getRversion() < "3.3.0") setInternet2() + download.file(sprintf("https://github.com/rwinlib/arrow/archive/v%s.zip", VERSION), "lib.zip", quiet = TRUE) + dir.create("../windows", showWarnings = FALSE) + unzip("lib.zip", exdir = "../windows") + unlink("lib.zip") +} From 9dcc347bf88241103358ebc4b0e0b4dbca40c187 Mon Sep 17 00:00:00 2001 From: Javier Luraschi Date: Fri, 22 Mar 2019 12:53:40 -0700 Subject: [PATCH 02/17] fix apache rat warnings --- r/src/Makevars.win | 17 +++++++++++++++++ r/tools/winlibs.R | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/r/src/Makevars.win b/r/src/Makevars.win index c8a927d83f9..bcb92cdf4ec 100644 --- a/r/src/Makevars.win +++ b/r/src/Makevars.win @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + VERSION = 0.12.0.9000 RWINLIB = ../windows/arrow-$(VERSION) PKG_CPPFLAGS = -I$(RWINLIB)/include -DARROW_STATIC -DPARQUET_STATIC diff --git a/r/tools/winlibs.R b/r/tools/winlibs.R index e9bf112acf8..12391c77d63 100644 --- a/r/tools/winlibs.R +++ b/r/tools/winlibs.R @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Download static arrow from rwinlib VERSION <- commandArgs(TRUE) if(!file.exists(sprintf("../windows/arrow-%s/include/arrow/api.h", VERSION))){ From aa91e2b531b2dfa3db68ea05e77698b3bfb6307e Mon Sep 17 00:00:00 2001 From: Javier Luraschi Date: Fri, 22 Mar 2019 14:58:20 -0700 Subject: [PATCH 03/17] modivy R's makevars.win file on release script --- dev/release/00-prepare.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dev/release/00-prepare.sh b/dev/release/00-prepare.sh index 8596892de50..293811d330a 100755 --- a/dev/release/00-prepare.sh +++ b/dev/release/00-prepare.sh @@ -89,6 +89,14 @@ update_versions() { git add DESCRIPTION cd - + cd "${SOURCE_DIR}/../../r/src" + sed -i.bak -E -e \ + "s/^VERSION = .+/VERSION = ${r_version}/" \ + Makevars.win + rm -f Makevars.win.bak + git add Makevars.win + cd - + cd "${SOURCE_DIR}/../../ruby" sed -i.bak -E -e \ "s/^ VERSION = \".+\"/ VERSION = \"${version}\"/g" \ From ee21495dfd1dd54a4af12afd9d403c8a14b5e4f0 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Thu, 28 Mar 2019 15:03:18 +0100 Subject: [PATCH 04/17] using R_xlen_t instead of size_t. causes trouble in win32 --- r/src/array__to_vector.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/r/src/array__to_vector.cpp b/r/src/array__to_vector.cpp index f4017c0ab5f..219b19d1a50 100644 --- a/r/src/array__to_vector.cpp +++ b/r/src/array__to_vector.cpp @@ -113,7 +113,7 @@ Status SomeNull_Ingest(SEXP data, R_xlen_t start, R_xlen_t n, if (array->null_count()) { arrow::internal::BitmapReader bitmap_reader(array->null_bitmap()->data(), array->offset(), n); - for (size_t i = 0; i < n; i++, bitmap_reader.Next(), ++p_data, ++p_values) { + for (R_xlen_t i = 0; i < n; i++, bitmap_reader.Next(), ++p_data, ++p_values) { *p_data = bitmap_reader.IsSet() ? lambda(*p_values) : default_value(); } } else { @@ -251,11 +251,11 @@ class Converter_Boolean : public Converter { arrow::internal::BitmapReader null_reader(array->null_bitmap()->data(), array->offset(), n); - for (size_t i = 0; i < n; i++, data_reader.Next(), null_reader.Next(), ++p_data) { + for (R_xlen_t i = 0; i < n; i++, data_reader.Next(), null_reader.Next(), ++p_data) { *p_data = null_reader.IsSet() ? data_reader.IsSet() : NA_LOGICAL; } } else { - for (size_t i = 0; i < n; i++, data_reader.Next(), ++p_data) { + for (R_xlen_t i = 0; i < n; i++, data_reader.Next(), ++p_data) { *p_data = data_reader.IsSet(); } } @@ -472,12 +472,12 @@ class Converter_Decimal : public Converter { internal::BitmapReader bitmap_reader(array->null_bitmap()->data(), array->offset(), n); - for (size_t i = 0; i < n; i++, bitmap_reader.Next(), ++p_data) { + for (R_xlen_t i = 0; i < n; i++, bitmap_reader.Next(), ++p_data) { *p_data = bitmap_reader.IsSet() ? std::stod(decimals_arr.FormatValue(i).c_str()) : NA_REAL; } } else { - for (size_t i = 0; i < n; i++, ++p_data) { + for (R_xlen_t i = 0; i < n; i++, ++p_data) { *p_data = std::stod(decimals_arr.FormatValue(i).c_str()); } } @@ -514,7 +514,7 @@ class Converter_Int64 : public Converter { if (array->null_count()) { internal::BitmapReader bitmap_reader(array->null_bitmap()->data(), array->offset(), n); - for (size_t i = 0; i < n; i++, bitmap_reader.Next(), ++p_data) { + for (R_xlen_t i = 0; i < n; i++, bitmap_reader.Next(), ++p_data) { *p_data = bitmap_reader.IsSet() ? p_values[i] : NA_INT64; } } else { From 169918fff536c52e399f20f65316a513ee88b902 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Thu, 28 Mar 2019 16:10:35 +0100 Subject: [PATCH 05/17] split int_cast in versions for signed and unsigned --- r/src/array_from_vector.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/r/src/array_from_vector.cpp b/r/src/array_from_vector.cpp index 83d9f78ae5d..82a283abf05 100644 --- a/r/src/array_from_vector.cpp +++ b/r/src/array_from_vector.cpp @@ -182,7 +182,7 @@ using internal::checked_cast; namespace internal { -template +template ::value, Target>::type = 0> Status int_cast(T x, Target* out) { if (x < std::numeric_limits::min() || x > std::numeric_limits::max()) { return Status::Invalid("Value is too large to fit in C integer type"); @@ -191,6 +191,15 @@ Status int_cast(T x, Target* out) { return Status::OK(); } +template ::value, Target>::type = 0> +Status int_cast(T x, Target* out) { + if (x < 0 || x > std::numeric_limits::max()) { + return Status::Invalid("Value is too large to fit in C integer type"); + } + *out = static_cast(x); + return Status::OK(); +} + template Status double_cast(Int x, double* out) { *out = static_cast(x); From d1ba6dd04b77fff7f5ca670c3f57d17af4534e32 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Thu, 28 Mar 2019 16:39:41 +0100 Subject: [PATCH 06/17] initialize value. --- r/src/array_from_vector.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/src/array_from_vector.cpp b/r/src/array_from_vector.cpp index 82a283abf05..1270299f39b 100644 --- a/r/src/array_from_vector.cpp +++ b/r/src/array_from_vector.cpp @@ -306,7 +306,7 @@ struct Unbox> { if (*p == na) { builder->UnsafeAppendNull(); } else { - CType value; + CType value = 0; RETURN_NOT_OK(internal::int_cast(*p, &value)); builder->UnsafeAppend(value); } @@ -384,7 +384,7 @@ struct Unbox { if (*p == NA_INTEGER) { builder->UnsafeAppendNull(); } else { - float value; + float value = 0; RETURN_NOT_OK(internal::float_cast(*p, &value)); builder->UnsafeAppend(value); } From e2e451e8a1646d537c2f134ae5bd86dcdd8ae8cf Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Thu, 28 Mar 2019 17:41:52 +0100 Subject: [PATCH 07/17] usigned comparison --- r/src/array_from_vector.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/r/src/array_from_vector.cpp b/r/src/array_from_vector.cpp index 1270299f39b..23316a81414 100644 --- a/r/src/array_from_vector.cpp +++ b/r/src/array_from_vector.cpp @@ -191,9 +191,14 @@ Status int_cast(T x, Target* out) { return Status::OK(); } +template +struct usigned_type; + template ::value, Target>::type = 0> Status int_cast(T x, Target* out) { - if (x < 0 || x > std::numeric_limits::max()) { + // we need to compare between unsigned integers + uint64_t x64 = x; + if (x64 < 0 || x64 > std::numeric_limits::max()) { return Status::Invalid("Value is too large to fit in C integer type"); } *out = static_cast(x); From 59025b6fd09f087ecdb31d341a07e2decf6e7286 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Fri, 29 Mar 2019 09:38:30 +0100 Subject: [PATCH 08/17] use int64_t instead of size_t --- r/src/array.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/src/array.cpp b/r/src/array.cpp index 9ef8f5f182c..e4ad68d4b58 100644 --- a/r/src/array.cpp +++ b/r/src/array.cpp @@ -99,7 +99,7 @@ LogicalVector Array__Mask(const std::shared_ptr& array) { LogicalVector res(no_init(n)); arrow::internal::BitmapReader bitmap_reader(array->null_bitmap()->data(), array->offset(), n); - for (size_t i = 0; i < array->length(); i++, bitmap_reader.Next()) { + for (int64_t i = 0; i < n; i++, bitmap_reader.Next()) { res[i] = bitmap_reader.IsSet(); } return res; From 1d50c3032ff0c998a079815c63585c40846c93d8 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Fri, 29 Mar 2019 09:44:35 +0100 Subject: [PATCH 09/17] lint --- r/src/array_from_vector.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/r/src/array_from_vector.cpp b/r/src/array_from_vector.cpp index 23316a81414..ac72a4a8214 100644 --- a/r/src/array_from_vector.cpp +++ b/r/src/array_from_vector.cpp @@ -182,7 +182,8 @@ using internal::checked_cast; namespace internal { -template ::value, Target>::type = 0> +template ::value, Target>::type = 0> Status int_cast(T x, Target* out) { if (x < std::numeric_limits::min() || x > std::numeric_limits::max()) { return Status::Invalid("Value is too large to fit in C integer type"); @@ -194,7 +195,8 @@ Status int_cast(T x, Target* out) { template struct usigned_type; -template ::value, Target>::type = 0> +template ::value, Target>::type = 0> Status int_cast(T x, Target* out) { // we need to compare between unsigned integers uint64_t x64 = x; From 7927aa0b3c874a3e3dc99343110a318ef867a1e9 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Wed, 3 Apr 2019 09:36:57 +0200 Subject: [PATCH 10/17] add @jeroen as author and use 0.13.0 as the version to prepare for cran release --- r/DESCRIPTION | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/r/DESCRIPTION b/r/DESCRIPTION index 7ffa8cec191..99ddd645d1f 100644 --- a/r/DESCRIPTION +++ b/r/DESCRIPTION @@ -1,10 +1,11 @@ Package: arrow Title: R Integration to 'Apache' 'Arrow' -Version: 0.12.0.9000 +Version: 0.13.0 Authors@R: c( person("Romain", "François", email = "romain@rstudio.com", role = c("aut", "cre")), person("Javier", "Luraschi", email = "javier@rstudio.com", role = c("ctb")), person("Jeffrey", "Wong", email = "jeffreyw@netflix.com", role = c("ctb")), + person("Jeroen", "Ooms", email = "jeroen@berkeley.edu", role = c("aut")), person("Apache Arrow", email = "dev@arrow.apache.org", role = c("aut", "cph")) ) Description: R Integration to 'Apache' 'Arrow'. From b0fd448e90ec423ffbed7f8148fe43ee4d8a46ff Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Wed, 3 Apr 2019 09:43:34 +0200 Subject: [PATCH 11/17] Remove incorrect stuff from configure/make setup, from @jeroen. use version 0.13.0 of C++ arrow lib on windows, enable parquet --- r/configure | 22 +++++----------------- r/src/Makevars.in | 5 ++--- r/src/Makevars.win | 21 ++++++--------------- 3 files changed, 13 insertions(+), 35 deletions(-) diff --git a/r/configure b/r/configure index a3bc690b39a..e5d4de2a177 100755 --- a/r/configure +++ b/r/configure @@ -32,24 +32,13 @@ PKG_RPM_NAME="arrow" PKG_CSW_NAME="arrow" PKG_BREW_NAME="apache-arrow" PKG_TEST_HEADER="" -PKG_LIBS="" +PKG_LIBS="-larrow -lparquet" # Use pkg-config if available pkg-config --version >/dev/null 2>&1 if [ $? -eq 0 ]; then - PKGCONFIG_CFLAGS=$(pkg-config --cflags --silence-errors arrow) - if [ $? -ne 0 ]; then - echo "Apache Arrow C++ was not found using pkg-config" - exit 1 - fi - PKGCONFIG_LIBS=$(pkg-config --libs arrow) - PKGCONFIG_CFLAGS_PARQUET=$(pkg-config --cflags --silence-errors parquet) - if [ $? -eq 0 ]; then - PKGCONFIG_CFLAGS="${PKGCONFIG_CFLAGS} ${PKGCONFIG_CFLAGS_PARQUET} -DARROW_R_WITH_PARQUET" - PKGCONFIG_LIBS="${PKGCONFIG_LIBS} $(pkg-config --libs parquet)" - fi -else - PKG_LIBS="-larrow -lparquet" + PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}` + PKGCONFIG_LIBS=`pkg-config --libs ${PKG_CONFIG_NAME}` fi # Note that cflags may be empty in case of success @@ -83,9 +72,8 @@ CXX11STD=$("${R_HOME}"/bin/R CMD config CXX11STD) CPPFLAGS=$("${R_HOME}"/bin/R CMD config CPPFLAGS) # If libarrow uses the old GLIBCXX ABI, so we have to use it too -PKG_CXXFLAGS="$C_VISIBILITY" if [ "$ARROW_USE_OLD_CXXABI" ]; then - PKG_CXXFLAGS="$PKG_CXXFLAGS -D_GLIBCXX_USE_CXX11_ABI=0" + $PKG_CFLAGS="$PKG_CFLAGS -D_GLIBCXX_USE_CXX11_ABI=0" fi # Test configuration @@ -108,7 +96,7 @@ if [ $? -ne 0 ]; then fi # Write to Makevars -sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" -e "s|@pkgcxxflags@|$PKG_CXXFLAGS|" src/Makevars.in > src/Makevars +sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" src/Makevars.in > src/Makevars # Success exit 0 diff --git a/r/src/Makevars.in b/r/src/Makevars.in index 0d2808736c0..cb8576a19eb 100644 --- a/r/src/Makevars.in +++ b/r/src/Makevars.in @@ -16,7 +16,6 @@ # under the License. PKG_CPPFLAGS=@cflags@ -PKG_CXXFLAGS=@pkgcxxflags@ +PKG_CXXFLAGS=$(CXX_VISIBILITY) CXX_STD=CXX11 -PKG_LIBS=@libs@ -Wl,-rpath,/usr/local/lib -#CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" +PKG_LIBS=@libs@ diff --git a/r/src/Makevars.win b/r/src/Makevars.win index 8a9664a22e0..840dd1f63fe 100644 --- a/r/src/Makevars.win +++ b/r/src/Makevars.win @@ -15,24 +15,15 @@ # specific language governing permissions and limitations # under the License. -VERSION = 0.12.0.9000 - -ifeq "$(ARROW_PATH)" "" - RWINLIB = ../windows/arrow-$(VERSION) - ARROW_INCLUDE = $(RWINLIB)/include - ARROW_LIBS = $(RWINLIB)/lib$(subst gcc,,$(COMPILED_BY))$(R_ARCH) - OTHER_LIBS = -L$(RWINLIB)/lib$(R_ARCH) -else - ARROW_INCLUDE = $(ARROW_PATH)/include - ARROW_LIBS = $(ARROW_PATH)/lib -endif - -PKG_CPPFLAGS = -I$(ARROW_INCLUDE) -DARROW_STATIC -DPARQUET_STATIC +VERSION = 0.13.0 +RWINLIB = ../windows/arrow-$(VERSION) +PKG_CPPFLAGS = -I$(RWINLIB)/include -DARROW_STATIC -DPARQUET_STATIC \ + -DARROW_R_WITH_PARQUET CXX_STD = CXX11 PKG_LIBS = \ - -L$(ARROW_LIBS) \ - $(OTHER_LIBS) \ + -L$(RWINLIB)/lib$(subst gcc,,$(COMPILED_BY))$(R_ARCH) \ + -L$(RWINLIB)/lib$(R_ARCH) \ -lparquet -larrow -lthrift -lboost_regex-mt-s -ldouble-conversion -lz -lws2_32 #all: clean From 892af4d44d40db1d75889d8d250e07264304c710 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Wed, 3 Apr 2019 11:23:19 +0200 Subject: [PATCH 12/17] enable parquet on mac. from https://github.com/jeroen/arrow/commit/833a2dc7554b6dcee364ff39206ed9a7d9fe59e0 --- r/configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/configure b/r/configure index e5d4de2a177..147b35070dc 100755 --- a/r/configure +++ b/r/configure @@ -57,7 +57,7 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then curl -sfL "https://jeroen.github.io/autobrew/$PKG_BREW_NAME" > autobrew source autobrew fi - PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include" + PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include -DARROW_R_WITH_PARQUET" PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS" fi From 06858d0c47a4a6c76ef088f58250de9474e248b2 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Wed, 3 Apr 2019 12:17:55 +0200 Subject: [PATCH 13/17] revert configure and Makevars.in changes so that it works fine on mac os --- r/configure | 24 ++++++++++++++++++------ r/src/Makevars.in | 5 +++-- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/r/configure b/r/configure index 147b35070dc..a3bc690b39a 100755 --- a/r/configure +++ b/r/configure @@ -32,13 +32,24 @@ PKG_RPM_NAME="arrow" PKG_CSW_NAME="arrow" PKG_BREW_NAME="apache-arrow" PKG_TEST_HEADER="" -PKG_LIBS="-larrow -lparquet" +PKG_LIBS="" # Use pkg-config if available pkg-config --version >/dev/null 2>&1 if [ $? -eq 0 ]; then - PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}` - PKGCONFIG_LIBS=`pkg-config --libs ${PKG_CONFIG_NAME}` + PKGCONFIG_CFLAGS=$(pkg-config --cflags --silence-errors arrow) + if [ $? -ne 0 ]; then + echo "Apache Arrow C++ was not found using pkg-config" + exit 1 + fi + PKGCONFIG_LIBS=$(pkg-config --libs arrow) + PKGCONFIG_CFLAGS_PARQUET=$(pkg-config --cflags --silence-errors parquet) + if [ $? -eq 0 ]; then + PKGCONFIG_CFLAGS="${PKGCONFIG_CFLAGS} ${PKGCONFIG_CFLAGS_PARQUET} -DARROW_R_WITH_PARQUET" + PKGCONFIG_LIBS="${PKGCONFIG_LIBS} $(pkg-config --libs parquet)" + fi +else + PKG_LIBS="-larrow -lparquet" fi # Note that cflags may be empty in case of success @@ -57,7 +68,7 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then curl -sfL "https://jeroen.github.io/autobrew/$PKG_BREW_NAME" > autobrew source autobrew fi - PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include -DARROW_R_WITH_PARQUET" + PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include" PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS" fi @@ -72,8 +83,9 @@ CXX11STD=$("${R_HOME}"/bin/R CMD config CXX11STD) CPPFLAGS=$("${R_HOME}"/bin/R CMD config CPPFLAGS) # If libarrow uses the old GLIBCXX ABI, so we have to use it too +PKG_CXXFLAGS="$C_VISIBILITY" if [ "$ARROW_USE_OLD_CXXABI" ]; then - $PKG_CFLAGS="$PKG_CFLAGS -D_GLIBCXX_USE_CXX11_ABI=0" + PKG_CXXFLAGS="$PKG_CXXFLAGS -D_GLIBCXX_USE_CXX11_ABI=0" fi # Test configuration @@ -96,7 +108,7 @@ if [ $? -ne 0 ]; then fi # Write to Makevars -sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" src/Makevars.in > src/Makevars +sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" -e "s|@pkgcxxflags@|$PKG_CXXFLAGS|" src/Makevars.in > src/Makevars # Success exit 0 diff --git a/r/src/Makevars.in b/r/src/Makevars.in index cb8576a19eb..0d2808736c0 100644 --- a/r/src/Makevars.in +++ b/r/src/Makevars.in @@ -16,6 +16,7 @@ # under the License. PKG_CPPFLAGS=@cflags@ -PKG_CXXFLAGS=$(CXX_VISIBILITY) +PKG_CXXFLAGS=@pkgcxxflags@ CXX_STD=CXX11 -PKG_LIBS=@libs@ +PKG_LIBS=@libs@ -Wl,-rpath,/usr/local/lib +#CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" From 21e0d9dff7d108ca0b3eb58940a84f78f4134e22 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Wed, 3 Apr 2019 14:38:18 +0200 Subject: [PATCH 14/17] Revert "enable parquet on mac. from https://github.com/jeroen/arrow/commit/833a2dc7554b6dcee364ff39206ed9a7d9fe59e0" This reverts commit 892af4d44d40db1d75889d8d250e07264304c710. --- r/configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/configure b/r/configure index 147b35070dc..e5d4de2a177 100755 --- a/r/configure +++ b/r/configure @@ -57,7 +57,7 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then curl -sfL "https://jeroen.github.io/autobrew/$PKG_BREW_NAME" > autobrew source autobrew fi - PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include -DARROW_R_WITH_PARQUET" + PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include" PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS" fi From 30ab27980a9aac60d17bc9c8a19d15ca41e80a17 Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Wed, 3 Apr 2019 15:09:26 +0200 Subject: [PATCH 15/17] not yet supported --- r/configure | 3 --- 1 file changed, 3 deletions(-) diff --git a/r/configure b/r/configure index a3bc690b39a..61891006557 100755 --- a/r/configure +++ b/r/configure @@ -95,9 +95,6 @@ echo "#include $PKG_TEST_HEADER" | ${CXXCPP} ${CPPFLAGS} ${PKG_CFLAGS} ${CXX11FL if [ $? -ne 0 ]; then echo "------------------------- ANTICONF ERROR ---------------------------" echo "Configuration failed because $PKG_CONFIG_NAME was not found. Try installing:" - echo " * deb: $PKG_DEB_NAME (Debian, Ubuntu, etc)" - echo " * rpm: $PKG_RPM_NAME (Fedora, CentOS, RHEL)" - echo " * csw: $PKG_CSW_NAME (Solaris)" echo " * brew: $PKG_BREW_NAME (Mac OSX)" echo "If $PKG_CONFIG_NAME is already installed, check that 'pkg-config' is in your" echo "PATH and PKG_CONFIG_PATH contains a $PKG_CONFIG_NAME.pc file. If pkg-config" From a44d1f553a3d1716dfafa7bb02753b8c4ed13d3f Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Wed, 3 Apr 2019 16:37:11 +0200 Subject: [PATCH 16/17] using configure and Makevars.* from @jeroen --- r/configure | 29 ++++++++--------------------- r/src/Makevars.in | 5 ++--- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/r/configure b/r/configure index 61891006557..89442701614 100755 --- a/r/configure +++ b/r/configure @@ -27,29 +27,17 @@ # Library settings PKG_CONFIG_NAME="arrow parquet" -PKG_DEB_NAME="arrow" -PKG_RPM_NAME="arrow" -PKG_CSW_NAME="arrow" +PKG_DEB_NAME="(unsuppored)" +PKG_RPM_NAME="(unsuppored)" PKG_BREW_NAME="apache-arrow" PKG_TEST_HEADER="" -PKG_LIBS="" +PKG_LIBS="-larrow -lparquet" # Use pkg-config if available pkg-config --version >/dev/null 2>&1 if [ $? -eq 0 ]; then - PKGCONFIG_CFLAGS=$(pkg-config --cflags --silence-errors arrow) - if [ $? -ne 0 ]; then - echo "Apache Arrow C++ was not found using pkg-config" - exit 1 - fi - PKGCONFIG_LIBS=$(pkg-config --libs arrow) - PKGCONFIG_CFLAGS_PARQUET=$(pkg-config --cflags --silence-errors parquet) - if [ $? -eq 0 ]; then - PKGCONFIG_CFLAGS="${PKGCONFIG_CFLAGS} ${PKGCONFIG_CFLAGS_PARQUET} -DARROW_R_WITH_PARQUET" - PKGCONFIG_LIBS="${PKGCONFIG_LIBS} $(pkg-config --libs parquet)" - fi -else - PKG_LIBS="-larrow -lparquet" + PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}` + PKGCONFIG_LIBS=`pkg-config --libs ${PKG_CONFIG_NAME}` fi # Note that cflags may be empty in case of success @@ -68,7 +56,7 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then curl -sfL "https://jeroen.github.io/autobrew/$PKG_BREW_NAME" > autobrew source autobrew fi - PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include" + PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include -DARROW_R_WITH_PARQUET" PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS" fi @@ -83,9 +71,8 @@ CXX11STD=$("${R_HOME}"/bin/R CMD config CXX11STD) CPPFLAGS=$("${R_HOME}"/bin/R CMD config CPPFLAGS) # If libarrow uses the old GLIBCXX ABI, so we have to use it too -PKG_CXXFLAGS="$C_VISIBILITY" if [ "$ARROW_USE_OLD_CXXABI" ]; then - PKG_CXXFLAGS="$PKG_CXXFLAGS -D_GLIBCXX_USE_CXX11_ABI=0" + $PKG_CFLAGS="$PKG_CFLAGS -D_GLIBCXX_USE_CXX11_ABI=0" fi # Test configuration @@ -105,7 +92,7 @@ if [ $? -ne 0 ]; then fi # Write to Makevars -sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" -e "s|@pkgcxxflags@|$PKG_CXXFLAGS|" src/Makevars.in > src/Makevars +sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" src/Makevars.in > src/Makevars # Success exit 0 diff --git a/r/src/Makevars.in b/r/src/Makevars.in index 0d2808736c0..cb8576a19eb 100644 --- a/r/src/Makevars.in +++ b/r/src/Makevars.in @@ -16,7 +16,6 @@ # under the License. PKG_CPPFLAGS=@cflags@ -PKG_CXXFLAGS=@pkgcxxflags@ +PKG_CXXFLAGS=$(CXX_VISIBILITY) CXX_STD=CXX11 -PKG_LIBS=@libs@ -Wl,-rpath,/usr/local/lib -#CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" +PKG_LIBS=@libs@ From 5806d5ebcf2a4f98c302ed785741208f1842721a Mon Sep 17 00:00:00 2001 From: Romain Francois Date: Thu, 11 Apr 2019 11:28:06 +0200 Subject: [PATCH 17/17] cran comments --- dev/release/rat_exclude_files.txt | 1 + r/.Rbuildignore | 1 + r/cran-comments.md | 27 +++++++++++++++++++++++++++ 3 files changed, 29 insertions(+) create mode 100644 r/cran-comments.md diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index ff1d191e03c..46ddff37402 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -188,6 +188,7 @@ r/arrow.Rproj r/README.md r/README.Rmd r/man/*.Rd +r/cran-comments.md .gitattributes ruby/red-arrow/.yardopts rust/arrow/test/data/*.csv diff --git a/r/.Rbuildignore b/r/.Rbuildignore index b51d7bc46ab..9d39e74b519 100644 --- a/r/.Rbuildignore +++ b/r/.Rbuildignore @@ -9,3 +9,4 @@ Dockerfile .*\.tar\.gz ^windows clang_format.sh +^cran-comments\.md$ diff --git a/r/cran-comments.md b/r/cran-comments.md new file mode 100644 index 00000000000..c99d06a2288 --- /dev/null +++ b/r/cran-comments.md @@ -0,0 +1,27 @@ +## Test environments +* local OS X install, R 3.5.3 +* win-builder (devel and release) + +## R CMD check results + +0 errors | 0 warnings | 1 note + +* This is a new release. + +## Platform support + +This package supports Windows and macOS but not Linux. + +The Arrow project is cross-language development platform +for in-memory data, it spans several languages and +their code base is quite large (about 150K lines of C +sources and more than 600K lines across all languages). + +In the future, the Apache Arrow project will release +binaries in the official Fedora and Debian repos; +we're working on hard on this, but due to the size, +this is likely to be implemented until next year. + +In the meantime, R users can install the Linux binaries +from custom repos or build Arrow from source when using +Linux.