From 36f7a92ae2f21a3a5d174a1e231c1ed4a82cb0ed Mon Sep 17 00:00:00 2001 From: gnguy Date: Tue, 12 Nov 2019 10:25:56 -0800 Subject: [PATCH 1/7] Add lz4 and zstd Addressing https://issues.apache.org/jira/browse/ARROW-6960, adding lz4 and zstd, which was added in rtools via https://github.com/r-windows/rtools-packages/pull/47 --- ci/PKGBUILD | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/PKGBUILD b/ci/PKGBUILD index 7eb0598b5dd..917335cdeae 100644 --- a/ci/PKGBUILD +++ b/ci/PKGBUILD @@ -28,7 +28,9 @@ depends=("${MINGW_PACKAGE_PREFIX}-boost" "${MINGW_PACKAGE_PREFIX}-double-conversion" "${MINGW_PACKAGE_PREFIX}-thrift" "${MINGW_PACKAGE_PREFIX}-snappy" - "${MINGW_PACKAGE_PREFIX}-zlib") + "${MINGW_PACKAGE_PREFIX}-zlib" + "${MINGW_PACKAGE_PREFIX}-lz4" + "${MINGW_PACKAGE_PREFIX}-zstd") makedepends=("${MINGW_PACKAGE_PREFIX}-cmake" "${MINGW_PACKAGE_PREFIX}-gcc") options=("staticlibs" "strip" "!buildflags") @@ -85,6 +87,8 @@ build() { -DARROW_JEMALLOC=OFF \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_ZSTD=ON \ -DARROW_USE_GLOG=OFF \ -DARROW_BUILD_UTILITIES=OFF \ -Ddouble-conversion_ROOT="${MINGW_PREFIX}" \ From 6099995aa9a509655a00373cfcd4abc68de9aaeb Mon Sep 17 00:00:00 2001 From: gnguy Date: Thu, 14 Nov 2019 13:48:21 -0800 Subject: [PATCH 2/7] Add zstd and lz4 to sh script and create new codec availability test script --- ci/windows-pkg-arrow-for-r.sh | 6 ++++++ r/tests/testthat/test-codec-availability.R | 24 ++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 r/tests/testthat/test-codec-availability.R diff --git a/ci/windows-pkg-arrow-for-r.sh b/ci/windows-pkg-arrow-for-r.sh index 23a518bad83..7049da46579 100644 --- a/ci/windows-pkg-arrow-for-r.sh +++ b/ci/windows-pkg-arrow-for-r.sh @@ -64,6 +64,12 @@ wget https://dl.bintray.com/rtools/mingw32/mingw-w64-i686-double-conversion-3.1. wget https://dl.bintray.com/rtools/mingw64/mingw-w64-x86_64-openssl-1.1.1.a-1-any.pkg.tar.xz wget https://dl.bintray.com/rtools/mingw32/mingw-w64-i686-openssl-1.1.1.a-1-any.pkg.tar.xz +wget https://dl.bintray.com/rtools/mingw64/:mingw-w64-x86_64-zstd-1.4.3-1-any.pkg.tar.xz +wget https://dl.bintray.com/rtools/mingw32/:mingw-w64-i686-zstd-1.4.3-1-any.pkg.tar.xz + +wget https://dl.bintray.com/rtools/mingw64/:mingw-w64-x86_64-lz4-1.8.2-1-any.pkg.tar.xz +wget https://dl.bintray.com/rtools/mingw32/:mingw-w64-i686-lz4-1.8.2-1-any.pkg.tar.xz + # These are the other Rtools 4.0 packages, for future reference # wget https://dl.bintray.com/rtools/mingw32/mingw-w64-i686-boost-1.67.0-9002-any.pkg.tar.xz # wget https://dl.bintray.com/rtools/mingw64/mingw-w64-x86_64-boost-1.67.0-9002-any.pkg.tar.xz diff --git a/r/tests/testthat/test-codec-availability.R b/r/tests/testthat/test-codec-availability.R new file mode 100644 index 00000000000..fba8e9a9043 --- /dev/null +++ b/r/tests/testthat/test-codec-availability.R @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +context("CodecAvailability") + +test_that("codec_is_available", { + expect_true(codec_is_available("lz4")) + expect_true(codec_is_available("zstd")) +}) From c4d7e964a824abb27022c3a17b699eb8e157cf18 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 18 Nov 2019 11:23:47 -0800 Subject: [PATCH 3/7] Move test and only run it on Appveyor --- r/tests/testthat/test-codec-availability.R | 24 ---------------------- r/tests/testthat/test-compressed.R | 7 +++++++ 2 files changed, 7 insertions(+), 24 deletions(-) delete mode 100644 r/tests/testthat/test-codec-availability.R diff --git a/r/tests/testthat/test-codec-availability.R b/r/tests/testthat/test-codec-availability.R deleted file mode 100644 index fba8e9a9043..00000000000 --- a/r/tests/testthat/test-codec-availability.R +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -context("CodecAvailability") - -test_that("codec_is_available", { - expect_true(codec_is_available("lz4")) - expect_true(codec_is_available("zstd")) -}) diff --git a/r/tests/testthat/test-compressed.R b/r/tests/testthat/test-compressed.R index 93d4e334d0b..dcb5c52b935 100644 --- a/r/tests/testthat/test-compressed.R +++ b/r/tests/testthat/test-compressed.R @@ -25,6 +25,13 @@ test_that("codec_is_available", { expect_true(codec_is_available("GZIP")) }) +if (identical(Sys.getenv("APPVEYOR"), "True")) { + test_that("Compression codecs are included in the Windows build", { + expect_true(codec_is_available("lz4")) + expect_true(codec_is_available("zstd")) + }) +} + test_that("Codec attributes", { skip_if_not_available("gzip") cod <- Codec$create("gzip") From e7a38538b09ccdc8f1a8e8f2f514046f91d1afb9 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 18 Nov 2019 14:53:11 -0800 Subject: [PATCH 4/7] Remove colons --- ci/windows-pkg-arrow-for-r.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/windows-pkg-arrow-for-r.sh b/ci/windows-pkg-arrow-for-r.sh index 7049da46579..ae9a52b51b0 100644 --- a/ci/windows-pkg-arrow-for-r.sh +++ b/ci/windows-pkg-arrow-for-r.sh @@ -64,11 +64,11 @@ wget https://dl.bintray.com/rtools/mingw32/mingw-w64-i686-double-conversion-3.1. wget https://dl.bintray.com/rtools/mingw64/mingw-w64-x86_64-openssl-1.1.1.a-1-any.pkg.tar.xz wget https://dl.bintray.com/rtools/mingw32/mingw-w64-i686-openssl-1.1.1.a-1-any.pkg.tar.xz -wget https://dl.bintray.com/rtools/mingw64/:mingw-w64-x86_64-zstd-1.4.3-1-any.pkg.tar.xz -wget https://dl.bintray.com/rtools/mingw32/:mingw-w64-i686-zstd-1.4.3-1-any.pkg.tar.xz +wget https://dl.bintray.com/rtools/mingw64/mingw-w64-x86_64-zstd-1.4.3-1-any.pkg.tar.xz +wget https://dl.bintray.com/rtools/mingw32/mingw-w64-i686-zstd-1.4.3-1-any.pkg.tar.xz -wget https://dl.bintray.com/rtools/mingw64/:mingw-w64-x86_64-lz4-1.8.2-1-any.pkg.tar.xz -wget https://dl.bintray.com/rtools/mingw32/:mingw-w64-i686-lz4-1.8.2-1-any.pkg.tar.xz +wget https://dl.bintray.com/rtools/mingw64/mingw-w64-x86_64-lz4-1.8.2-1-any.pkg.tar.xz +wget https://dl.bintray.com/rtools/mingw32/mingw-w64-i686-lz4-1.8.2-1-any.pkg.tar.xz # These are the other Rtools 4.0 packages, for future reference # wget https://dl.bintray.com/rtools/mingw32/mingw-w64-i686-boost-1.67.0-9002-any.pkg.tar.xz From 8878df2ee8a9e2b2f1802afac1df73f8ae498406 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 18 Nov 2019 14:54:07 -0800 Subject: [PATCH 5/7] Run only R, for testing --- appveyor.yml | 58 ++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 65b44e54933..0b161ea94d9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -60,36 +60,36 @@ environment: ARCH: "64" matrix: - # NOTE: clcache seems to work best with Ninja and worst with msbuild - # (as generated by cmake) - - JOB: "Build" - GENERATOR: Ninja - CONFIGURATION: "Release" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - - JOB: "Toolchain" - GENERATOR: Ninja - CONFIGURATION: "Release" - ARROW_S3: "ON" - ARROW_BUILD_FLIGHT: "ON" - ARROW_BUILD_GANDIVA: "ON" - # NOTE: Since ARROW-5403 we have disabled the static CRT build - # - JOB: "Static_Crt_Build" + # # NOTE: clcache seems to work best with Ninja and worst with msbuild + # # (as generated by cmake) + # - JOB: "Build" # GENERATOR: Ninja - - JOB: "Build_Debug" - GENERATOR: Ninja - CONFIGURATION: "Debug" - - JOB: "MinGW32" - MINGW_ARCH: i686 - MINGW_PACKAGE_PREFIX: mingw-w64-i686 - MINGW_PREFIX: c:\msys64\mingw32 - MSYSTEM: MINGW32 - USE_CLCACHE: false - - JOB: "MinGW64" - MINGW_ARCH: x86_64 - MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 - MINGW_PREFIX: c:\msys64\mingw64 - MSYSTEM: MINGW64 - USE_CLCACHE: false + # CONFIGURATION: "Release" + # APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + # - JOB: "Toolchain" + # GENERATOR: Ninja + # CONFIGURATION: "Release" + # ARROW_S3: "ON" + # ARROW_BUILD_FLIGHT: "ON" + # ARROW_BUILD_GANDIVA: "ON" + # # NOTE: Since ARROW-5403 we have disabled the static CRT build + # # - JOB: "Static_Crt_Build" + # # GENERATOR: Ninja + # - JOB: "Build_Debug" + # GENERATOR: Ninja + # CONFIGURATION: "Debug" + # - JOB: "MinGW32" + # MINGW_ARCH: i686 + # MINGW_PACKAGE_PREFIX: mingw-w64-i686 + # MINGW_PREFIX: c:\msys64\mingw32 + # MSYSTEM: MINGW32 + # USE_CLCACHE: false + # - JOB: "MinGW64" + # MINGW_ARCH: x86_64 + # MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 + # MINGW_PREFIX: c:\msys64\mingw64 + # MSYSTEM: MINGW64 + # USE_CLCACHE: false - JOB: "R with libarrow" USE_CLCACHE: false TEST_R_WITH_ARROW: "TRUE" From edef776833abf24b4a390e7abbe04b34e116ebb5 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 18 Nov 2019 15:13:04 -0800 Subject: [PATCH 6/7] Add libs to configure.win --- r/configure.win | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/configure.win b/r/configure.win index f1c93ac5862..74003b3bec7 100644 --- a/r/configure.win +++ b/r/configure.win @@ -41,7 +41,7 @@ RWINLIB="../windows/$(ls windows/ | grep ^arrow-)" OPENSSL_LIBS="-lcrypto -lcrypt32" PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW" -PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -lthrift -lsnappy -lboost_regex-mt-s -lboost_system-mt-s -lz ${OPENSSL_LIBS} -lws2_32" +PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -lthrift -lsnappy -lboost_regex-mt-s -lboost_system-mt-s -lz -lzstd -llz4 ${OPENSSL_LIBS} -lws2_32" echo "*** Writing Makevars.win" sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" src/Makevars.in > src/Makevars.win From e5421073f40a59f20c220c43a5218fb9dec9a385 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 18 Nov 2019 15:43:18 -0800 Subject: [PATCH 7/7] Revert "Run only R, for testing" This reverts commit 8878df2ee8a9e2b2f1802afac1df73f8ae498406. --- appveyor.yml | 58 ++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 0b161ea94d9..65b44e54933 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -60,36 +60,36 @@ environment: ARCH: "64" matrix: - # # NOTE: clcache seems to work best with Ninja and worst with msbuild - # # (as generated by cmake) - # - JOB: "Build" + # NOTE: clcache seems to work best with Ninja and worst with msbuild + # (as generated by cmake) + - JOB: "Build" + GENERATOR: Ninja + CONFIGURATION: "Release" + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + - JOB: "Toolchain" + GENERATOR: Ninja + CONFIGURATION: "Release" + ARROW_S3: "ON" + ARROW_BUILD_FLIGHT: "ON" + ARROW_BUILD_GANDIVA: "ON" + # NOTE: Since ARROW-5403 we have disabled the static CRT build + # - JOB: "Static_Crt_Build" # GENERATOR: Ninja - # CONFIGURATION: "Release" - # APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - # - JOB: "Toolchain" - # GENERATOR: Ninja - # CONFIGURATION: "Release" - # ARROW_S3: "ON" - # ARROW_BUILD_FLIGHT: "ON" - # ARROW_BUILD_GANDIVA: "ON" - # # NOTE: Since ARROW-5403 we have disabled the static CRT build - # # - JOB: "Static_Crt_Build" - # # GENERATOR: Ninja - # - JOB: "Build_Debug" - # GENERATOR: Ninja - # CONFIGURATION: "Debug" - # - JOB: "MinGW32" - # MINGW_ARCH: i686 - # MINGW_PACKAGE_PREFIX: mingw-w64-i686 - # MINGW_PREFIX: c:\msys64\mingw32 - # MSYSTEM: MINGW32 - # USE_CLCACHE: false - # - JOB: "MinGW64" - # MINGW_ARCH: x86_64 - # MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 - # MINGW_PREFIX: c:\msys64\mingw64 - # MSYSTEM: MINGW64 - # USE_CLCACHE: false + - JOB: "Build_Debug" + GENERATOR: Ninja + CONFIGURATION: "Debug" + - JOB: "MinGW32" + MINGW_ARCH: i686 + MINGW_PACKAGE_PREFIX: mingw-w64-i686 + MINGW_PREFIX: c:\msys64\mingw32 + MSYSTEM: MINGW32 + USE_CLCACHE: false + - JOB: "MinGW64" + MINGW_ARCH: x86_64 + MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 + MINGW_PREFIX: c:\msys64\mingw64 + MSYSTEM: MINGW64 + USE_CLCACHE: false - JOB: "R with libarrow" USE_CLCACHE: false TEST_R_WITH_ARROW: "TRUE"