From af1852e79c4d9341b60b5f4f9424efc3447605ef Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 12 Jun 2019 13:25:11 -0700 Subject: [PATCH 01/16] Throw this at appveyor and see how far off we are --- appveyor.yml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 3e0e645d988..91b13aefb59 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -101,6 +101,12 @@ environment: GO111MODULE: on GOTOOLDIR: '%GOROOT%\pkg\tool\windows_amd64' PATH: '%GOPATH%\bin;%GOROOT%\bin;%PATH%' + - JOB: "R" + MINGW_ARCH: x86_64 + MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 + MINGW_PREFIX: c:\msys64\mingw64 + MSYSTEM: MINGW64 + USE_CLCACHE: false MSVC_DEFAULT_OPTIONS: ON APPVEYOR_SAVE_CACHE_ON_ERROR: true @@ -118,6 +124,53 @@ before_build: build_script: - call ci\appveyor-build.bat +for: +- + matrix: + only: + - JOB: "R" + init: + ps: | + $ErrorActionPreference = "Stop" + Invoke-WebRequest http://raw.githubusercontent.com/krlmlr/r-appveyor/master/scripts/appveyor-tool.ps1 -OutFile "..\appveyor-tool.ps1" + Import-Module '..\appveyor-tool.ps1' + install: + ps: Bootstrap + + # Adapt as necessary starting from here + + build_script: + - pushd r + - travis-tool.sh install_deps + + test_script: + - travis-tool.sh run_tests + + on_failure: + - travis-tool.sh dump_logs + + environment: + NOT_CRAN: true + + artifacts: + - path: '*.Rcheck\**\*.log' + name: Logs + + - path: '*.Rcheck\**\*.out' + name: Logs + + - path: '*.Rcheck\**\*.fail' + name: Logs + + - path: '*.Rcheck\**\*.Rout' + name: Logs + + - path: '\*_*.tar.gz' + name: Bits + + - path: '\*_*.zip' + name: Bits + # Disable test discovery test: off From 94c9a42df61d68186510dc1216df9e8053a122cd Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 12 Jun 2019 13:27:10 -0700 Subject: [PATCH 02/16] Debug with fewer jobs in the matrix --- appveyor.yml | 76 ++++++++++++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 91b13aefb59..6e6bd4647ea 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -62,45 +62,45 @@ environment: GENERATOR: Ninja CONFIGURATION: "Release" APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - - JOB: "Toolchain" - GENERATOR: Visual Studio 14 2015 Win64 - CONFIGURATION: "Release" - ARROW_BUILD_FLIGHT: "ON" - ARROW_BUILD_GANDIVA: "ON" - # NOTE: Since ARROW-5403 we have disabled the static CRT build - # - JOB: "Static_Crt_Build" + # - JOB: "Toolchain" + # GENERATOR: Visual Studio 14 2015 Win64 + # CONFIGURATION: "Release" + # ARROW_BUILD_FLIGHT: "ON" + # ARROW_BUILD_GANDIVA: "ON" + # # NOTE: Since ARROW-5403 we have disabled the static CRT build + # # - JOB: "Static_Crt_Build" + # # GENERATOR: Ninja + # - JOB: "Build_Debug" # GENERATOR: Ninja - - JOB: "Build_Debug" - GENERATOR: Ninja - CONFIGURATION: "Debug" - - JOB: "MinGW32" - MINGW_ARCH: i686 - MINGW_PACKAGE_PREFIX: mingw-w64-i686 - MINGW_PREFIX: c:\msys64\mingw32 - MSYSTEM: MINGW32 - USE_CLCACHE: false - - JOB: "MinGW64" - MINGW_ARCH: x86_64 - MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 - MINGW_PREFIX: c:\msys64\mingw64 - MSYSTEM: MINGW64 - USE_CLCACHE: false - - JOB: "Rust" - TARGET: x86_64-pc-windows-msvc - USE_CLCACHE: false - - JOB: "C#" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - USE_CLCACHE: false - - JOB: "Go" - MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 - MINGW_PREFIX: c:\msys64\mingw64 - MSYSTEM: MINGW64 - USE_CLCACHE: false - GOROOT: 'c:\go112' - GOPATH: c:\gopath - GO111MODULE: on - GOTOOLDIR: '%GOROOT%\pkg\tool\windows_amd64' - PATH: '%GOPATH%\bin;%GOROOT%\bin;%PATH%' + # CONFIGURATION: "Debug" + # - JOB: "MinGW32" + # MINGW_ARCH: i686 + # MINGW_PACKAGE_PREFIX: mingw-w64-i686 + # MINGW_PREFIX: c:\msys64\mingw32 + # MSYSTEM: MINGW32 + # USE_CLCACHE: false + # - JOB: "MinGW64" + # MINGW_ARCH: x86_64 + # MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 + # MINGW_PREFIX: c:\msys64\mingw64 + # MSYSTEM: MINGW64 + # USE_CLCACHE: false + # - JOB: "Rust" + # TARGET: x86_64-pc-windows-msvc + # USE_CLCACHE: false + # - JOB: "C#" + # APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + # USE_CLCACHE: false + # - JOB: "Go" + # MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 + # MINGW_PREFIX: c:\msys64\mingw64 + # MSYSTEM: MINGW64 + # USE_CLCACHE: false + # GOROOT: 'c:\go112' + # GOPATH: c:\gopath + # GO111MODULE: on + # GOTOOLDIR: '%GOROOT%\pkg\tool\windows_amd64' + # PATH: '%GOPATH%\bin;%GOROOT%\bin;%PATH%' - JOB: "R" MINGW_ARCH: x86_64 MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 From 39236f3f74a533e802bc6a2a1a581addcac48c82 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 12 Jun 2019 13:28:27 -0700 Subject: [PATCH 03/16] Put mine first and fast-finish --- appveyor.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 6e6bd4647ea..eec3714ebdd 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -58,6 +58,12 @@ environment: matrix: # NOTE: clcache seems to work best with Ninja and worst with msbuild # (as generated by cmake) + - JOB: "R" + MINGW_ARCH: x86_64 + MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 + MINGW_PREFIX: c:\msys64\mingw64 + MSYSTEM: MINGW64 + USE_CLCACHE: false - JOB: "Build" GENERATOR: Ninja CONFIGURATION: "Release" @@ -101,12 +107,6 @@ environment: # GO111MODULE: on # GOTOOLDIR: '%GOROOT%\pkg\tool\windows_amd64' # PATH: '%GOPATH%\bin;%GOROOT%\bin;%PATH%' - - JOB: "R" - MINGW_ARCH: x86_64 - MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 - MINGW_PREFIX: c:\msys64\mingw64 - MSYSTEM: MINGW64 - USE_CLCACHE: false MSVC_DEFAULT_OPTIONS: ON APPVEYOR_SAVE_CACHE_ON_ERROR: true @@ -176,3 +176,6 @@ test: off after_build: - if "%USE_CLCACHE%" == "true" clcache -s + +matrix: + fast_finish: true From 8c3e898f8fbd85ea9c3afb55a708577f664f2ce7 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 12 Jun 2019 13:29:03 -0700 Subject: [PATCH 04/16] nm we already fast finish --- appveyor.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index eec3714ebdd..932ba0dc68d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -176,6 +176,3 @@ test: off after_build: - if "%USE_CLCACHE%" == "true" clcache -s - -matrix: - fast_finish: true From f7327ed230cbb6ccfd10762c7c6bcbfa317619e3 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 12 Jun 2019 13:34:02 -0700 Subject: [PATCH 05/16] Find shell script up a level --- appveyor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 932ba0dc68d..0597c198d62 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -141,13 +141,13 @@ for: build_script: - pushd r - - travis-tool.sh install_deps + - ../travis-tool.sh install_deps test_script: - - travis-tool.sh run_tests + - ../travis-tool.sh run_tests on_failure: - - travis-tool.sh dump_logs + - ../travis-tool.sh dump_logs environment: NOT_CRAN: true From d5e8f4ed488a91468ef439edcd4bc2029be891fa Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 12 Jun 2019 13:39:14 -0700 Subject: [PATCH 06/16] Bootstrap within the package dir --- appveyor.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 0597c198d62..bdbee42dd01 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -19,7 +19,7 @@ os: Visual Studio 2015 only_commits: - # Skip commits not related to Python, C++, C#, Go or Rust + # Skip commits not related to Python, C++, C#, Go, R, or Rust files: - appveyor.yml - c_glib/ @@ -29,6 +29,7 @@ only_commits: - format/ - go/ - python/ + - r/ - rust/ cache: @@ -135,19 +136,21 @@ for: Invoke-WebRequest http://raw.githubusercontent.com/krlmlr/r-appveyor/master/scripts/appveyor-tool.ps1 -OutFile "..\appveyor-tool.ps1" Import-Module '..\appveyor-tool.ps1' install: - ps: Bootstrap + - pushd r + - ps: Bootstrap + - popd # Adapt as necessary starting from here build_script: - pushd r - - ../travis-tool.sh install_deps + - travis-tool.sh install_deps test_script: - - ../travis-tool.sh run_tests + - travis-tool.sh run_tests on_failure: - - ../travis-tool.sh dump_logs + - travis-tool.sh dump_logs environment: NOT_CRAN: true From 1ba4839e2686abeb9aafaa4e4ba39973865e3cd9 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 12 Jun 2019 13:44:24 -0700 Subject: [PATCH 07/16] let the debugging begin --- appveyor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index bdbee42dd01..a104008df64 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -139,6 +139,11 @@ for: - pushd r - ps: Bootstrap - popd + - python ci\detect-changes.py > generated_changes.bat + # Populate ARROW_CI_* variables + - call generated_changes.bat + # Decide to exit if current job doesn't exercise affected topics + - call ci\appveyor-filter-changes.bat # Adapt as necessary starting from here From b6f6c8de70bca30247d1d8c42f649c22d977a1e4 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 12 Jun 2019 13:47:30 -0700 Subject: [PATCH 08/16] No popd? --- appveyor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index a104008df64..97ba6df1396 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -138,7 +138,8 @@ for: install: - pushd r - ps: Bootstrap - - popd + # No need to popd because Bootstrap changes directory? + #- popd - python ci\detect-changes.py > generated_changes.bat # Populate ARROW_CI_* variables - call generated_changes.bat From 18803c35318d3983524d21ce137528b13ebc234b Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 12 Jun 2019 13:52:58 -0700 Subject: [PATCH 09/16] cd this way --- appveyor.yml | 5 ++--- ci/appveyor-filter-changes.bat | 7 +++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 97ba6df1396..a20110eb26b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -136,10 +136,9 @@ for: Invoke-WebRequest http://raw.githubusercontent.com/krlmlr/r-appveyor/master/scripts/appveyor-tool.ps1 -OutFile "..\appveyor-tool.ps1" Import-Module '..\appveyor-tool.ps1' install: - - pushd r + - cd r - ps: Bootstrap - # No need to popd because Bootstrap changes directory? - #- popd + - cd %APPVEYOR_BUILD_FOLDER% - python ci\detect-changes.py > generated_changes.bat # Populate ARROW_CI_* variables - call generated_changes.bat diff --git a/ci/appveyor-filter-changes.bat b/ci/appveyor-filter-changes.bat index 1e82ecd5f20..75efb51b430 100644 --- a/ci/appveyor-filter-changes.bat +++ b/ci/appveyor-filter-changes.bat @@ -43,6 +43,13 @@ if "%JOB%" == "Rust" ( echo === appveyor exit ) +) else if "%JOB%" == "R" ( + if "%ARROW_CI_R_AFFECTED%" == "0" ( + echo === + echo === No C++ or R changes, exiting job + echo === + appveyor exit + ) ) else ( if "%ARROW_CI_PYTHON_AFFECTED%" == "0" ( echo === From 15c0342d156265cf4477e4ba7fcb762ac04634b1 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 12 Jun 2019 14:04:26 -0700 Subject: [PATCH 10/16] Try building first, see where that goes --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index a20110eb26b..3fc2dcf4ec3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -148,6 +148,7 @@ for: # Adapt as necessary starting from here build_script: + - call ci\appveyor-build.bat - pushd r - travis-tool.sh install_deps From fb199110aa631a463258af5bc441bd671dbf9a7a Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 12 Jun 2019 14:50:33 -0700 Subject: [PATCH 11/16] Try this name hack --- appveyor.yml | 4 ++-- ci/appveyor-filter-changes.bat | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 3fc2dcf4ec3..29bfb6fbae2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -59,7 +59,7 @@ environment: matrix: # NOTE: clcache seems to work best with Ninja and worst with msbuild # (as generated by cmake) - - JOB: "R" + - JOB: "MinGW64 R" MINGW_ARCH: x86_64 MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 MINGW_PREFIX: c:\msys64\mingw64 @@ -129,7 +129,7 @@ for: - matrix: only: - - JOB: "R" + - JOB: "MinGW64 R" init: ps: | $ErrorActionPreference = "Stop" diff --git a/ci/appveyor-filter-changes.bat b/ci/appveyor-filter-changes.bat index 75efb51b430..a2d150a49e1 100644 --- a/ci/appveyor-filter-changes.bat +++ b/ci/appveyor-filter-changes.bat @@ -43,7 +43,7 @@ if "%JOB%" == "Rust" ( echo === appveyor exit ) -) else if "%JOB%" == "R" ( +) else if "%JOB%" == "MinGW64 R" ( if "%ARROW_CI_R_AFFECTED%" == "0" ( echo === echo === No C++ or R changes, exiting job From 19b60df0795dc2dcd00596e00b958e5b3b3e7938 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 13 Jun 2019 12:59:44 -0700 Subject: [PATCH 12/16] Enable R package to install on windows even if rwinlib isn't found --- appveyor.yml | 4 +--- r/configure.win | 45 +++++++++++++++++++++++++++++++++++++++++++++ r/src/Makevars.win | 38 -------------------------------------- r/tools/winlibs.R | 6 +++--- 4 files changed, 49 insertions(+), 44 deletions(-) create mode 100644 r/configure.win delete mode 100644 r/src/Makevars.win diff --git a/appveyor.yml b/appveyor.yml index 29bfb6fbae2..48897e3db5e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -145,10 +145,8 @@ for: # Decide to exit if current job doesn't exercise affected topics - call ci\appveyor-filter-changes.bat - # Adapt as necessary starting from here - build_script: - - call ci\appveyor-build.bat + # - call ci\appveyor-build.bat - pushd r - travis-tool.sh install_deps diff --git a/r/configure.win b/r/configure.win new file mode 100644 index 00000000000..3a26b55a1ed --- /dev/null +++ b/r/configure.win @@ -0,0 +1,45 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# generate code +if [ "$ARROW_R_DEV" == "TRUE" ]; then + echo "*** Generating code with data-raw/codegen.R" + "${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" data-raw/codegen.R +fi + +VERSION=$(grep ^Version DESCRIPTION | sed s/Version:\ //) +"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" "tools/winlibs.R" $VERSION + +if [ $? -ne 0 ]; then + echo "------------------------- NOTE ---------------------------" + echo "After installation, please run arrow::install_arrow() to install" + echo "required runtime libraries" + echo "---------------------------------------------------------" +else + RWINLIB="../windows/arrow-${VERSION}" + PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_R_WITH_PARQUET -DARROW_R_WITH_ARROW" + PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow -lthrift -lboost_regex-mt-s -ldouble-conversion -lz -lws2_32" +fi + +# Write to Makevars +echo "*** Writing Makevars.win" +sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" src/Makevars.in > src/Makevars.win +# Success +exit 0 diff --git a/r/src/Makevars.win b/r/src/Makevars.win deleted file mode 100644 index ebad1d5f8ba..00000000000 --- a/r/src/Makevars.win +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -VERSION = 0.13.0.9000 -RWINLIB = ../windows/arrow-$(VERSION) -PKG_CPPFLAGS = -I$(RWINLIB)/include -DARROW_STATIC -DPARQUET_STATIC \ - -DARROW_R_WITH_PARQUET -CXX_STD = CXX11 - -PKG_LIBS = \ - -L$(RWINLIB)/lib$(subst gcc,,$(COMPILED_BY))$(R_ARCH) \ - -L$(RWINLIB)/lib$(R_ARCH) \ - -lparquet -larrow -lthrift -lboost_regex-mt-s -ldouble-conversion -lz -lws2_32 - -#all: clean -all: $(SHLIB) - -$(OBJECTS): winlibs - -winlibs: - "${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" "../tools/winlibs.R" $(VERSION) - -clean: - rm -f $(SHLIB) $(OBJECTS) diff --git a/r/tools/winlibs.R b/r/tools/winlibs.R index 12391c77d63..4f6b4fa07ab 100644 --- a/r/tools/winlibs.R +++ b/r/tools/winlibs.R @@ -17,10 +17,10 @@ # Download static arrow from rwinlib VERSION <- commandArgs(TRUE) -if(!file.exists(sprintf("../windows/arrow-%s/include/arrow/api.h", VERSION))){ +if(!file.exists(sprintf("windows/arrow-%s/include/arrow/api.h", VERSION))){ if(getRversion() < "3.3.0") setInternet2() download.file(sprintf("https://github.com/rwinlib/arrow/archive/v%s.zip", VERSION), "lib.zip", quiet = TRUE) - dir.create("../windows", showWarnings = FALSE) - unzip("lib.zip", exdir = "../windows") + dir.create("windows", showWarnings = FALSE) + unzip("lib.zip", exdir = "windows") unlink("lib.zip") } From f4cf5c3dd869b7586b35167ed54af42eea50a24d Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 13 Jun 2019 14:17:30 -0700 Subject: [PATCH 13/16] Some appveyor.yml cleanup --- appveyor.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 48897e3db5e..17752a03edd 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -59,11 +59,7 @@ environment: matrix: # NOTE: clcache seems to work best with Ninja and worst with msbuild # (as generated by cmake) - - JOB: "MinGW64 R" - MINGW_ARCH: x86_64 - MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 - MINGW_PREFIX: c:\msys64\mingw64 - MSYSTEM: MINGW64 + - JOB: "R without libarrow" USE_CLCACHE: false - JOB: "Build" GENERATOR: Ninja @@ -129,23 +125,21 @@ for: - matrix: only: - - JOB: "MinGW64 R" + - JOB: "R without libarrow" init: ps: | $ErrorActionPreference = "Stop" Invoke-WebRequest http://raw.githubusercontent.com/krlmlr/r-appveyor/master/scripts/appveyor-tool.ps1 -OutFile "..\appveyor-tool.ps1" Import-Module '..\appveyor-tool.ps1' - install: + + before_build: - cd r - ps: Bootstrap - cd %APPVEYOR_BUILD_FOLDER% - - python ci\detect-changes.py > generated_changes.bat - # Populate ARROW_CI_* variables - - call generated_changes.bat - # Decide to exit if current job doesn't exercise affected topics - - call ci\appveyor-filter-changes.bat build_script: + # TODO: for the version of the R tests that builds, probably can't use this + # ci script because we need to build with the Rtools toolchain # - call ci\appveyor-build.bat - pushd r - travis-tool.sh install_deps From 567f04142dca16c89ef8d062dc1b4988344a10f1 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 13 Jun 2019 14:36:13 -0700 Subject: [PATCH 14/16] Cleanups --- appveyor.yml | 82 +++++++++++++++++----------------- ci/appveyor-filter-changes.bat | 2 +- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 17752a03edd..62d989f6ec3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -59,52 +59,52 @@ environment: matrix: # NOTE: clcache seems to work best with Ninja and worst with msbuild # (as generated by cmake) - - JOB: "R without libarrow" - USE_CLCACHE: false - JOB: "Build" GENERATOR: Ninja CONFIGURATION: "Release" APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - # - JOB: "Toolchain" - # GENERATOR: Visual Studio 14 2015 Win64 - # CONFIGURATION: "Release" - # ARROW_BUILD_FLIGHT: "ON" - # ARROW_BUILD_GANDIVA: "ON" - # # NOTE: Since ARROW-5403 we have disabled the static CRT build - # # - JOB: "Static_Crt_Build" - # # GENERATOR: Ninja - # - JOB: "Build_Debug" + - JOB: "Toolchain" + GENERATOR: Visual Studio 14 2015 Win64 + CONFIGURATION: "Release" + ARROW_BUILD_FLIGHT: "ON" + ARROW_BUILD_GANDIVA: "ON" + # NOTE: Since ARROW-5403 we have disabled the static CRT build + # - JOB: "Static_Crt_Build" # GENERATOR: Ninja - # CONFIGURATION: "Debug" - # - JOB: "MinGW32" - # MINGW_ARCH: i686 - # MINGW_PACKAGE_PREFIX: mingw-w64-i686 - # MINGW_PREFIX: c:\msys64\mingw32 - # MSYSTEM: MINGW32 - # USE_CLCACHE: false - # - JOB: "MinGW64" - # MINGW_ARCH: x86_64 - # MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 - # MINGW_PREFIX: c:\msys64\mingw64 - # MSYSTEM: MINGW64 - # USE_CLCACHE: false - # - JOB: "Rust" - # TARGET: x86_64-pc-windows-msvc - # USE_CLCACHE: false - # - JOB: "C#" - # APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - # USE_CLCACHE: false - # - JOB: "Go" - # MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 - # MINGW_PREFIX: c:\msys64\mingw64 - # MSYSTEM: MINGW64 - # USE_CLCACHE: false - # GOROOT: 'c:\go112' - # GOPATH: c:\gopath - # GO111MODULE: on - # GOTOOLDIR: '%GOROOT%\pkg\tool\windows_amd64' - # PATH: '%GOPATH%\bin;%GOROOT%\bin;%PATH%' - + - JOB: "Build_Debug" + GENERATOR: Ninja + CONFIGURATION: "Debug" + - JOB: "MinGW32" + MINGW_ARCH: i686 + MINGW_PACKAGE_PREFIX: mingw-w64-i686 + MINGW_PREFIX: c:\msys64\mingw32 + MSYSTEM: MINGW32 + USE_CLCACHE: false + - JOB: "MinGW64" + MINGW_ARCH: x86_64 + MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 + MINGW_PREFIX: c:\msys64\mingw64 + MSYSTEM: MINGW64 + USE_CLCACHE: false + - JOB: "Rust" + TARGET: x86_64-pc-windows-msvc + USE_CLCACHE: false + - JOB: "C#" + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + USE_CLCACHE: false + - JOB: "Go" + MINGW_PACKAGE_PREFIX: mingw-w64-x86_64 + MINGW_PREFIX: c:\msys64\mingw64 + MSYSTEM: MINGW64 + USE_CLCACHE: false + GOROOT: 'c:\go112' + GOPATH: c:\gopath + GO111MODULE: on + GOTOOLDIR: '%GOROOT%\pkg\tool\windows_amd64' + PATH: '%GOPATH%\bin;%GOROOT%\bin;%PATH%' + - JOB: "R without libarrow" + USE_CLCACHE: false + MSVC_DEFAULT_OPTIONS: ON APPVEYOR_SAVE_CACHE_ON_ERROR: true diff --git a/ci/appveyor-filter-changes.bat b/ci/appveyor-filter-changes.bat index a2d150a49e1..c265091a98a 100644 --- a/ci/appveyor-filter-changes.bat +++ b/ci/appveyor-filter-changes.bat @@ -43,7 +43,7 @@ if "%JOB%" == "Rust" ( echo === appveyor exit ) -) else if "%JOB%" == "MinGW64 R" ( +) else if "%JOB:~,2%" == "R " ( if "%ARROW_CI_R_AFFECTED%" == "0" ( echo === echo === No C++ or R changes, exiting job From 53c3a2084e7656e627f6f414dc9da5e95c11714b Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 13 Jun 2019 15:44:21 -0700 Subject: [PATCH 15/16] Makevars.win is dynamically generated now, no need to tweak it in the release script --- dev/release/00-prepare-test.rb | 14 -------------- dev/release/00-prepare.sh | 8 -------- 2 files changed, 22 deletions(-) diff --git a/dev/release/00-prepare-test.rb b/dev/release/00-prepare-test.rb index 8a1936d7735..8ce8ca1bfc2 100644 --- a/dev/release/00-prepare-test.rb +++ b/dev/release/00-prepare-test.rb @@ -130,13 +130,6 @@ def test_update_version_pre_tag "+Version: #{@release_version}"], ], }, - { - path: "r/src/Makevars.win", - hunks: [ - ["-VERSION = #{@previous_version}.9000", - "+VERSION = #{@release_version}"], - ], - }, { path: "ruby/red-arrow-cuda/lib/arrow-cuda/version.rb", hunks: [ @@ -279,13 +272,6 @@ def test_update_version_post_tag "+Version: #{@release_version}.9000"], ], }, - { - path: "r/src/Makevars.win", - hunks: [ - ["-VERSION = #{@release_version}", - "+VERSION = #{@release_version}.9000"], - ], - }, { path: "ruby/red-arrow-cuda/lib/arrow-cuda/version.rb", hunks: [ diff --git a/dev/release/00-prepare.sh b/dev/release/00-prepare.sh index 724df83ecd9..f9b891fe99f 100755 --- a/dev/release/00-prepare.sh +++ b/dev/release/00-prepare.sh @@ -101,14 +101,6 @@ update_versions() { git add DESCRIPTION cd - - cd "${SOURCE_DIR}/../../r/src" - sed -i.bak -E -e \ - "s/^VERSION = .+/VERSION = ${r_version}/" \ - Makevars.win - rm -f Makevars.win.bak - git add Makevars.win - cd - - cd "${SOURCE_DIR}/../../ruby" sed -i.bak -E -e \ "s/^ VERSION = \".+\"/ VERSION = \"${version}\"/g" \ From 6d6a74e312c49aed72aec98d4ab6ec820bf78517 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 17 Jun 2019 12:33:31 -0700 Subject: [PATCH 16/16] PR feedback --- appveyor.yml | 14 +++++++------- r/configure.win | 6 ++++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 60652899783..a4995a89e6d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -105,7 +105,7 @@ environment: PATH: '%GOPATH%\bin;%GOROOT%\bin;%PATH%' - JOB: "R without libarrow" USE_CLCACHE: false - + MSVC_DEFAULT_OPTIONS: ON APPVEYOR_SAVE_CACHE_ON_ERROR: true @@ -155,22 +155,22 @@ for: NOT_CRAN: true artifacts: - - path: '*.Rcheck\**\*.log' + - path: 'r\*.Rcheck\**\*.log' name: Logs - - path: '*.Rcheck\**\*.out' + - path: 'r\*.Rcheck\**\*.out' name: Logs - - path: '*.Rcheck\**\*.fail' + - path: 'r\*.Rcheck\**\*.fail' name: Logs - - path: '*.Rcheck\**\*.Rout' + - path: 'r\*.Rcheck\**\*.Rout' name: Logs - - path: '\*_*.tar.gz' + - path: 'r\*_*.tar.gz' name: Bits - - path: '\*_*.zip' + - path: 'r\*_*.zip' name: Bits # Disable test discovery diff --git a/r/configure.win b/r/configure.win index 3a26b55a1ed..75702e6156d 100644 --- a/r/configure.win +++ b/r/configure.win @@ -25,20 +25,22 @@ if [ "$ARROW_R_DEV" == "TRUE" ]; then fi VERSION=$(grep ^Version DESCRIPTION | sed s/Version:\ //) +# Try to find/download a C++ Arrow binary "${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" "tools/winlibs.R" $VERSION - +# If binary not found, script exits nonzero if [ $? -ne 0 ]; then + # We'll have to install without libarrow. Note to the user how to rectify. echo "------------------------- NOTE ---------------------------" echo "After installation, please run arrow::install_arrow() to install" echo "required runtime libraries" echo "---------------------------------------------------------" else + # Set the right flags to point to and enable arrow/parquet RWINLIB="../windows/arrow-${VERSION}" PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_R_WITH_PARQUET -DARROW_R_WITH_ARROW" PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow -lthrift -lboost_regex-mt-s -ldouble-conversion -lz -lws2_32" fi -# Write to Makevars echo "*** Writing Makevars.win" sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" src/Makevars.in > src/Makevars.win # Success