From d154939b05f3905cd74836170d88bd78c46d7786 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Mon, 29 Nov 2021 14:46:02 -0600 Subject: [PATCH 1/8] move duckdb to remotes --- r/DESCRIPTION | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/r/DESCRIPTION b/r/DESCRIPTION index 08cb5d44038..2c889098921 100644 --- a/r/DESCRIPTION +++ b/r/DESCRIPTION @@ -47,7 +47,6 @@ Suggests: decor, distro, dplyr, - duckdb (>= 0.2.8), hms, knitr, lubridate, @@ -59,6 +58,8 @@ Suggests: testthat (>= 3.1.0), tibble, withr +Remotes: + duckdb/duckdb/tools/rpkg Collate: 'arrowExports.R' 'enums.R' From 0635fa88ee958ad88899d51c02e41e987080fee2 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Mon, 29 Nov 2021 15:26:56 -0600 Subject: [PATCH 2/8] A different tactic --- ci/scripts/r_deps.sh | 6 ++++++ r/DESCRIPTION | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ci/scripts/r_deps.sh b/ci/scripts/r_deps.sh index 79977fca16d..75efb8c5de6 100755 --- a/ci/scripts/r_deps.sh +++ b/ci/scripts/r_deps.sh @@ -41,6 +41,12 @@ fi # but we want to error/fail the build. # options(warn=2) turns warnings into errors ${R_BIN} -e "options(warn=2); install.packages('remotes'); remotes::install_cran(c('glue', 'rcmdcheck', 'sys')); remotes::install_deps(INSTALL_opts = '"${INSTALL_ARGS}"')" + +# (Temporarily) install DuckDB from source to avoid their Unity builds (though only if we haven't filtered it out of the deps above) +if grep -q "duckdb" DESCRIPTION; then + ${R_BIN} -e "remotes::install_github('duckdb/duckdb', subdir = '/tools/rpkg')" +fi + # Separately install the optional/test dependencies but don't error on them, # they're not available everywhere and that's ok ${R_BIN} -e "remotes::install_deps(dependencies = TRUE, INSTALL_opts = '"${INSTALL_ARGS}"')" diff --git a/r/DESCRIPTION b/r/DESCRIPTION index 2c889098921..08cb5d44038 100644 --- a/r/DESCRIPTION +++ b/r/DESCRIPTION @@ -47,6 +47,7 @@ Suggests: decor, distro, dplyr, + duckdb (>= 0.2.8), hms, knitr, lubridate, @@ -58,8 +59,6 @@ Suggests: testthat (>= 3.1.0), tibble, withr -Remotes: - duckdb/duckdb/tools/rpkg Collate: 'arrowExports.R' 'enums.R' From c25d769a3d925d5c40814e25f21e16f12136d271 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Mon, 29 Nov 2021 15:49:21 -0600 Subject: [PATCH 3/8] oops --- ci/scripts/r_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/r_deps.sh b/ci/scripts/r_deps.sh index 75efb8c5de6..99f31ec9f7e 100755 --- a/ci/scripts/r_deps.sh +++ b/ci/scripts/r_deps.sh @@ -44,7 +44,7 @@ ${R_BIN} -e "options(warn=2); install.packages('remotes'); remotes::install_cran # (Temporarily) install DuckDB from source to avoid their Unity builds (though only if we haven't filtered it out of the deps above) if grep -q "duckdb" DESCRIPTION; then - ${R_BIN} -e "remotes::install_github('duckdb/duckdb', subdir = '/tools/rpkg')" + ${R_BIN} -e "remotes::install_github('duckdb/duckdb', subdir = '/tools/rpkg', build = FALSE)" fi # Separately install the optional/test dependencies but don't error on them, From b108c8cdfcd0a1f793a41c86e833d6be2dec7ba0 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Mon, 29 Nov 2021 16:34:47 -0600 Subject: [PATCH 4/8] set the python symlinks before R deps --- ci/docker/linux-apt-r.dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile index 894f53bc0a3..f2c9575d6ec 100644 --- a/ci/docker/linux-apt-r.dockerfile +++ b/ci/docker/linux-apt-r.dockerfile @@ -84,6 +84,10 @@ RUN cat /arrow/ci/etc/rprofile >> $(R RHOME)/etc/Rprofile.site # Also ensure parallel compilation of C/C++ code RUN echo "MAKEFLAGS=-j$(R -s -e 'cat(parallel::detectCores())')" >> $(R RHOME)/etc/Renviron.site +# Set up Python 3 and its dependencies +RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ + ln -s /usr/bin/pip3 /usr/local/bin/pip + COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ COPY r/DESCRIPTION /arrow/r/ RUN /arrow/ci/scripts/r_deps.sh /arrow @@ -93,10 +97,6 @@ RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_gcs_testbench.sh ${arch} default -# Set up Python 3 and its dependencies -RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ - ln -s /usr/bin/pip3 /usr/local/bin/pip - COPY python/requirements-build.txt /arrow/python/ RUN pip install -r arrow/python/requirements-build.txt From 0b771629c2175f7f6669c0fef1b697c3b228047c Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Mon, 29 Nov 2021 17:41:12 -0600 Subject: [PATCH 5/8] also need to link python in linuc-r.dockerfile --- ci/docker/linux-r.dockerfile | 4 ++++ ci/scripts/r_deps.sh | 1 + 2 files changed, 5 insertions(+) diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile index ca701e259c4..1cbde3207ed 100644 --- a/ci/docker/linux-r.dockerfile +++ b/ci/docker/linux-r.dockerfile @@ -46,6 +46,10 @@ COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/r_docker_configure.sh +# Set up Python 3 and its dependencies +RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ + ln -s /usr/bin/pip3 /usr/local/bin/pip + COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ COPY r/DESCRIPTION /arrow/r/ RUN /arrow/ci/scripts/r_deps.sh /arrow diff --git a/ci/scripts/r_deps.sh b/ci/scripts/r_deps.sh index 99f31ec9f7e..ce0d0f69560 100755 --- a/ci/scripts/r_deps.sh +++ b/ci/scripts/r_deps.sh @@ -43,6 +43,7 @@ fi ${R_BIN} -e "options(warn=2); install.packages('remotes'); remotes::install_cran(c('glue', 'rcmdcheck', 'sys')); remotes::install_deps(INSTALL_opts = '"${INSTALL_ARGS}"')" # (Temporarily) install DuckDB from source to avoid their Unity builds (though only if we haven't filtered it out of the deps above) +# Remove when there is a DuckDB release > 0.3.1-1 if grep -q "duckdb" DESCRIPTION; then ${R_BIN} -e "remotes::install_github('duckdb/duckdb', subdir = '/tools/rpkg', build = FALSE)" fi From a660b47d9a753ee3633a001334d057254e95a4d9 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Mon, 29 Nov 2021 18:05:52 -0600 Subject: [PATCH 6/8] Use RSPM binary if it's in the repos --- ci/scripts/r_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/r_deps.sh b/ci/scripts/r_deps.sh index ce0d0f69560..00e2b4c650a 100755 --- a/ci/scripts/r_deps.sh +++ b/ci/scripts/r_deps.sh @@ -45,7 +45,7 @@ ${R_BIN} -e "options(warn=2); install.packages('remotes'); remotes::install_cran # (Temporarily) install DuckDB from source to avoid their Unity builds (though only if we haven't filtered it out of the deps above) # Remove when there is a DuckDB release > 0.3.1-1 if grep -q "duckdb" DESCRIPTION; then - ${R_BIN} -e "remotes::install_github('duckdb/duckdb', subdir = '/tools/rpkg', build = FALSE)" + ${R_BIN} -e "if (all(!grepl('packagemanager.rstudio', options('repos')))) { remotes::install_github('duckdb/duckdb', subdir = '/tools/rpkg', build = FALSE) }" fi # Separately install the optional/test dependencies but don't error on them, From 5be557ca98c8c1e9d5a48713c3aa6595351c1a49 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Tue, 30 Nov 2021 12:00:15 -0800 Subject: [PATCH 7/8] Update ci/scripts/r_deps.sh Co-authored-by: Neal Richardson --- ci/scripts/r_deps.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/scripts/r_deps.sh b/ci/scripts/r_deps.sh index 00e2b4c650a..0fa4368cd8a 100755 --- a/ci/scripts/r_deps.sh +++ b/ci/scripts/r_deps.sh @@ -42,7 +42,9 @@ fi # options(warn=2) turns warnings into errors ${R_BIN} -e "options(warn=2); install.packages('remotes'); remotes::install_cran(c('glue', 'rcmdcheck', 'sys')); remotes::install_deps(INSTALL_opts = '"${INSTALL_ARGS}"')" -# (Temporarily) install DuckDB from source to avoid their Unity builds (though only if we haven't filtered it out of the deps above) +# (Temporarily) install DuckDB from source to avoid their Unity builds +# (though only if we haven't filtered it out of the deps above, +# and if we can't get a binary from RSPM) # Remove when there is a DuckDB release > 0.3.1-1 if grep -q "duckdb" DESCRIPTION; then ${R_BIN} -e "if (all(!grepl('packagemanager.rstudio', options('repos')))) { remotes::install_github('duckdb/duckdb', subdir = '/tools/rpkg', build = FALSE) }" From a1d4a07baccfc1349255c1a526dcbc2b4ad9e7bb Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Tue, 30 Nov 2021 16:44:58 -0600 Subject: [PATCH 8/8] Skip flaky DuckDB roundtrip for now --- r/tests/testthat/test-duckdb.R | 1 + 1 file changed, 1 insertion(+) diff --git a/r/tests/testthat/test-duckdb.R b/r/tests/testthat/test-duckdb.R index 1b3012bcef8..b31fffda95a 100644 --- a/r/tests/testthat/test-duckdb.R +++ b/r/tests/testthat/test-duckdb.R @@ -66,6 +66,7 @@ test_that("to_duckdb", { }) test_that("to_duckdb then to_arrow", { + skip("Flaky, unskip when ARROW-14745 is merged") ds <- InMemoryDataset$create(example_data) ds_rt <- ds %>%