From 78e41cc4b3de1be1e1d6d7c9cbf2352a97386e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Saint-Jacques?= Date: Tue, 18 Jun 2019 12:28:34 -0400 Subject: [PATCH 1/8] Fix trusty docker build --- cpp/Dockerfile.ubuntu-trusty | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/Dockerfile.ubuntu-trusty b/cpp/Dockerfile.ubuntu-trusty index d44491c82fb..bca51773436 100644 --- a/cpp/Dockerfile.ubuntu-trusty +++ b/cpp/Dockerfile.ubuntu-trusty @@ -62,6 +62,7 @@ ENV CC=gcc \ ARROW_BUILD_TESTS=ON \ ARROW_DEPENDENCY_SOURCE=SYSTEM \ ARROW_FLIGHT=OFF \ + ARROW_WITH_OPENSSL=OFF \ ARROW_GANDIVA=OFF \ ARROW_GANDIVA_JAVA=OFF \ ARROW_ORC=OFF \ From bd85741cc9cafb53c86c7bdceadc16d4f0d2e3fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Saint-Jacques?= Date: Wed, 19 Jun 2019 11:11:05 -0400 Subject: [PATCH 2/8] Add custom docker and vendor boost conda's cmake3.2 does not support https for third party download. --- cpp/Dockerfile.cmake | 70 ++++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 8 +++-- 2 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 cpp/Dockerfile.cmake diff --git a/cpp/Dockerfile.cmake b/cpp/Dockerfile.cmake new file mode 100644 index 00000000000..013b2b170ee --- /dev/null +++ b/cpp/Dockerfile.cmake @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM ubuntu:18.04 + +# install build essentials +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + ca-certificates \ + curl \ + ccache \ + g++ \ + gcc \ + git \ + libidn11 \ + ninja-build \ + pkg-config \ + tzdata \ + wget \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# install conda and required packages +ARG EXTRA_CONDA_PKGS +ENV PATH=/opt/conda/bin:$PATH \ + CONDA_PREFIX=/opt/conda +COPY ci/docker_install_conda.sh \ + ci/conda_env_cpp.yml \ + ci/conda_env_unix.yml \ + /arrow/ci/ +RUN arrow/ci/docker_install_conda.sh && \ + conda install -q -c conda-forge \ + --file arrow/ci/conda_env_cpp.yml \ + --file arrow/ci/conda_env_unix.yml \ + $EXTRA_CONDA_PKGS && \ + conda clean --all + +ARG CMAKE_VERSION=3.2.3 +ARG CMAKE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz +RUN curl ${CMAKE_URL} -Lo cmake-${CMAKE_VERSION}.tar.gz && \ + mkdir /opt/cmake && tar -zxvf cmake-${CMAKE_VERSION}.tar.gz -C /opt/cmake +ENV PATH=/opt/cmake/cmake-${CMAKE_VERSION}-Linux-x86_64/bin:$PATH + +RUN which cmake && cmake --version + +ENV CC=gcc \ + CXX=g++ \ + ARROW_GANDIVA=OFF \ + ARROW_BUILD_TESTS=ON \ + ARROW_DEPENDENCY_SOURCE=CONDA \ + ARROW_HOME=$CONDA_PREFIX \ + PARQUET_HOME=$CONDA_PREFIX + +# build and test +CMD ["arrow/ci/docker_build_and_test_cpp.sh"] diff --git a/docker-compose.yml b/docker-compose.yml index e986c11f432..6ff601deba4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -149,11 +149,15 @@ services: shm_size: 2G build: context: . - dockerfile: cpp/Dockerfile + dockerfile: cpp/Dockerfile.cmake args: - EXTRA_CONDA_PKGS: cmake=3.2 + CMAKE_VERSION: 3.2.3 environment: + # Vendor boost to avoid dealing with stale FindBoost. + ARROW_BOOST_SOURCE: "BUNDLED" ARROW_ORC: "OFF" + # uriparser requires cmake 3.3 + ARROW_FLIGHT: "OFF" PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data volumes: *ubuntu-volumes From 7544850a4b90e9c493bb949f74eee7cb1c08a9d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Saint-Jacques?= Date: Thu, 20 Jun 2019 16:56:49 -0400 Subject: [PATCH 3/8] Remove debug docker command --- cpp/Dockerfile.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/Dockerfile.cmake b/cpp/Dockerfile.cmake index 013b2b170ee..9160cd7e853 100644 --- a/cpp/Dockerfile.cmake +++ b/cpp/Dockerfile.cmake @@ -56,8 +56,6 @@ RUN curl ${CMAKE_URL} -Lo cmake-${CMAKE_VERSION}.tar.gz && \ mkdir /opt/cmake && tar -zxvf cmake-${CMAKE_VERSION}.tar.gz -C /opt/cmake ENV PATH=/opt/cmake/cmake-${CMAKE_VERSION}-Linux-x86_64/bin:$PATH -RUN which cmake && cmake --version - ENV CC=gcc \ CXX=g++ \ ARROW_GANDIVA=OFF \ From 4b1c29fd80e7f751398b28e7e4d68c0777c9e0ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Saint-Jacques?= Date: Fri, 21 Jun 2019 09:33:14 -0400 Subject: [PATCH 4/8] ARROW-5653: [CI] Fix conda libraries --- ci/docker_build_and_test_cpp.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ci/docker_build_and_test_cpp.sh b/ci/docker_build_and_test_cpp.sh index 99b9460fee5..f2b9f54c881 100755 --- a/ci/docker_build_and_test_cpp.sh +++ b/ci/docker_build_and_test_cpp.sh @@ -24,5 +24,11 @@ pushd /build/cpp export ARROW_TEST_DATA=/arrow/testing/data export PARQUET_TEST_DATA=/arrow/cpp/submodules/parquet-testing/data +# ARROW-5653 after install target, RPATH is modified on linux. Add this such +# that libraries are found with conda. +if [[ ! -z "${CONDA_PREFIX}" ]]; then + export LD_LIBRARY_PATH=${CONDA_PREFIX}/lib +fi + ninja unittest popd From 3b70ff3058ce58a9ea9704b6762f792859d60d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Saint-Jacques?= Date: Fri, 21 Jun 2019 10:54:50 -0400 Subject: [PATCH 5/8] Fix rust docker image --- rust/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rust/Dockerfile b/rust/Dockerfile index 77eba7b7fd4..a6e0463253e 100644 --- a/rust/Dockerfile +++ b/rust/Dockerfile @@ -23,4 +23,7 @@ ENV CARGO_TARGET_DIR=/build/rust COPY rust/Cargo.toml /arrow/rust/Cargo.toml WORKDIR /arrow/rust +ENV ARROW_TEST_DATA=/arrow/testing/data +ENV PARQUET_TEST_DATA=/arrow/cpp/submodules/parquet-testing/data + CMD ["cargo", "test"] From 6c632072df4563c544ed0c9b893b8d88858c4c0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Saint-Jacques?= Date: Fri, 21 Jun 2019 12:10:46 -0400 Subject: [PATCH 6/8] Fix c_glib PKG_CONFIG_PATH in docker --- c_glib/Dockerfile | 2 +- dev/tasks/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/c_glib/Dockerfile b/c_glib/Dockerfile index 163db7f9ff4..a7ef13785fd 100644 --- a/c_glib/Dockerfile +++ b/c_glib/Dockerfile @@ -40,7 +40,7 @@ ENV ARROW_BUILD_TESTS=OFF \ ARROW_BUILD_UTILITIES=OFF \ ARROW_INSTALL_NAME_RPATH=OFF \ LD_LIBRARY_PATH="${CONDA_PREFIX}/lib" \ - PKG_CONFIG_PATH="${CONDA_PREFIX}/lib/pkgconfig" \ + PKG_CONFIG_PATH="${CONDA_PREFIX}/lib/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig" \ GI_TYPELIB_PATH="${CONDA_PREFIX}/lib/girepository-1.0" # build, install and test diff --git a/dev/tasks/tests.yml b/dev/tasks/tests.yml index f8dc8178ded..634e252d18e 100644 --- a/dev/tasks/tests.yml +++ b/dev/tasks/tests.yml @@ -130,7 +130,7 @@ tasks: commands: - docker-compose build cpp - docker-compose build c_glib - - docker-compose run cpp + - docker-compose run c_glib docker-go: ci: circle From 01f67dd653bd1cfc53ee0c04bab8b8baf63d84b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Saint-Jacques?= Date: Fri, 21 Jun 2019 14:26:07 -0400 Subject: [PATCH 7/8] Fix R docker image - Ensure that LD_LIBRARY_PATH is set before installing R packages because conda's pkg-config would pickup libicu from conda at compile but LD_LIBRARY_PATH was not exported yet, thus package would fail tests. - Fix JSON payload on systems where locales are missing. --- r/Dockerfile | 18 +++++++++++------- r/tests/testthat/test-json.R | 4 ++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/r/Dockerfile b/r/Dockerfile index 63ff3e7b100..f2964691185 100644 --- a/r/Dockerfile +++ b/r/Dockerfile @@ -23,6 +23,8 @@ ENV CC=gcc \ # r-base includes tzdata. Get around interactive stop in that package ENV DEBIAN_FRONTEND=noninteractive +# workaround for install_github Github API rate limit +ENV CI=true # Build R # [1] https://www.digitalocean.com/community/tutorials/how-to-install-r-on-ubuntu-18-04 @@ -50,7 +52,15 @@ RUN apt-get update -y && \ texlive-latex-base && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ - Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')" && \ + +# So that arrowExports.* files are generated +ENV ARROW_R_DEV=TRUE + +# Tell R where it can find the source code for arrow +ENV PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/build/cpp/src/arrow:/opt/conda/lib/pkgconfig +ENV LD_LIBRARY_PATH=/opt/conda/lib/:/build/cpp/src/arrow:/arrow/r/src + +RUN Rscript -e "install.packages('devtools', repos = 'http://cran.rstudio.com')" && \ Rscript -e "devtools::install_github('romainfrancois/decor')" && \ Rscript -e "install.packages(c( \ 'Rcpp', 'dplyr', 'stringr', 'glue', 'vctrs', \ @@ -65,12 +75,6 @@ RUN apt-get update -y && \ 'lubridate'), \ repos = 'https://cran.rstudio.com')" -# So that arrowExports.* files are generated -ENV ARROW_R_DEV=TRUE - -# Tell R where it can find the source code for arrow -ENV PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/build/cpp/src/arrow:/opt/conda/lib/pkgconfig -ENV LD_LIBRARY_PATH=/opt/conda/lib/:/build/cpp/src/arrow:/arrow/r/src # build, install, test R package CMD ["/bin/bash", "-c", "/arrow/ci/docker_build_cpp.sh && \ diff --git a/r/tests/testthat/test-json.R b/r/tests/testthat/test-json.R index 206dfdd22a8..45daf33392c 100644 --- a/r/tests/testthat/test-json.R +++ b/r/tests/testthat/test-json.R @@ -24,7 +24,7 @@ test_that("Can read json file with scalars columns (ARROW-5503)", { { "hello": 3.25, "world": null } { "hello": 3.125, "world": null, "yo": "\u5fcd" } { "hello": 0.0, "world": true, "yo": null } - ', tf) + ', tf, useBytes=TRUE) tab1 <- read_json_arrow(tf, as_tibble = FALSE) tab2 <- read_json_arrow(mmap_open(tf), as_tibble = FALSE) @@ -52,7 +52,7 @@ test_that("read_json_arrow() converts to tibble", { { "hello": 3.25, "world": null } { "hello": 3.125, "world": null, "yo": "\u5fcd" } { "hello": 0.0, "world": true, "yo": null } - ', tf) + ', tf, useBytes=TRUE) tab1 <- read_json_arrow(tf) tab2 <- read_json_arrow(mmap_open(tf)) From 9b946dad7da789e8116e57cb745264ae7144e4cb Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 21 Jun 2019 17:04:15 -0500 Subject: [PATCH 8/8] Code review feedback --- c_glib/Dockerfile | 2 +- r/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/c_glib/Dockerfile b/c_glib/Dockerfile index a7ef13785fd..163db7f9ff4 100644 --- a/c_glib/Dockerfile +++ b/c_glib/Dockerfile @@ -40,7 +40,7 @@ ENV ARROW_BUILD_TESTS=OFF \ ARROW_BUILD_UTILITIES=OFF \ ARROW_INSTALL_NAME_RPATH=OFF \ LD_LIBRARY_PATH="${CONDA_PREFIX}/lib" \ - PKG_CONFIG_PATH="${CONDA_PREFIX}/lib/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig" \ + PKG_CONFIG_PATH="${CONDA_PREFIX}/lib/pkgconfig" \ GI_TYPELIB_PATH="${CONDA_PREFIX}/lib/girepository-1.0" # build, install and test diff --git a/r/Dockerfile b/r/Dockerfile index f2964691185..541bc19f1d8 100644 --- a/r/Dockerfile +++ b/r/Dockerfile @@ -23,7 +23,7 @@ ENV CC=gcc \ # r-base includes tzdata. Get around interactive stop in that package ENV DEBIAN_FRONTEND=noninteractive -# workaround for install_github Github API rate limit +# workaround for install_github GitHub API rate limit ENV CI=true # Build R