From 6d151787eb8b7ba453bd4a8a0cbf65a31c3aff01 Mon Sep 17 00:00:00 2001 From: "Korn, Uwe" Date: Thu, 27 Sep 2018 10:01:54 +0200 Subject: [PATCH 1/2] ARROW-3327: Use local Arrow checkout instead of separate clone --- ci/travis_script_manylinux.sh | 4 +-- python/manylinux1/Dockerfile-x86_64 | 38 ----------------------------- python/manylinux1/README.md | 21 +++++----------- python/manylinux1/build_arrow.sh | 5 ++-- 4 files changed, 10 insertions(+), 58 deletions(-) delete mode 100644 python/manylinux1/Dockerfile-x86_64 diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index 386a68488a1..588d0f9a7bb 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -21,9 +21,7 @@ set -ex pushd python/manylinux1 -git clone ../../ arrow -docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 . -docker run --shm-size=2g --rm -e PYARROW_PARALLEL=3 -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh +docker run --shm-size=2g --rm -e PYARROW_PARALLEL=3 -v $PWD:/io -v $PWD/../../:/arrow quay.io/xhochy/arrow_manylinux1_x86_64_base:latest /io/build_arrow.sh # Testing for https://issues.apache.org/jira/browse/ARROW-2657 # These tests cannot be run inside of the docker container, since TensorFlow diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64 deleted file mode 100644 index f79564dcc56..00000000000 --- a/python/manylinux1/Dockerfile-x86_64 +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-3335 - -ADD arrow /arrow -WORKDIR /arrow/cpp -RUN mkdir build-plain -WORKDIR /arrow/cpp/build-plain -RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/arrow-dist \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DARROW_BUILD_TESTS=OFF \ - -DARROW_BUILD_SHARED=ON \ - -DARROW_BOOST_USE_SHARED=ON \ - -DARROW_JEMALLOC=ON \ - -DARROW_PARQUET=ON \ - -DARROW_RPATH_ORIGIN=ON \ - -DARROW_JEMALLOC_USE_SHARED=OFF \ - -DBoost_NAMESPACE=arrow_boost \ - -DBOOST_ROOT=/arrow_boost_dist .. -RUN ninja install - -ADD scripts/check_arrow_visibility.sh / -RUN /check_arrow_visibility.sh diff --git a/python/manylinux1/README.md b/python/manylinux1/README.md index 2ec662d502e..580aea3e015 100644 --- a/python/manylinux1/README.md +++ b/python/manylinux1/README.md @@ -32,22 +32,17 @@ for all supported Python versions and place them in the `dist` folder. ### Build instructions ```bash -# Create a clean copy of the arrow source tree -git clone ../../ arrow -# Build the native baseimage -docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 . # Build the python packages -docker run --shm-size=2g --rm -t -i -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh +docker run --shm-size=2g --rm -t -i -v $PWD:/io -v $PWD/../../:/arrow quay.io/xhochy/arrow_manylinux1_x86_64_base:latest /io/build_arrow.sh # Now the new packages are located in the dist/ folder ls -l dist/ ``` ### Updating the build environment -In addition to the docker images that contains the Arrow C++ and Parquet C++ -builds, we also have another base image that only contains their dependencies. -This image is less often updated. In the case we want to update a dependency to -a new version, we also need to adjust it. You can rebuild this image using +The base docker image is less often updated. In the case we want to update +a dependency to a new version, we also need to adjust it. You can rebuild +this image using ```bash docker build -t arrow_manylinux1_x86_64_base -f Dockerfile-x86_64_base . @@ -57,9 +52,5 @@ For each dependency, we have a bash script in the directory `scripts/` that downloads the sources, builds and installs them. At the end of each dependency build the sources are removed again so that only the binary installation of a dependency is persisted in the docker image. When you do local adjustments to -this image, you need to change the `FROM` line in `Dockerfile-x86_64` to pick up -the new image: - -``` -FROM arrow_manylinux1_x86_64_base -``` +this image, you need to change the name of the docker image in the `docker run` +command. diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index 04034e7ed11..e48fdbbd003 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -65,7 +65,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do fi echo "=== (${PYTHON}) Building Arrow C++ libraries ===" - ARROW_BUILD_DIR=/arrow/cpp/build-PY${PYTHON}-${U_WIDTH} + ARROW_BUILD_DIR=/tmp/build-PY${PYTHON}-${U_WIDTH} mkdir -p "${ARROW_BUILD_DIR}" pushd "${ARROW_BUILD_DIR}" PATH="${CPYTHON_PATH}/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release \ @@ -77,13 +77,14 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do -DARROW_JEMALLOC=ON \ -DARROW_RPATH_ORIGIN=ON \ -DARROW_PYTHON=ON \ + -DARROW_PARQUET=ON \ -DPythonInterp_FIND_VERSION=${PYTHON} \ -DARROW_PLASMA=ON \ -DARROW_TENSORFLOW=ON \ -DARROW_ORC=ON \ -DBoost_NAMESPACE=arrow_boost \ -DBOOST_ROOT=/arrow_boost_dist \ - -GNinja .. + -GNinja /arrow/cpp ninja install popd From b2ac5a6bddf4e48c42603fb7e4d8671013beb4f4 Mon Sep 17 00:00:00 2001 From: "Korn, Uwe" Date: Thu, 27 Sep 2018 10:12:28 +0200 Subject: [PATCH 2/2] Readd visibility checks --- python/manylinux1/build_arrow.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index e48fdbbd003..6e257025d6c 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -88,6 +88,9 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do ninja install popd + # Check that we don't expose any unwanted symbols + /io/scripts/check_arrow_visibility.sh + # Clear output directory rm -rf dist/ echo "=== (${PYTHON}) Building wheel ==="