From 085de0f8e52a3a4a0bd42eb97478564c9f3a9903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 16 Oct 2019 09:43:46 -0400 Subject: [PATCH 01/12] use travis wait to for compiling openssl [skip ci] --- dev/tasks/python-wheels/travis.osx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index ec3e97d2fe9..c579894b155 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -47,7 +47,7 @@ before_install: - brew uninstall boost cgal postgis sfcgal - brew update - brew upgrade cmake - - brew install bison flex grpc openssl@1.1 llvm@7 zlib gperftools + - travis_wait brew install bison flex grpc openssl@1.1 llvm@7 zlib gperftools # Remove shared grpc libraries installed by brew to make sure # we are linked against the static ones. - rm -f /usr/local/opt/grpc/lib/*.dylib From a02ad43b87823c15a7b9e152e0aece7b92bfaad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 16 Oct 2019 09:58:09 -0400 Subject: [PATCH 02/12] use different dir to build conda linux artifacts [skip ci] --- dev/tasks/conda-recipes/build_steps.sh | 4 ++-- dev/tasks/conda-recipes/run_docker_build.sh | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh index 9f243f3a763..8c44e327157 100755 --- a/dev/tasks/conda-recipes/build_steps.sh +++ b/dev/tasks/conda-recipes/build_steps.sh @@ -34,6 +34,6 @@ conda build \ "${FEEDSTOCK_ROOT}/pyarrow" \ -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" \ - --output-folder "${FEEDSTOCK_ROOT}/build_artifacts" + --output-folder "${OUTPUT_FOLDER}" -touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" +touch "${OUTPUT_FOLDER}/conda-forge-build-done-${CONFIG}" diff --git a/dev/tasks/conda-recipes/run_docker_build.sh b/dev/tasks/conda-recipes/run_docker_build.sh index 99476c59e9a..46e3d0c6cf1 100755 --- a/dev/tasks/conda-recipes/run_docker_build.sh +++ b/dev/tasks/conda-recipes/run_docker_build.sh @@ -58,7 +58,9 @@ export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" docker run ${DOCKER_RUN_ARGS} \ --shm-size=2G \ -v "${ARROW_ROOT}":/arrow:rw,z \ + -v "${ARTIFACTS}":/build_artifacts:rw,z \ -e FEEDSTOCK_ROOT="/arrow/dev/tasks/conda-recipes" \ + -e OUTPUT_FOLDER="/build_artifacts" \ -e CONFIG \ -e HOST_USER_ID \ -e UPLOAD_PACKAGES \ From 8cb11b997b782fd06cf934ddc28e64df10238ccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 16 Oct 2019 10:59:31 -0400 Subject: [PATCH 03/12] wait 30 minutes [skip ci] --- dev/tasks/python-wheels/travis.osx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index c579894b155..c895fca0204 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -47,7 +47,7 @@ before_install: - brew uninstall boost cgal postgis sfcgal - brew update - brew upgrade cmake - - travis_wait brew install bison flex grpc openssl@1.1 llvm@7 zlib gperftools + - travis_wait 30 brew install bison flex grpc openssl@1.1 llvm@7 zlib gperftools # Remove shared grpc libraries installed by brew to make sure # we are linked against the static ones. - rm -f /usr/local/opt/grpc/lib/*.dylib From 04839966020f8d1ef2aa4ea6b04db0b80ec00ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 16 Oct 2019 12:30:33 -0400 Subject: [PATCH 04/12] pin conda build for linux [skip ci] --- dev/tasks/conda-recipes/build_steps.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh index 8c44e327157..f73ce6720fd 100755 --- a/dev/tasks/conda-recipes/build_steps.sh +++ b/dev/tasks/conda-recipes/build_steps.sh @@ -25,6 +25,10 @@ setup_conda_rc "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}" source run_conda_forge_build_setup +# pin conda build, 3.18.10 has introduced issue: +# "Can't merge/copy source into subdirectory of itself." +conda install -y conda-build=3.18.9 + # make the build number clobber make_build_number "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}" From 0fb47c831dac8a93a34fdce1b3523ca86c63cdc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 17 Oct 2019 09:42:32 -0400 Subject: [PATCH 05/12] use folder outside of arrow's directory for building artifacts [skip ci] --- dev/tasks/conda-recipes/azure.linux.yml | 7 ++++--- dev/tasks/conda-recipes/build_steps.sh | 13 ++++++------- dev/tasks/conda-recipes/run_docker_build.sh | 13 ++++++------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index ce58ba0f7b4..4c3c907b12a 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -26,9 +26,10 @@ jobs: git -C arrow submodule update --init --recursive displayName: Clone arrow - - script: CI=azure ./run_docker_build.sh + - script: | + mkdir build_artifacts + CI=azure arrow/dev/tasks/conda-recipes/run_docker_build.sh $(pwd)/build_artifacts displayName: Run docker build - workingDirectory: arrow/dev/tasks/conda-recipes # Using github release tries to find a common ancestor between the # currently pushed tag and the latest tag of the github repository @@ -47,7 +48,7 @@ jobs: upload-artifacts \ --sha {{ task.branch }} \ --tag {{ task.tag }} \ - --pattern "arrow/dev/tasks/conda-recipes/build_artifacts/linux-64/*.tar.bz2" + --pattern "build_artifacts/linux-64/*.tar.bz2" env: CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN) displayName: Upload packages as a GitHub release diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh index f73ce6720fd..49793eaa7dd 100755 --- a/dev/tasks/conda-recipes/build_steps.sh +++ b/dev/tasks/conda-recipes/build_steps.sh @@ -6,6 +6,9 @@ # benefit from the improvement. set -xeuo pipefail + +output_dir=${1} + export PYTHONUNBUFFERED=1 export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" @@ -14,7 +17,7 @@ export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml" cat >~/.condarc </dev/null && pwd )" ARROW_ROOT=$(cd "$THISDIR/../../.."; pwd;) FEEDSTOCK_ROOT=$THISDIR @@ -23,8 +25,6 @@ if hash docker-machine 2> /dev/null && docker-machine active > /dev/null; then export HOST_USER_ID=$(docker-machine ssh $(docker-machine active) id -u) fi -ARTIFACTS="$FEEDSTOCK_ROOT/build_artifacts" - if [ -z "$CONFIG" ]; then set +x FILES=`ls .ci_support/linux_*` @@ -46,8 +46,8 @@ if [ -z "${DOCKER_IMAGE}" ]; then fi fi -mkdir -p "$ARTIFACTS" -DONE_CANARY="$ARTIFACTS/conda-forge-build-done-${CONFIG}" +mkdir -p "${build_dir}" +DONE_CANARY="${build_dir}/conda-forge-build-done-${CONFIG}" rm -f "$DONE_CANARY" if [ -z "${CI}" ]; then @@ -58,16 +58,15 @@ export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" docker run ${DOCKER_RUN_ARGS} \ --shm-size=2G \ -v "${ARROW_ROOT}":/arrow:rw,z \ - -v "${ARTIFACTS}":/build_artifacts:rw,z \ + -v "${build_dir}":/build:rw \ -e FEEDSTOCK_ROOT="/arrow/dev/tasks/conda-recipes" \ - -e OUTPUT_FOLDER="/build_artifacts" \ -e CONFIG \ -e HOST_USER_ID \ -e UPLOAD_PACKAGES \ -e ARROW_VERSION \ -e CI \ $DOCKER_IMAGE \ - bash /arrow/dev/tasks/conda-recipes/build_steps.sh + bash /arrow/dev/tasks/conda-recipes/build_steps.sh /build # verify that the end of the script was reached test -f "$DONE_CANARY" From 960b25f2bde5330517a4bc2e245df41b902cac22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 17 Oct 2019 09:50:52 -0400 Subject: [PATCH 06/12] use newer xcode for building osx wheels [skip ci] --- dev/tasks/python-wheels/travis.osx.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index c895fca0204..f732a45eb8b 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -15,7 +15,7 @@ # limitations under the License. os: osx -osx_image: xcode8.3 +osx_image: xcode9.2 language: generic # don't build twice @@ -81,7 +81,7 @@ install: # move built wheels to a top level directory - mv -v arrow/python/dist/* dist/ # reinstall openssl because travis' deployment script depends on it - - brew install openssl + - brew install openssl@1.1 deploy: provider: releases From 97fa57c8a18a5def93e89861de154e7addd26b84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 17 Oct 2019 13:08:02 -0400 Subject: [PATCH 07/12] force conda bld path [skip ci] --- dev/tasks/conda-recipes/build_steps.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh index 49793eaa7dd..5a76b0263a2 100755 --- a/dev/tasks/conda-recipes/build_steps.sh +++ b/dev/tasks/conda-recipes/build_steps.sh @@ -31,6 +31,8 @@ source run_conda_forge_build_setup # make the build number clobber make_build_number "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}" +export CONDA_BLD_PATH="${output_dir}" + conda build \ "${FEEDSTOCK_ROOT}/arrow-cpp" \ "${FEEDSTOCK_ROOT}/parquet-cpp" \ From 05972290ceed43c073cdbe1750f1a7cfc0dc7e93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 17 Oct 2019 13:10:41 -0400 Subject: [PATCH 08/12] use xcode 9.3 for osx wheels [skip ci] --- dev/tasks/python-wheels/travis.osx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index f732a45eb8b..7d4177fd418 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -15,7 +15,7 @@ # limitations under the License. os: osx -osx_image: xcode9.2 +osx_image: xcode9.3 language: generic # don't build twice From eae0e4f1b62f133ea04fd5bce0725420d991c13a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 17 Oct 2019 16:49:11 -0400 Subject: [PATCH 09/12] turn off BZ2 for windows wheels [skip ci] --- dev/tasks/python-wheels/win-build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index c4e092f0ddb..95d6a879040 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -49,7 +49,7 @@ cmake -G "%GENERATOR%" ^ -DARROW_DEPENDENCY_SOURCE=CONDA ^ -DOPENSSL_ROOT_DIR=%CONDA_PREFIX%/Library ^ -DARROW_CXXFLAGS="/MP" ^ - -DARROW_WITH_BZ2=ON ^ + -DARROW_WITH_BZ2=OFF ^ -DARROW_WITH_ZLIB=ON ^ -DARROW_WITH_ZSTD=ON ^ -DARROW_WITH_LZ4=ON ^ From 6c7e763f7a078bbb0dc6e3ff190f198911ed2b4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 18 Oct 2019 10:13:39 -0400 Subject: [PATCH 10/12] enforce artifact uploading for windows wheel [skip ci] --- dev/tasks/python-wheels/win-build.bat | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 95d6a879040..160e4126770 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -49,13 +49,13 @@ cmake -G "%GENERATOR%" ^ -DARROW_DEPENDENCY_SOURCE=CONDA ^ -DOPENSSL_ROOT_DIR=%CONDA_PREFIX%/Library ^ -DARROW_CXXFLAGS="/MP" ^ - -DARROW_WITH_BZ2=OFF ^ + -DARROW_WITH_BZ2=ON ^ -DARROW_WITH_ZLIB=ON ^ -DARROW_WITH_ZSTD=ON ^ -DARROW_WITH_LZ4=ON ^ -DARROW_WITH_SNAPPY=ON ^ -DARROW_WITH_BROTLI=ON ^ - -DARROW_FLIGHT=ON ^ + -DARROW_FLIGHT=OFF ^ -DARROW_PYTHON=ON ^ -DARROW_PARQUET=ON ^ -DARROW_GANDIVA=ON ^ @@ -92,10 +92,10 @@ call conda create -n wheel-test -c conda-forge -q -y ^ call activate wheel-test @rem install the built wheel -pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B +@rem pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B @rem test the imports -python -c "import pyarrow; import pyarrow.parquet; import pyarrow.flight; import pyarrow.gandiva;" || exit /B +@rem python -c "import pyarrow; import pyarrow.parquet; import pyarrow.flight; import pyarrow.gandiva;" || exit /B @rem run the python tests -pytest -rs --pyargs pyarrow || exit /B +@rem pytest -rs --pyargs pyarrow || exit /B From e0ea665bf03294a059642afc6fbbe0ef528eca67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 18 Oct 2019 11:53:13 -0400 Subject: [PATCH 11/12] enable flight [skip ci] --- dev/tasks/python-wheels/win-build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 160e4126770..3f517389a22 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -55,7 +55,7 @@ cmake -G "%GENERATOR%" ^ -DARROW_WITH_LZ4=ON ^ -DARROW_WITH_SNAPPY=ON ^ -DARROW_WITH_BROTLI=ON ^ - -DARROW_FLIGHT=OFF ^ + -DARROW_FLIGHT=ON ^ -DARROW_PYTHON=ON ^ -DARROW_PARQUET=ON ^ -DARROW_GANDIVA=ON ^ From 5cd34f3eccebb16f1a569e96532ad1660c225e61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 18 Oct 2019 13:28:36 -0400 Subject: [PATCH 12/12] revert windows wheel testing [skip ci] --- dev/tasks/python-wheels/win-build.bat | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 3f517389a22..c4e092f0ddb 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -92,10 +92,10 @@ call conda create -n wheel-test -c conda-forge -q -y ^ call activate wheel-test @rem install the built wheel -@rem pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B +pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B @rem test the imports -@rem python -c "import pyarrow; import pyarrow.parquet; import pyarrow.flight; import pyarrow.gandiva;" || exit /B +python -c "import pyarrow; import pyarrow.parquet; import pyarrow.flight; import pyarrow.gandiva;" || exit /B @rem run the python tests -@rem pytest -rs --pyargs pyarrow || exit /B +pytest -rs --pyargs pyarrow || exit /B