From 2037f78fc8a5842a123930656afe73f7a1e25cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 21 Jun 2019 15:06:04 +0200 Subject: [PATCH 01/39] port conda recipes to azure pipelines --- dev/release/rat_exclude_files.txt | 3 +- .../azure-pipelines-linux.yml | 31 ++++++ .../.azure-pipelines/azure-pipelines-osx.yml | 73 +++++++++++++ .../.azure-pipelines/azure-pipelines-win.yml | 100 ++++++++++++++++++ .../.azure-pipelines/build_steps.sh | 39 +++++++ .../.azure-pipelines/run_docker_build.sh | 73 +++++++++++++ .../linux_python2.7.yaml | 26 +---- .../linux_python3.6.yaml | 26 +---- .../linux_python3.7.yaml | 26 +---- .../osx_python2.7.yaml | 26 +---- .../osx_python3.6.yaml | 26 +---- .../osx_python3.7.yaml | 26 +---- ...lervs2015cxx_compilervs2015python3.6.yaml} | 27 +---- ...lervs2015cxx_compilervs2015python3.7.yaml} | 27 +---- dev/tasks/conda-recipes/README.md | 69 ++++++++++++ dev/tasks/conda-recipes/azure-pipelines.yml | 29 +++++ .../azure-template-clone-arrow.yml | 23 ++++ .../azure-template-upload-artifacts.yml | 16 +++ dev/tasks/crossbow.py | 6 +- dev/tasks/tasks.yml | 35 +++--- 20 files changed, 509 insertions(+), 198 deletions(-) create mode 100755 dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-linux.yml create mode 100755 dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-osx.yml create mode 100755 dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-win.yml create mode 100755 dev/tasks/conda-recipes/.azure-pipelines/build_steps.sh create mode 100755 dev/tasks/conda-recipes/.azure-pipelines/run_docker_build.sh rename dev/tasks/conda-recipes/{variants => .ci_support}/linux_python2.7.yaml (54%) rename dev/tasks/conda-recipes/{variants => .ci_support}/linux_python3.6.yaml (54%) rename dev/tasks/conda-recipes/{variants => .ci_support}/linux_python3.7.yaml (54%) rename dev/tasks/conda-recipes/{variants => .ci_support}/osx_python2.7.yaml (60%) rename dev/tasks/conda-recipes/{variants => .ci_support}/osx_python3.6.yaml (60%) rename dev/tasks/conda-recipes/{variants => .ci_support}/osx_python3.7.yaml (60%) rename dev/tasks/conda-recipes/{variants/win_c_compilervs2015cxx_compilervs2015python3.6vc14.yaml => .ci_support/win_c_compilervs2015cxx_compilervs2015python3.6.yaml} (52%) rename dev/tasks/conda-recipes/{variants/win_c_compilervs2015cxx_compilervs2015python3.7vc14.yaml => .ci_support/win_c_compilervs2015cxx_compilervs2015python3.7.yaml} (52%) create mode 100644 dev/tasks/conda-recipes/README.md create mode 100644 dev/tasks/conda-recipes/azure-pipelines.yml create mode 100644 dev/tasks/conda-recipes/azure-template-clone-arrow.yml create mode 100644 dev/tasks/conda-recipes/azure-template-upload-artifacts.yml diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 25b1ecd5dfc..e4f6496c75b 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -130,7 +130,8 @@ dev/tasks/linux-packages/debian/plasma-store-server.install dev/tasks/linux-packages/debian/rules dev/tasks/linux-packages/debian/source/format dev/tasks/linux-packages/debian/watch -dev/tasks/conda-recipes/variants/*.yaml +dev/tasks/conda-recipes/.azure-pipelines/* +dev/tasks/conda-recipes/.ci_support/* docs/requirements.txt go/arrow/go.sum go/arrow/Gopkg.lock diff --git a/dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-linux.yml b/dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-linux.yml new file mode 100755 index 00000000000..49f081ef6bd --- /dev/null +++ b/dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-linux.yml @@ -0,0 +1,31 @@ +# This file was generated automatically from conda-smithy. To update this configuration, +# update the conda-forge.yml and/or the recipe/meta.yaml. +# -*- mode: yaml -*- + +jobs: +- job: linux + pool: + vmImage: ubuntu-16.04 + timeoutInMinutes: 360 + ############ instead of strategy ############### + variables: + UPLOAD_PACKAGES: False + DOCKER_IMAGE: condaforge/linux-anvil-comp7 + steps: + ############### clone arrow and change directory ################# + - template: azure-template-clone-arrow.yml + + # configure qemu binfmt-misc running. This allows us to run docker containers + # embedded qemu-static + - script: | + docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes + ls /proc/sys/fs/binfmt_misc/ + condition: not(startsWith(variables['CONFIG'], 'linux_64')) + displayName: Configure binfmt_misc + + - script: | + export CI=azure + .azure-pipelines/run_docker_build.sh + displayName: Run docker build + env: + BINSTAR_TOKEN: $(BINSTAR_TOKEN) diff --git a/dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-osx.yml b/dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-osx.yml new file mode 100755 index 00000000000..d5c1df35bc1 --- /dev/null +++ b/dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-osx.yml @@ -0,0 +1,73 @@ +# This file was generated automatically from conda-smithy. To update this configuration, +# update the conda-forge.yml and/or the recipe/meta.yaml. +# -*- mode: yaml -*- + +jobs: +- job: osx + pool: + vmImage: macOS-10.13 + timeoutInMinutes: 360 + ############ instead of strategy ############### + variables: + UPLOAD_PACKAGES: False + steps: + ############### clone arrow and change directory ################# + - template: azure-template-clone-arrow.yml + + # TODO: Fast finish on azure pipelines? + - script: | + echo "Fast Finish" + + - script: | + echo "Removing homebrew from Azure to avoid conflicts." + curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/uninstall > ~/uninstall_homebrew + chmod +x ~/uninstall_homebrew + ~/uninstall_homebrew -fq + rm ~/uninstall_homebrew + displayName: Remove homebrew + + - bash: | + echo "##vso[task.prependpath]$CONDA/bin" + sudo chown -R $USER $CONDA + displayName: Add conda to PATH + + - script: | + source activate base + conda install -n base -c conda-forge --quiet --yes conda-forge-ci-setup=2 conda-build + displayName: 'Add conda-forge-ci-setup=2' + + - script: | + source activate base + echo "Configuring conda." + + setup_conda_rc ./ ./recipe ./.ci_support/${CONFIG}.yaml + export CI=azure + source run_conda_forge_build_setup + conda update --yes --quiet --override-channels -c conda-forge -c defaults --all + env: { + OSX_FORCE_SDK_DOWNLOAD: "1" + } + displayName: Configure conda and conda-build + + - script: | + source activate base + mangle_compiler ./ ./recipe ./.ci_support/${CONFIG}.yaml + displayName: Mangle compiler + + - script: | + source activate base + make_build_number ./ ./recipe ./.ci_support/${CONFIG}.yaml + displayName: Generate build number clobber file + + - script: | + source activate base + conda build ./recipe -m ./.ci_support/${CONFIG}.yaml --clobber-file ./.ci_support/clobber_${CONFIG}.yaml + displayName: Build recipe + + - script: | + source activate base + upload_package ./ ./recipe ./.ci_support/${CONFIG}.yaml + displayName: Upload recipe + env: + BINSTAR_TOKEN: $(BINSTAR_TOKEN) + condition: not(eq(variables['UPLOAD_PACKAGES'], 'False')) diff --git a/dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-win.yml b/dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-win.yml new file mode 100755 index 00000000000..48716963f71 --- /dev/null +++ b/dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-win.yml @@ -0,0 +1,100 @@ +# This file was generated automatically from conda-smithy. To update this configuration, +# update the conda-forge.yml and/or the recipe/meta.yaml. +# -*- mode: yaml -*- + +jobs: +- job: win + pool: + vmImage: vs2017-win2016 + timeoutInMinutes: 360 + ############ instead of strategy ############### + variables: + CONDA_BLD_PATH: D:\\bld\\ + UPLOAD_PACKAGES: False + steps: + ############### clone arrow and change directory ################# + - template: azure-template-clone-arrow.yml + + # TODO: Fast finish on azure pipelines? + - script: | + ECHO ON + + - script: | + choco install vcpython27 -fdv -y --debug + condition: contains(variables['CONFIG'], 'vs2008') + displayName: Install vcpython27.msi (if needed) + + # Cygwin's git breaks conda-build. (See https://github.com/conda-forge/conda-smithy-feedstock/pull/2.) + # - script: rmdir C:\cygwin /s /q + # continueOnError: true + + - powershell: | + Set-PSDebug -Trace 1 + + $batchcontent = @" + ECHO ON + SET vcpython=C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0 + + DIR "%vcpython%" + + CALL "%vcpython%\vcvarsall.bat" %* + "@ + + $batchDir = "C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\VC" + $batchPath = "$batchDir" + "\vcvarsall.bat" + New-Item -Path $batchPath -ItemType "file" -Force + + Set-Content -Value $batchcontent -Path $batchPath + + Get-ChildItem -Path $batchDir + + Get-ChildItem -Path ($batchDir + '\..') + + condition: contains(variables['CONFIG'], 'vs2008') + displayName: Patch vs2008 (if needed) + + - task: CondaEnvironment@1 + inputs: + packageSpecs: 'python=3.6 conda-build conda conda-forge::conda-forge-ci-setup=2' # Optional + installOptions: "-c conda-forge" + updateConda: false + displayName: Install conda-build and activate environment + + - script: set PYTHONUNBUFFERED=1 + + # Configure the VM + - script: setup_conda_rc .\ .\recipe .\.ci_support\%CONFIG%.yaml + + # Configure the VM. + - script: | + set "CI=azure" + run_conda_forge_build_setup + displayName: conda-forge build setup + + + - script: | + rmdir C:\strawberry /s /q + continueOnError: true + displayName: remove strawberryperl + + # Special cased version setting some more things! + - script: | + conda.exe build recipe -m .ci_support\%CONFIG%.yaml + displayName: Build recipe (vs2008) + env: + VS90COMNTOOLS: "C:\\Program Files (x86)\\Common Files\\Microsoft\\Visual C++ for Python\\9.0\\VC\\bin" + PYTHONUNBUFFERED: 1 + condition: contains(variables['CONFIG'], 'vs2008') + + - script: | + conda.exe build recipe -m .ci_support\%CONFIG%.yaml + displayName: Build recipe + env: + PYTHONUNBUFFERED: 1 + condition: not(contains(variables['CONFIG'], 'vs2008')) + + - script: | + upload_package .\ .\recipe .ci_support\%CONFIG%.yaml + env: + BINSTAR_TOKEN: $(BINSTAR_TOKEN) + condition: not(eq(variables['UPLOAD_PACKAGES'], 'False')) diff --git a/dev/tasks/conda-recipes/.azure-pipelines/build_steps.sh b/dev/tasks/conda-recipes/.azure-pipelines/build_steps.sh new file mode 100755 index 00000000000..8a4af44fecd --- /dev/null +++ b/dev/tasks/conda-recipes/.azure-pipelines/build_steps.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# PLEASE NOTE: This script has been automatically generated by conda-smithy. Any changes here +# will be lost next time ``conda smithy rerender`` is run. If you would like to make permanent +# changes to this script, consider a proposal to conda-smithy so that other feedstocks can also +# benefit from the improvement. + +set -xeuo pipefail +export PYTHONUNBUFFERED=1 +export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" +export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}" +export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" +export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml" + +cat >~/.condarc </dev/null && pwd )" +PROVIDER_DIR="$(basename $THISDIR)" + +FEEDSTOCK_ROOT=$(cd "$(dirname "$0")/.."; pwd;) +RECIPE_ROOT="${FEEDSTOCK_ROOT}/recipe" + +docker info + +# In order for the conda-build process in the container to write to the mounted +# volumes, we need to run with the same id as the host machine, which is +# normally the owner of the mounted volumes, or at least has write permission +export HOST_USER_ID=$(id -u) +# Check if docker-machine is being used (normally on OSX) and get the uid from +# the VM +if hash docker-machine 2> /dev/null && docker-machine active > /dev/null; then + export HOST_USER_ID=$(docker-machine ssh $(docker-machine active) id -u) +fi + +ARTIFACTS="$FEEDSTOCK_ROOT/build_artifacts" + +if [ -z "$CONFIG" ]; then + set +x + FILES=`ls .ci_support/linux_*` + CONFIGS="" + for file in $FILES; do + CONFIGS="${CONFIGS}'${file:12:-5}' or "; + done + echo "Need to set CONFIG env variable. Value can be one of ${CONFIGS:0:-4}" + exit 1 +fi + +if [ -z "${DOCKER_IMAGE}" ]; then + SHYAML_INSTALLED="$(shyaml --version || echo NO)" + if [ "${SHYAML_INSTALLED}" == "NO" ]; then + echo "WARNING: DOCKER_IMAGE variable not set and shyaml not installed. Falling back to condaforge/linux-anvil-comp7" + DOCKER_IMAGE="condaforge/linux-anvil-comp7" + else + DOCKER_IMAGE="$(cat "${FEEDSTOCK_ROOT}/.ci_support/${CONFIG}.yaml" | shyaml get-value docker_image.0 condaforge/linux-anvil-comp7 )" + fi +fi + +mkdir -p "$ARTIFACTS" +DONE_CANARY="$ARTIFACTS/conda-forge-build-done-${CONFIG}" +rm -f "$DONE_CANARY" + +if [ -z "${CI}" ]; then + DOCKER_RUN_ARGS="-it " +fi + +export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" +docker run ${DOCKER_RUN_ARGS} \ + -v "${RECIPE_ROOT}":/home/conda/recipe_root:ro,z \ + -v "${FEEDSTOCK_ROOT}":/home/conda/feedstock_root:rw,z \ + -e CONFIG \ + -e BINSTAR_TOKEN \ + -e HOST_USER_ID \ + -e UPLOAD_PACKAGES \ + -e CI \ + $DOCKER_IMAGE \ + bash \ + /home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh + +# verify that the end of the script was reached +test -f "$DONE_CANARY" \ No newline at end of file diff --git a/dev/tasks/conda-recipes/variants/linux_python2.7.yaml b/dev/tasks/conda-recipes/.ci_support/linux_python2.7.yaml similarity index 54% rename from dev/tasks/conda-recipes/variants/linux_python2.7.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_python2.7.yaml index d92357b805d..8757d949095 100644 --- a/dev/tasks/conda-recipes/variants/linux_python2.7.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_python2.7.yaml @@ -1,40 +1,24 @@ boost_cpp: -- 1.68.0 +- 1.70.0 c_compiler: - gcc +c_compiler_version: +- '7' channel_sources: - conda-forge,defaults channel_targets: - conda-forge main cxx_compiler: - gxx +cxx_compiler_version: +- '7' docker_image: - condaforge/linux-anvil-comp7 -libprotobuf: -- '3.7' -lz4_c: -- 1.8.1 pin_run_as_build: boost-cpp: max_pin: x.x.x - libprotobuf: - max_pin: x.x - lz4-c: - max_pin: x.x.x python: min_pin: x.x max_pin: x.x - snappy: - max_pin: x.x.x - zlib: - max_pin: x.x - zstd: - max_pin: x.x.x python: - '2.7' -snappy: -- 1.1.7 -zlib: -- '1.2' -zstd: -- 1.3.3 diff --git a/dev/tasks/conda-recipes/variants/linux_python3.6.yaml b/dev/tasks/conda-recipes/.ci_support/linux_python3.6.yaml similarity index 54% rename from dev/tasks/conda-recipes/variants/linux_python3.6.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_python3.6.yaml index a934bd1687a..6af2d070c75 100644 --- a/dev/tasks/conda-recipes/variants/linux_python3.6.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_python3.6.yaml @@ -1,40 +1,24 @@ boost_cpp: -- 1.68.0 +- 1.70.0 c_compiler: - gcc +c_compiler_version: +- '7' channel_sources: - conda-forge,defaults channel_targets: - conda-forge main cxx_compiler: - gxx +cxx_compiler_version: +- '7' docker_image: - condaforge/linux-anvil-comp7 -libprotobuf: -- '3.7' -lz4_c: -- 1.8.1 pin_run_as_build: boost-cpp: max_pin: x.x.x - libprotobuf: - max_pin: x.x - lz4-c: - max_pin: x.x.x python: min_pin: x.x max_pin: x.x - snappy: - max_pin: x.x.x - zlib: - max_pin: x.x - zstd: - max_pin: x.x.x python: - '3.6' -snappy: -- 1.1.7 -zlib: -- '1.2' -zstd: -- 1.3.3 diff --git a/dev/tasks/conda-recipes/variants/linux_python3.7.yaml b/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml similarity index 54% rename from dev/tasks/conda-recipes/variants/linux_python3.7.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml index 736cd01ba26..9022d934233 100644 --- a/dev/tasks/conda-recipes/variants/linux_python3.7.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml @@ -1,40 +1,24 @@ boost_cpp: -- 1.68.0 +- 1.70.0 c_compiler: - gcc +c_compiler_version: +- '7' channel_sources: - conda-forge,defaults channel_targets: - conda-forge main cxx_compiler: - gxx +cxx_compiler_version: +- '7' docker_image: - condaforge/linux-anvil-comp7 -libprotobuf: -- '3.7' -lz4_c: -- 1.8.1 pin_run_as_build: boost-cpp: max_pin: x.x.x - libprotobuf: - max_pin: x.x - lz4-c: - max_pin: x.x.x python: min_pin: x.x max_pin: x.x - snappy: - max_pin: x.x.x - zlib: - max_pin: x.x - zstd: - max_pin: x.x.x python: - '3.7' -snappy: -- 1.1.7 -zlib: -- '1.2' -zstd: -- 1.3.3 diff --git a/dev/tasks/conda-recipes/variants/osx_python2.7.yaml b/dev/tasks/conda-recipes/.ci_support/osx_python2.7.yaml similarity index 60% rename from dev/tasks/conda-recipes/variants/osx_python2.7.yaml rename to dev/tasks/conda-recipes/.ci_support/osx_python2.7.yaml index 2817157c236..771f49ffc71 100644 --- a/dev/tasks/conda-recipes/variants/osx_python2.7.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_python2.7.yaml @@ -1,19 +1,19 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' boost_cpp: -- 1.68.0 +- 1.70.0 c_compiler: - clang +c_compiler_version: +- '4' channel_sources: - conda-forge,defaults channel_targets: - conda-forge main cxx_compiler: - clangxx -libprotobuf: -- '3.7' -lz4_c: -- 1.8.1 +cxx_compiler_version: +- '4' macos_machine: - x86_64-apple-darwin13.4.0 macos_min_version: @@ -21,24 +21,8 @@ macos_min_version: pin_run_as_build: boost-cpp: max_pin: x.x.x - libprotobuf: - max_pin: x.x - lz4-c: - max_pin: x.x.x python: min_pin: x.x max_pin: x.x - snappy: - max_pin: x.x.x - zlib: - max_pin: x.x - zstd: - max_pin: x.x.x python: - '2.7' -snappy: -- 1.1.7 -zlib: -- '1.2' -zstd: -- 1.3.3 diff --git a/dev/tasks/conda-recipes/variants/osx_python3.6.yaml b/dev/tasks/conda-recipes/.ci_support/osx_python3.6.yaml similarity index 60% rename from dev/tasks/conda-recipes/variants/osx_python3.6.yaml rename to dev/tasks/conda-recipes/.ci_support/osx_python3.6.yaml index 5e87a2d7ec9..ed2e5a0fb4a 100644 --- a/dev/tasks/conda-recipes/variants/osx_python3.6.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_python3.6.yaml @@ -1,19 +1,19 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' boost_cpp: -- 1.68.0 +- 1.70.0 c_compiler: - clang +c_compiler_version: +- '4' channel_sources: - conda-forge,defaults channel_targets: - conda-forge main cxx_compiler: - clangxx -libprotobuf: -- '3.7' -lz4_c: -- 1.8.1 +cxx_compiler_version: +- '4' macos_machine: - x86_64-apple-darwin13.4.0 macos_min_version: @@ -21,24 +21,8 @@ macos_min_version: pin_run_as_build: boost-cpp: max_pin: x.x.x - libprotobuf: - max_pin: x.x - lz4-c: - max_pin: x.x.x python: min_pin: x.x max_pin: x.x - snappy: - max_pin: x.x.x - zlib: - max_pin: x.x - zstd: - max_pin: x.x.x python: - '3.6' -snappy: -- 1.1.7 -zlib: -- '1.2' -zstd: -- 1.3.3 diff --git a/dev/tasks/conda-recipes/variants/osx_python3.7.yaml b/dev/tasks/conda-recipes/.ci_support/osx_python3.7.yaml similarity index 60% rename from dev/tasks/conda-recipes/variants/osx_python3.7.yaml rename to dev/tasks/conda-recipes/.ci_support/osx_python3.7.yaml index 631716d23ed..5a0946c6e15 100644 --- a/dev/tasks/conda-recipes/variants/osx_python3.7.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_python3.7.yaml @@ -1,19 +1,19 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' boost_cpp: -- 1.68.0 +- 1.70.0 c_compiler: - clang +c_compiler_version: +- '4' channel_sources: - conda-forge,defaults channel_targets: - conda-forge main cxx_compiler: - clangxx -libprotobuf: -- '3.7' -lz4_c: -- 1.8.1 +cxx_compiler_version: +- '4' macos_machine: - x86_64-apple-darwin13.4.0 macos_min_version: @@ -21,24 +21,8 @@ macos_min_version: pin_run_as_build: boost-cpp: max_pin: x.x.x - libprotobuf: - max_pin: x.x - lz4-c: - max_pin: x.x.x python: min_pin: x.x max_pin: x.x - snappy: - max_pin: x.x.x - zlib: - max_pin: x.x - zstd: - max_pin: x.x.x python: - '3.7' -snappy: -- 1.1.7 -zlib: -- '1.2' -zstd: -- 1.3.3 diff --git a/dev/tasks/conda-recipes/variants/win_c_compilervs2015cxx_compilervs2015python3.6vc14.yaml b/dev/tasks/conda-recipes/.ci_support/win_c_compilervs2015cxx_compilervs2015python3.6.yaml similarity index 52% rename from dev/tasks/conda-recipes/variants/win_c_compilervs2015cxx_compilervs2015python3.6vc14.yaml rename to dev/tasks/conda-recipes/.ci_support/win_c_compilervs2015cxx_compilervs2015python3.6.yaml index b87a06eb3ab..9961c53449e 100644 --- a/dev/tasks/conda-recipes/variants/win_c_compilervs2015cxx_compilervs2015python3.6vc14.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_c_compilervs2015cxx_compilervs2015python3.6.yaml @@ -1,5 +1,5 @@ boost_cpp: -- 1.68.0 +- 1.70.0 c_compiler: - vs2015 channel_sources: @@ -8,40 +8,15 @@ channel_targets: - conda-forge main cxx_compiler: - vs2015 -libprotobuf: -- '3.7' -lz4_c: -- 1.8.1 pin_run_as_build: boost-cpp: max_pin: x.x.x - libprotobuf: - max_pin: x.x - lz4-c: - max_pin: x.x.x python: min_pin: x.x max_pin: x.x - snappy: - max_pin: x.x.x - vc: - max_pin: x - zlib: - max_pin: x.x - zstd: - max_pin: x.x.x python: - '3.6' -snappy: -- 1.1.7 -vc: -- '14' zip_keys: - - python - - vc - c_compiler - cxx_compiler -zlib: -- '1.2' -zstd: -- 1.3.3 diff --git a/dev/tasks/conda-recipes/variants/win_c_compilervs2015cxx_compilervs2015python3.7vc14.yaml b/dev/tasks/conda-recipes/.ci_support/win_c_compilervs2015cxx_compilervs2015python3.7.yaml similarity index 52% rename from dev/tasks/conda-recipes/variants/win_c_compilervs2015cxx_compilervs2015python3.7vc14.yaml rename to dev/tasks/conda-recipes/.ci_support/win_c_compilervs2015cxx_compilervs2015python3.7.yaml index 5c710ffa5e7..77abc37c4d2 100644 --- a/dev/tasks/conda-recipes/variants/win_c_compilervs2015cxx_compilervs2015python3.7vc14.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_c_compilervs2015cxx_compilervs2015python3.7.yaml @@ -1,5 +1,5 @@ boost_cpp: -- 1.68.0 +- 1.70.0 c_compiler: - vs2015 channel_sources: @@ -8,40 +8,15 @@ channel_targets: - conda-forge main cxx_compiler: - vs2015 -libprotobuf: -- '3.7' -lz4_c: -- 1.8.1 pin_run_as_build: boost-cpp: max_pin: x.x.x - libprotobuf: - max_pin: x.x - lz4-c: - max_pin: x.x.x python: min_pin: x.x max_pin: x.x - snappy: - max_pin: x.x.x - vc: - max_pin: x - zlib: - max_pin: x.x - zstd: - max_pin: x.x.x python: - '3.7' -snappy: -- 1.1.7 -vc: -- '14' zip_keys: - - python - - vc - c_compiler - cxx_compiler -zlib: -- '1.2' -zstd: -- 1.3.3 diff --git a/dev/tasks/conda-recipes/README.md b/dev/tasks/conda-recipes/README.md new file mode 100644 index 00000000000..2041fb7ec26 --- /dev/null +++ b/dev/tasks/conda-recipes/README.md @@ -0,0 +1,69 @@ + + +# Conda Forge recipes + +This directory must be migrated periodically with the upstrem updates of +[arrow-cpp-feedstock][arrow-cpp-feedsotkc], +[parquet-cpp-feedstock][parquet-cpp-feedstock] and +[pyarrow-feedstock][pyarrow-feedstock] +conda-forge repositories because of multiple vendored files. + +## Keeping the recipes synchronized + +The recipes here are tested on nightly basis, so they follow the development +versions of arrow instead of the upstream recipes, which are suitable for the +latest releases. + +### Backporting from the upstream feedstocks + +So most of the cases these recipes are more accurate, altough the upstream +feedstock regurarly receive automatic updates by the conda-forge team, and +we need to backport those changes to the crossbow recipes. Most of these +updates are touching the version pinning files (``.ci_support`) and other CI +related configuration files. + +Because all three recipes must be built in the same continuous integration +job prefer porting from the [pyarrow feedstock][pyarrow-feedstock]. + +#### Updating the variants: + +Copy the configuration files from `pyarrow-feedstock/.ci_support` to the +`.ci_support` folder. + +#### Updating the CI configurations: + +The `.azure-pipelines/azure-pipelines-[linux|osx|win].yml` should be ported +to the local counterparts under `.azure-pipelines` with keeping the crossbow +related parts (the cloning of arrow and the jinja templated variables) and +moving the matrix definitions like [this][matrix-definition] to the crossbow +[tasks.yml][../tasks.yml] config file. + + +### Porting recipes from crossbow to the upstream feedstocks + +Theoretically these recipes should be up to date with the actual version of +Arrow, so during the release procedure the content of these recipes should be +copied to the upstream feedstocks. + + +[arrow-cpp-feedstock]: https://github.com/conda-forge/arrow-cpp-feedstock +[parquet-cpp-feedstock]: https://github.com/conda-forge/parquet-cpp-feedstock +[pyarrow-cpp-feedstock]: https://github.com/conda-forge/pyarrow-feedstock +[matric-definition]: https://github.com/conda-forge/pyarrow-feedstock/blob/master/.azure-pipelines/azure-pipelines-linux.yml#L12 diff --git a/dev/tasks/conda-recipes/azure-pipelines.yml b/dev/tasks/conda-recipes/azure-pipelines.yml new file mode 100644 index 00000000000..c7c9c9be7dd --- /dev/null +++ b/dev/tasks/conda-recipes/azure-pipelines.yml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +parameters: + # these parameters are used by the azure-template-*.yml files + arrow_remote: {{ arrow.remote }} + arrow_branch: {{ arrow.branch }} + arrow_head: {{ arrow.head }} + +variables: + # vairables are used by .azure-pipelines/azure-pipelines-[linux|osx|win].yml + CONFIG: {{ config }} + +jobs: + - template: ./.azure-pipelines/azure-pipelines-{{ platform }}.yml diff --git a/dev/tasks/conda-recipes/azure-template-clone-arrow.yml b/dev/tasks/conda-recipes/azure-template-clone-arrow.yml new file mode 100644 index 00000000000..f11d30aedab --- /dev/null +++ b/dev/tasks/conda-recipes/azure-template-clone-arrow.yml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +steps: + - script: | + git clone --no-checkout ${{ parameters.arrow_remote }} arrow + git -C arrow fetch -t ${{ parameters.arrow_remote }} ${{ parameters.arrow_branch }} + git -C arrow checkout ${{ parameters.arrow_head }} + pushd arrow/dev/tasks/conda-recipes diff --git a/dev/tasks/conda-recipes/azure-template-upload-artifacts.yml b/dev/tasks/conda-recipes/azure-template-upload-artifacts.yml new file mode 100644 index 00000000000..13a83393a91 --- /dev/null +++ b/dev/tasks/conda-recipes/azure-template-upload-artifacts.yml @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py index f7518d0f0d5..db626552fa4 100755 --- a/dev/tasks/crossbow.py +++ b/dev/tasks/crossbow.py @@ -106,6 +106,7 @@ def unflatten(mapping): _default_tree = { '.travis.yml': _default_travis_yml, '.circleci/config.yml': _default_circle_yml + # azure-pipelines.yml' TODO? } @@ -548,7 +549,7 @@ class Task: def __init__(self, platform, ci, template, artifacts=None, params=None): assert platform in {'win', 'osx', 'linux'} - assert ci in {'circle', 'travis', 'appveyor'} + assert ci in {'circle', 'travis', 'appveyor', 'azure'} self.ci = ci self.platform = platform self.template = template @@ -573,7 +574,8 @@ def filename(self): config_files = { 'circle': '.circleci/config.yml', 'travis': '.travis.yml', - 'appveyor': 'appveyor.yml' + 'appveyor': 'appveyor.yml', + 'azure': 'azure-pipelines.yml' } return config_files[self.ci] diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 875216cc713..ce2b980f53a 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -72,9 +72,9 @@ tasks: ############################## Conda Linux ################################## conda-linux-gcc-py27: - ci: travis + ci: azure platform: linux - template: conda-recipes/travis.linux.yml + template: conda-recipes/azure-pipelines.yml params: config: linux_python2.7 artifacts: @@ -82,9 +82,9 @@ tasks: - pyarrow-{no_rc_version}-py27(h[a-z0-9]+)_0.tar.bz2 conda-linux-gcc-py36: - ci: travis + ci: azure platform: linux - template: conda-recipes/travis.linux.yml + template: conda-recipes/azure-pipelines.yml params: config: linux_python3.6 artifacts: @@ -92,9 +92,9 @@ tasks: - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0.tar.bz2 conda-linux-gcc-py37: - ci: travis + ci: azure platform: linux - template: conda-recipes/travis.linux.yml + template: conda-recipes/azure-pipelines.yml params: config: linux_python3.7 artifacts: @@ -104,9 +104,9 @@ tasks: ############################## Conda OSX #################################### conda-osx-clang-py27: - ci: travis + ci: azure platform: osx - template: conda-recipes/travis.osx.yml + template: conda-recipes/azure-pipelines.yml params: config: osx_python2.7 artifacts: @@ -114,9 +114,9 @@ tasks: - pyarrow-{no_rc_version}-py27(h[a-z0-9]+)_0.tar.bz2 conda-osx-clang-py36: - ci: travis + ci: azure platform: osx - template: conda-recipes/travis.osx.yml + template: conda-recipes/azure-pipelines.yml params: config: osx_python3.6 artifacts: @@ -124,8 +124,9 @@ tasks: - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0.tar.bz2 conda-osx-clang-py37: + ci: azure platform: osx - template: conda-recipes/travis.osx.yml + template: conda-recipes/azure-pipelines.yml params: config: osx_python3.7 artifacts: @@ -135,21 +136,21 @@ tasks: ############################## Conda Windows ################################ conda-win-vs2015-py36: - ci: appveyor + ci: azure platform: win - template: conda-recipes/appveyor.yml + template: conda-recipes/azure-pipelines.yml params: - config: win_c_compilervs2015cxx_compilervs2015python3.6vc14 + config: win_c_compilervs2015cxx_compilervs2015python3.6 artifacts: - arrow-cpp-{no_rc_version}-py36_vc14(h[a-z0-9]+)_0.tar.bz2 - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0.tar.bz2 conda-win-vs2015-py37: - ci: appveyor + ci: azure platform: win - template: conda-recipes/appveyor.yml + template: conda-recipes/azure-pipelines.yml params: - config: win_c_compilervs2015cxx_compilervs2015python3.7vc14 + config: win_c_compilervs2015cxx_compilervs2015python3.7 artifacts: - arrow-cpp-{no_rc_version}-py37_vc14(h[a-z0-9]+)_0.tar.bz2 - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0.tar.bz2 From b2fd21a24327abe50207121166970d4df44ab675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 21 Jun 2019 15:31:49 +0200 Subject: [PATCH 02/39] use variables [skip ci]] --- dev/tasks/conda-recipes/azure-pipelines.yml | 12 +++++------- .../conda-recipes/azure-template-clone-arrow.yml | 8 ++++---- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/dev/tasks/conda-recipes/azure-pipelines.yml b/dev/tasks/conda-recipes/azure-pipelines.yml index c7c9c9be7dd..4e8245bca88 100644 --- a/dev/tasks/conda-recipes/azure-pipelines.yml +++ b/dev/tasks/conda-recipes/azure-pipelines.yml @@ -15,15 +15,13 @@ # specific language governing permissions and limitations # under the License. -parameters: - # these parameters are used by the azure-template-*.yml files - arrow_remote: {{ arrow.remote }} - arrow_branch: {{ arrow.branch }} - arrow_head: {{ arrow.head }} - variables: # vairables are used by .azure-pipelines/azure-pipelines-[linux|osx|win].yml + # and azure-template-*.yml files CONFIG: {{ config }} + arrow_remote: {{ arrow.remote }} + arrow_branch: {{ arrow.branch }} + arrow_head: {{ arrow.head }} jobs: - - template: ./.azure-pipelines/azure-pipelines-{{ platform }}.yml + - template: ./.azure-pipelines/azure-pipelines-{{ task.platform }}.yml diff --git a/dev/tasks/conda-recipes/azure-template-clone-arrow.yml b/dev/tasks/conda-recipes/azure-template-clone-arrow.yml index f11d30aedab..0e48d750420 100644 --- a/dev/tasks/conda-recipes/azure-template-clone-arrow.yml +++ b/dev/tasks/conda-recipes/azure-template-clone-arrow.yml @@ -17,7 +17,7 @@ steps: - script: | - git clone --no-checkout ${{ parameters.arrow_remote }} arrow - git -C arrow fetch -t ${{ parameters.arrow_remote }} ${{ parameters.arrow_branch }} - git -C arrow checkout ${{ parameters.arrow_head }} - pushd arrow/dev/tasks/conda-recipes + git clone --no-checkout ${{ variables.arrow_remote }} arrow + git -C arrow fetch -t ${{ variables.arrow_remote }} ${{ variables.arrow_branch }} + git -C arrow checkout ${{ variables.arrow_head }} + cd arrow/dev/tasks/conda-recipes From 53e8eb24fa261d73e08b17b7c535319112e2ee0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 21 Jun 2019 15:44:50 +0200 Subject: [PATCH 03/39] don't use azure templates [skip ci] --- LICENSE.txt | 2 +- dev/release/rat_exclude_files.txt | 3 +-- dev/tasks/conda-recipes/appveyor.yml | 17 ------------ dev/tasks/conda-recipes/arrow-cpp/bld.bat | 17 ------------ dev/tasks/conda-recipes/arrow-cpp/build.sh | 19 ------------- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 17 ------------ dev/tasks/conda-recipes/azure-pipelines.yml | 27 ------------------- .../azure-template-clone-arrow.yml | 23 ---------------- .../azure-template-upload-artifacts.yml | 16 ----------- ...re-pipelines-linux.yml => azure.linux.yml} | 17 ++++++------ .../azure-pipelines-osx.yml => azure.osx.yml} | 4 --- .../azure-pipelines-win.yml => azure.win.yml} | 4 --- .../{.azure-pipelines => }/build_steps.sh | 0 dev/tasks/conda-recipes/parquet-cpp/meta.yaml | 17 ------------ dev/tasks/conda-recipes/pyarrow/bld.bat | 17 ------------ dev/tasks/conda-recipes/pyarrow/build.sh | 17 ------------ dev/tasks/conda-recipes/pyarrow/meta.yaml | 17 ------------ .../run_docker_build.sh | 0 dev/tasks/conda-recipes/travis.linux.yml | 17 ------------ dev/tasks/conda-recipes/travis.osx.yml | 17 ------------ dev/tasks/tasks.yml | 16 +++++------ 21 files changed, 19 insertions(+), 265 deletions(-) delete mode 100644 dev/tasks/conda-recipes/azure-pipelines.yml delete mode 100644 dev/tasks/conda-recipes/azure-template-clone-arrow.yml delete mode 100644 dev/tasks/conda-recipes/azure-template-upload-artifacts.yml rename dev/tasks/conda-recipes/{.azure-pipelines/azure-pipelines-linux.yml => azure.linux.yml} (63%) rename dev/tasks/conda-recipes/{.azure-pipelines/azure-pipelines-osx.yml => azure.osx.yml} (92%) rename dev/tasks/conda-recipes/{.azure-pipelines/azure-pipelines-win.yml => azure.win.yml} (94%) rename dev/tasks/conda-recipes/{.azure-pipelines => }/build_steps.sh (100%) rename dev/tasks/conda-recipes/{.azure-pipelines => }/run_docker_build.sh (100%) diff --git a/LICENSE.txt b/LICENSE.txt index 1be16cc12e0..df5f5cce783 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -740,7 +740,7 @@ You can contact the author at : -------------------------------------------------------------------------------- -The files in dev/tasks/conda-recipes/variants have the following license +The files under dev/tasks/conda-recipes have the following license BSD 3-clause license Copyright (c) 2015-2018, conda-forge diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index e4f6496c75b..1d7973d2ffc 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -130,8 +130,7 @@ dev/tasks/linux-packages/debian/plasma-store-server.install dev/tasks/linux-packages/debian/rules dev/tasks/linux-packages/debian/source/format dev/tasks/linux-packages/debian/watch -dev/tasks/conda-recipes/.azure-pipelines/* -dev/tasks/conda-recipes/.ci_support/* +dev/tasks/conda-recipes/* docs/requirements.txt go/arrow/go.sum go/arrow/Gopkg.lock diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.yml index 5d677c63bfe..8ad63f48af4 100644 --- a/dev/tasks/conda-recipes/appveyor.yml +++ b/dev/tasks/conda-recipes/appveyor.yml @@ -1,20 +1,3 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - environment: ARROW_VERSION: {{ arrow.no_rc_version }} # regardless of the python version we build against diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld.bat b/dev/tasks/conda-recipes/arrow-cpp/bld.bat index c853a63d63c..429cb642d59 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld.bat @@ -1,20 +1,3 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. - mkdir "%SRC_DIR%"\cpp\build pushd "%SRC_DIR%"\cpp\build diff --git a/dev/tasks/conda-recipes/arrow-cpp/build.sh b/dev/tasks/conda-recipes/arrow-cpp/build.sh index dfe992ef345..3d307096745 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/build.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build.sh @@ -1,22 +1,3 @@ -#!/bin/sh - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - set -e set -x diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 15fc9ea9203..ff367383b59 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -1,20 +1,3 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - package: name: arrow-cpp version: {{ ARROW_VERSION }} diff --git a/dev/tasks/conda-recipes/azure-pipelines.yml b/dev/tasks/conda-recipes/azure-pipelines.yml deleted file mode 100644 index 4e8245bca88..00000000000 --- a/dev/tasks/conda-recipes/azure-pipelines.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -variables: - # vairables are used by .azure-pipelines/azure-pipelines-[linux|osx|win].yml - # and azure-template-*.yml files - CONFIG: {{ config }} - arrow_remote: {{ arrow.remote }} - arrow_branch: {{ arrow.branch }} - arrow_head: {{ arrow.head }} - -jobs: - - template: ./.azure-pipelines/azure-pipelines-{{ task.platform }}.yml diff --git a/dev/tasks/conda-recipes/azure-template-clone-arrow.yml b/dev/tasks/conda-recipes/azure-template-clone-arrow.yml deleted file mode 100644 index 0e48d750420..00000000000 --- a/dev/tasks/conda-recipes/azure-template-clone-arrow.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -steps: - - script: | - git clone --no-checkout ${{ variables.arrow_remote }} arrow - git -C arrow fetch -t ${{ variables.arrow_remote }} ${{ variables.arrow_branch }} - git -C arrow checkout ${{ variables.arrow_head }} - cd arrow/dev/tasks/conda-recipes diff --git a/dev/tasks/conda-recipes/azure-template-upload-artifacts.yml b/dev/tasks/conda-recipes/azure-template-upload-artifacts.yml deleted file mode 100644 index 13a83393a91..00000000000 --- a/dev/tasks/conda-recipes/azure-template-upload-artifacts.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-linux.yml b/dev/tasks/conda-recipes/azure.linux.yml similarity index 63% rename from dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-linux.yml rename to dev/tasks/conda-recipes/azure.linux.yml index 49f081ef6bd..c22b88b1200 100755 --- a/dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -1,19 +1,20 @@ -# This file was generated automatically from conda-smithy. To update this configuration, -# update the conda-forge.yml and/or the recipe/meta.yaml. -# -*- mode: yaml -*- - jobs: - job: linux pool: vmImage: ubuntu-16.04 timeoutInMinutes: 360 - ############ instead of strategy ############### + + ############### crossbow specific ################# variables: UPLOAD_PACKAGES: False DOCKER_IMAGE: condaforge/linux-anvil-comp7 steps: - ############### clone arrow and change directory ################# - - template: azure-template-clone-arrow.yml + - script: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout {{ arrow.head }} + cd arrow/dev/tasks/conda-recipes + ############ crossbow specific ############### # configure qemu binfmt-misc running. This allows us to run docker containers # embedded qemu-static @@ -25,7 +26,7 @@ jobs: - script: | export CI=azure - .azure-pipelines/run_docker_build.sh + run_docker_build.sh displayName: Run docker build env: BINSTAR_TOKEN: $(BINSTAR_TOKEN) diff --git a/dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-osx.yml b/dev/tasks/conda-recipes/azure.osx.yml similarity index 92% rename from dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-osx.yml rename to dev/tasks/conda-recipes/azure.osx.yml index d5c1df35bc1..d51d70222d7 100755 --- a/dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -1,7 +1,3 @@ -# This file was generated automatically from conda-smithy. To update this configuration, -# update the conda-forge.yml and/or the recipe/meta.yaml. -# -*- mode: yaml -*- - jobs: - job: osx pool: diff --git a/dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-win.yml b/dev/tasks/conda-recipes/azure.win.yml similarity index 94% rename from dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-win.yml rename to dev/tasks/conda-recipes/azure.win.yml index 48716963f71..01b0eb80aed 100755 --- a/dev/tasks/conda-recipes/.azure-pipelines/azure-pipelines-win.yml +++ b/dev/tasks/conda-recipes/azure.win.yml @@ -1,7 +1,3 @@ -# This file was generated automatically from conda-smithy. To update this configuration, -# update the conda-forge.yml and/or the recipe/meta.yaml. -# -*- mode: yaml -*- - jobs: - job: win pool: diff --git a/dev/tasks/conda-recipes/.azure-pipelines/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh similarity index 100% rename from dev/tasks/conda-recipes/.azure-pipelines/build_steps.sh rename to dev/tasks/conda-recipes/build_steps.sh diff --git a/dev/tasks/conda-recipes/parquet-cpp/meta.yaml b/dev/tasks/conda-recipes/parquet-cpp/meta.yaml index eebedd801ca..5eb999b5bd7 100644 --- a/dev/tasks/conda-recipes/parquet-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/parquet-cpp/meta.yaml @@ -1,20 +1,3 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - # ARROW-3229: this is a meta-package to prevent conflicts in the future {% set parquet_version = "1.5.1" %} diff --git a/dev/tasks/conda-recipes/pyarrow/bld.bat b/dev/tasks/conda-recipes/pyarrow/bld.bat index 37d7f188056..d02ceaffe48 100644 --- a/dev/tasks/conda-recipes/pyarrow/bld.bat +++ b/dev/tasks/conda-recipes/pyarrow/bld.bat @@ -1,20 +1,3 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. - pushd "%SRC_DIR%"\python @rem the symlinks for cmake modules don't work here diff --git a/dev/tasks/conda-recipes/pyarrow/build.sh b/dev/tasks/conda-recipes/pyarrow/build.sh index 56d6fa7d946..98b10a42e59 100644 --- a/dev/tasks/conda-recipes/pyarrow/build.sh +++ b/dev/tasks/conda-recipes/pyarrow/build.sh @@ -1,22 +1,5 @@ #!/bin/sh -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - set -e set -x diff --git a/dev/tasks/conda-recipes/pyarrow/meta.yaml b/dev/tasks/conda-recipes/pyarrow/meta.yaml index 4c8f6560b8a..8198390ac53 100644 --- a/dev/tasks/conda-recipes/pyarrow/meta.yaml +++ b/dev/tasks/conda-recipes/pyarrow/meta.yaml @@ -1,20 +1,3 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - package: name: pyarrow version: {{ ARROW_VERSION }} diff --git a/dev/tasks/conda-recipes/.azure-pipelines/run_docker_build.sh b/dev/tasks/conda-recipes/run_docker_build.sh similarity index 100% rename from dev/tasks/conda-recipes/.azure-pipelines/run_docker_build.sh rename to dev/tasks/conda-recipes/run_docker_build.sh diff --git a/dev/tasks/conda-recipes/travis.linux.yml b/dev/tasks/conda-recipes/travis.linux.yml index 9e9f94f4a6a..ba5143f210c 100644 --- a/dev/tasks/conda-recipes/travis.linux.yml +++ b/dev/tasks/conda-recipes/travis.linux.yml @@ -1,20 +1,3 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - os: linux dist: trusty language: generic diff --git a/dev/tasks/conda-recipes/travis.osx.yml b/dev/tasks/conda-recipes/travis.osx.yml index 3093cac9d77..956034db894 100644 --- a/dev/tasks/conda-recipes/travis.osx.yml +++ b/dev/tasks/conda-recipes/travis.osx.yml @@ -1,20 +1,3 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - os: osx osx_image: xcode9.4 language: generic diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index ce2b980f53a..1dfe95f3c18 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -74,7 +74,7 @@ tasks: conda-linux-gcc-py27: ci: azure platform: linux - template: conda-recipes/azure-pipelines.yml + template: conda-recipes/azure.linux.yml params: config: linux_python2.7 artifacts: @@ -84,7 +84,7 @@ tasks: conda-linux-gcc-py36: ci: azure platform: linux - template: conda-recipes/azure-pipelines.yml + template: conda-recipes/azure.linux.yml params: config: linux_python3.6 artifacts: @@ -94,7 +94,7 @@ tasks: conda-linux-gcc-py37: ci: azure platform: linux - template: conda-recipes/azure-pipelines.yml + template: conda-recipes/azure.linux.yml params: config: linux_python3.7 artifacts: @@ -106,7 +106,7 @@ tasks: conda-osx-clang-py27: ci: azure platform: osx - template: conda-recipes/azure-pipelines.yml + template: conda-recipes/azure.osx.yml params: config: osx_python2.7 artifacts: @@ -116,7 +116,7 @@ tasks: conda-osx-clang-py36: ci: azure platform: osx - template: conda-recipes/azure-pipelines.yml + template: conda-recipes/azure.osx.yml params: config: osx_python3.6 artifacts: @@ -126,7 +126,7 @@ tasks: conda-osx-clang-py37: ci: azure platform: osx - template: conda-recipes/azure-pipelines.yml + template: conda-recipes/azure.osx.yml params: config: osx_python3.7 artifacts: @@ -138,7 +138,7 @@ tasks: conda-win-vs2015-py36: ci: azure platform: win - template: conda-recipes/azure-pipelines.yml + template: conda-recipes/azure.win.yml params: config: win_c_compilervs2015cxx_compilervs2015python3.6 artifacts: @@ -148,7 +148,7 @@ tasks: conda-win-vs2015-py37: ci: azure platform: win - template: conda-recipes/azure-pipelines.yml + template: conda-recipes/azure.win.yml params: config: win_c_compilervs2015cxx_compilervs2015python3.7 artifacts: From b2425e6502cb12dea8770ec671cf721cd60de94e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 21 Jun 2019 15:59:20 +0200 Subject: [PATCH 04/39] fix working directory [skip ci] --- dev/tasks/conda-recipes/azure.linux.yml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index c22b88b1200..7d659d9ffcc 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -4,18 +4,11 @@ jobs: vmImage: ubuntu-16.04 timeoutInMinutes: 360 - ############### crossbow specific ################# variables: UPLOAD_PACKAGES: False DOCKER_IMAGE: condaforge/linux-anvil-comp7 - steps: - - script: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout {{ arrow.head }} - cd arrow/dev/tasks/conda-recipes - ############ crossbow specific ############### + steps: # configure qemu binfmt-misc running. This allows us to run docker containers # embedded qemu-static - script: | @@ -25,8 +18,10 @@ jobs: displayName: Configure binfmt_misc - script: | - export CI=azure - run_docker_build.sh + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout {{ arrow.head }} + + - script: export CI=azure ./run_docker_build.sh displayName: Run docker build - env: - BINSTAR_TOKEN: $(BINSTAR_TOKEN) + workingDirectory: arrow/dev/tasks/conda-recipes From 501d5534178a4beb7727101520d3122da1b1059a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 21 Jun 2019 16:21:17 +0200 Subject: [PATCH 05/39] set config [skip ci] --- dev/tasks/conda-recipes/azure.linux.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index 7d659d9ffcc..3975233cd09 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -5,6 +5,7 @@ jobs: timeoutInMinutes: 360 variables: + CONFIG: {{ config }} UPLOAD_PACKAGES: False DOCKER_IMAGE: condaforge/linux-anvil-comp7 @@ -22,6 +23,6 @@ jobs: git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} git -C arrow checkout {{ arrow.head }} - - script: export CI=azure ./run_docker_build.sh + - script: CI=azure ./run_docker_build.sh displayName: Run docker build workingDirectory: arrow/dev/tasks/conda-recipes From df31ff7dc45f2f22f1f6e058f060f2edc8de3f97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 21 Jun 2019 16:32:50 +0200 Subject: [PATCH 06/39] trying to fix feedstock and recipe roots [skip ci] --- dev/tasks/conda-recipes/run_docker_build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tasks/conda-recipes/run_docker_build.sh b/dev/tasks/conda-recipes/run_docker_build.sh index 3524f61e988..952c34a8c1f 100755 --- a/dev/tasks/conda-recipes/run_docker_build.sh +++ b/dev/tasks/conda-recipes/run_docker_build.sh @@ -10,8 +10,8 @@ set -xeo pipefail THISDIR="$( cd "$( dirname "$0" )" >/dev/null && pwd )" PROVIDER_DIR="$(basename $THISDIR)" -FEEDSTOCK_ROOT=$(cd "$(dirname "$0")/.."; pwd;) -RECIPE_ROOT="${FEEDSTOCK_ROOT}/recipe" +FEEDSTOCK_ROOT=$(cd "$(dirname "$0")"; pwd;) +RECIPE_ROOT="${FEEDSTOCK_ROOT}/pyarrow" docker info @@ -70,4 +70,4 @@ docker run ${DOCKER_RUN_ARGS} \ /home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh # verify that the end of the script was reached -test -f "$DONE_CANARY" \ No newline at end of file +test -f "$DONE_CANARY" From ce740d799466c6f8bd078206f659ea63e7a0123a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sat, 22 Jun 2019 10:03:42 +0200 Subject: [PATCH 07/39] fixing build_steps.sh path [skip ci] --- dev/tasks/conda-recipes/run_docker_build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/run_docker_build.sh b/dev/tasks/conda-recipes/run_docker_build.sh index 952c34a8c1f..68fe2c70519 100755 --- a/dev/tasks/conda-recipes/run_docker_build.sh +++ b/dev/tasks/conda-recipes/run_docker_build.sh @@ -8,7 +8,6 @@ set -xeo pipefail THISDIR="$( cd "$( dirname "$0" )" >/dev/null && pwd )" -PROVIDER_DIR="$(basename $THISDIR)" FEEDSTOCK_ROOT=$(cd "$(dirname "$0")"; pwd;) RECIPE_ROOT="${FEEDSTOCK_ROOT}/pyarrow" @@ -67,7 +66,7 @@ docker run ${DOCKER_RUN_ARGS} \ -e CI \ $DOCKER_IMAGE \ bash \ - /home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh + /home/conda/feedstock_root/build_steps.sh # verify that the end of the script was reached test -f "$DONE_CANARY" From 7b60c9d070c7a4c13e10c05425f0889815ad5993 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sat, 22 Jun 2019 18:23:01 +0200 Subject: [PATCH 08/39] pass arrow_version [skip ci] --- dev/tasks/conda-recipes/azure.linux.yml | 3 ++- dev/tasks/conda-recipes/run_docker_build.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index 3975233cd09..c5932daf4d8 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -6,8 +6,9 @@ jobs: variables: CONFIG: {{ config }} - UPLOAD_PACKAGES: False + ARROW_VERSION: {{ arrow.no_rc_version }} DOCKER_IMAGE: condaforge/linux-anvil-comp7 + UPLOAD_PACKAGES: False steps: # configure qemu binfmt-misc running. This allows us to run docker containers diff --git a/dev/tasks/conda-recipes/run_docker_build.sh b/dev/tasks/conda-recipes/run_docker_build.sh index 68fe2c70519..39fc7450106 100755 --- a/dev/tasks/conda-recipes/run_docker_build.sh +++ b/dev/tasks/conda-recipes/run_docker_build.sh @@ -63,6 +63,7 @@ docker run ${DOCKER_RUN_ARGS} \ -e BINSTAR_TOKEN \ -e HOST_USER_ID \ -e UPLOAD_PACKAGES \ + -e ARROW_VERSION \ -e CI \ $DOCKER_IMAGE \ bash \ From adae7c0f36fb2d2bbdd1a41f6bfa2564d7903072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 23 Jun 2019 19:44:29 +0200 Subject: [PATCH 09/39] build all three recipes --- dev/tasks/conda-recipes/build_steps.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh index 8a4af44fecd..6ca231593ae 100755 --- a/dev/tasks/conda-recipes/build_steps.sh +++ b/dev/tasks/conda-recipes/build_steps.sh @@ -29,11 +29,11 @@ source run_conda_forge_build_setup # make the build number clobber make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" -conda build "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ +conda build arrow-cpp parquet-cpp pyarrow -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then upload_package "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" fi -touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" \ No newline at end of file +touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" From 0e826ac43741412b7662f0a512112c626d722c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 23 Jun 2019 19:52:46 +0200 Subject: [PATCH 10/39] fix recipe directories [skip ci] --- dev/tasks/conda-recipes/build_steps.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh index 6ca231593ae..994261dc2f0 100755 --- a/dev/tasks/conda-recipes/build_steps.sh +++ b/dev/tasks/conda-recipes/build_steps.sh @@ -29,8 +29,12 @@ source run_conda_forge_build_setup # make the build number clobber make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" -conda build arrow-cpp parquet-cpp pyarrow -m "${CI_SUPPORT}/${CONFIG}.yaml" \ - --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" +conda build \ + -m "${CI_SUPPORT}/${CONFIG}.yaml" \ + --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" \ + "${RECIPE_ROOT}/arrow-cpp" \ + "${RECIPE_ROOT}/parquet-cpp" \ + "${RECIPE_ROOT}/pyarrow" if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then upload_package "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" From 9ef81c567721fb6ba83f0b412321be3b6ec2230a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 23 Jun 2019 20:01:17 +0200 Subject: [PATCH 11/39] use feedstock_root [skip ci] --- dev/tasks/conda-recipes/build_steps.sh | 13 ++++++------- dev/tasks/conda-recipes/run_docker_build.sh | 2 -- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh index 994261dc2f0..8911ce9acf4 100755 --- a/dev/tasks/conda-recipes/build_steps.sh +++ b/dev/tasks/conda-recipes/build_steps.sh @@ -8,7 +8,6 @@ set -xeuo pipefail export PYTHONUNBUFFERED=1 export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" -export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}" export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml" @@ -22,22 +21,22 @@ CONDARC conda install --yes --quiet conda-forge-ci-setup=2 conda-build -c conda-forge # set up the condarc -setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" +setup_conda_rc "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}" source run_conda_forge_build_setup # make the build number clobber -make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" +make_build_number "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}" conda build \ -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" \ - "${RECIPE_ROOT}/arrow-cpp" \ - "${RECIPE_ROOT}/parquet-cpp" \ - "${RECIPE_ROOT}/pyarrow" + "${FEEDSTOCK_ROOT}/arrow-cpp" \ + "${FEEDSTOCK_ROOT}/parquet-cpp" \ + "${FEEDSTOCK_ROOT}/pyarrow" if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then - upload_package "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" + upload_package "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}" fi touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" diff --git a/dev/tasks/conda-recipes/run_docker_build.sh b/dev/tasks/conda-recipes/run_docker_build.sh index 39fc7450106..bb71d13878e 100755 --- a/dev/tasks/conda-recipes/run_docker_build.sh +++ b/dev/tasks/conda-recipes/run_docker_build.sh @@ -10,7 +10,6 @@ set -xeo pipefail THISDIR="$( cd "$( dirname "$0" )" >/dev/null && pwd )" FEEDSTOCK_ROOT=$(cd "$(dirname "$0")"; pwd;) -RECIPE_ROOT="${FEEDSTOCK_ROOT}/pyarrow" docker info @@ -57,7 +56,6 @@ fi export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" docker run ${DOCKER_RUN_ARGS} \ - -v "${RECIPE_ROOT}":/home/conda/recipe_root:ro,z \ -v "${FEEDSTOCK_ROOT}":/home/conda/feedstock_root:rw,z \ -e CONFIG \ -e BINSTAR_TOKEN \ From 4b005892f297cc353980cb688758f0c88ba1aa68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 23 Jun 2019 20:24:36 +0200 Subject: [PATCH 12/39] try to fix assertion error [skip ci] --- dev/tasks/conda-recipes/build_steps.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh index 8911ce9acf4..18aa2b8ab4a 100755 --- a/dev/tasks/conda-recipes/build_steps.sh +++ b/dev/tasks/conda-recipes/build_steps.sh @@ -8,6 +8,7 @@ set -xeuo pipefail export PYTHONUNBUFFERED=1 export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" +export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}" export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml" @@ -21,22 +22,21 @@ CONDARC conda install --yes --quiet conda-forge-ci-setup=2 conda-build -c conda-forge # set up the condarc -setup_conda_rc "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}" +setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" source run_conda_forge_build_setup # make the build number clobber -make_build_number "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}" +make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" -conda build \ +pushd $FEEDSTOCK_ROOT +conda build arrow-cpp parquet-cpp pyarrow \ -m "${CI_SUPPORT}/${CONFIG}.yaml" \ - --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" \ - "${FEEDSTOCK_ROOT}/arrow-cpp" \ - "${FEEDSTOCK_ROOT}/parquet-cpp" \ - "${FEEDSTOCK_ROOT}/pyarrow" + --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" +popd if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then - upload_package "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}" + upload_package "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" fi touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" From 5cd9fa0b5e47ed42cd2595a689d80c9c96530fd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 23 Jun 2019 20:34:43 +0200 Subject: [PATCH 13/39] use pyarrow as recipe root --- dev/tasks/conda-recipes/build_steps.sh | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh index 18aa2b8ab4a..4fe89212c1d 100755 --- a/dev/tasks/conda-recipes/build_steps.sh +++ b/dev/tasks/conda-recipes/build_steps.sh @@ -8,7 +8,6 @@ set -xeuo pipefail export PYTHONUNBUFFERED=1 export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" -export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}" export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml" @@ -21,22 +20,18 @@ CONDARC conda install --yes --quiet conda-forge-ci-setup=2 conda-build -c conda-forge +pushd $FEEDSTOCK_ROOT + # set up the condarc -setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" +setup_conda_rc "${FEEDSTOCK_ROOT}" "pyarrow" "${CONFIG_FILE}" source run_conda_forge_build_setup # make the build number clobber -make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" +make_build_number "${FEEDSTOCK_ROOT}" "pyarrow" "${CONFIG_FILE}" -pushd $FEEDSTOCK_ROOT conda build arrow-cpp parquet-cpp pyarrow \ -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" -popd - -if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then - upload_package "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" -fi -touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" +popd From f50af1b517bb64b625265a4bcfa5874ac9bb8bdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Sun, 23 Jun 2019 21:22:03 +0200 Subject: [PATCH 14/39] path gymnastics [skip ci] --- .../conda-recipes/.ci_support/linux_python3.7.yaml | 2 ++ dev/tasks/conda-recipes/build_steps.sh | 13 ++++++------- dev/tasks/conda-recipes/run_docker_build.sh | 11 +++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml b/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml index 9022d934233..55685d02c32 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml @@ -22,3 +22,5 @@ pin_run_as_build: max_pin: x.x python: - '3.7' + + diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh index 4fe89212c1d..89f860c75b4 100755 --- a/dev/tasks/conda-recipes/build_steps.sh +++ b/dev/tasks/conda-recipes/build_steps.sh @@ -20,18 +20,17 @@ CONDARC conda install --yes --quiet conda-forge-ci-setup=2 conda-build -c conda-forge -pushd $FEEDSTOCK_ROOT - # set up the condarc -setup_conda_rc "${FEEDSTOCK_ROOT}" "pyarrow" "${CONFIG_FILE}" +setup_conda_rc "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}" source run_conda_forge_build_setup # make the build number clobber -make_build_number "${FEEDSTOCK_ROOT}" "pyarrow" "${CONFIG_FILE}" +make_build_number "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}" -conda build arrow-cpp parquet-cpp pyarrow \ +conda build \ + "${FEEDSTOCK_ROOT}/arrow-cpp" \ + "${FEEDSTOCK_ROOT}/parquet-cpp" \ + "${FEEDSTOCK_ROOT}/pyarrow" \ -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" - -popd diff --git a/dev/tasks/conda-recipes/run_docker_build.sh b/dev/tasks/conda-recipes/run_docker_build.sh index bb71d13878e..735bc1ea34b 100755 --- a/dev/tasks/conda-recipes/run_docker_build.sh +++ b/dev/tasks/conda-recipes/run_docker_build.sh @@ -8,8 +8,8 @@ set -xeo pipefail THISDIR="$( cd "$( dirname "$0" )" >/dev/null && pwd )" - -FEEDSTOCK_ROOT=$(cd "$(dirname "$0")"; pwd;) +ARROW_ROOT=$(cd "$THISDIR/../../.."; pwd;) +FEEDSTOCK_ROOT=$THISDIR docker info @@ -56,16 +56,15 @@ fi export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" docker run ${DOCKER_RUN_ARGS} \ - -v "${FEEDSTOCK_ROOT}":/home/conda/feedstock_root:rw,z \ + -v "${ARROW_ROOT}":/arrow:rw,z \ + -e FEEDSTOCK_ROOT="/arrow/dev/tasks/conda-recipes" \ -e CONFIG \ - -e BINSTAR_TOKEN \ -e HOST_USER_ID \ -e UPLOAD_PACKAGES \ -e ARROW_VERSION \ -e CI \ $DOCKER_IMAGE \ - bash \ - /home/conda/feedstock_root/build_steps.sh + bash /arrow/dev/tasks/conda-recipes/build_steps.sh # verify that the end of the script was reached test -f "$DONE_CANARY" From a874d1f99fe0c4ef5ededc5691f583a02a4532df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 24 Jun 2019 11:27:36 +0200 Subject: [PATCH 15/39] gandiva flags [skip ci] --- dev/tasks/conda-recipes/arrow-cpp/build.sh | 9 +++++++++ dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/build.sh b/dev/tasks/conda-recipes/arrow-cpp/build.sh index 3d307096745..6feb0a72ecd 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/build.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build.sh @@ -5,6 +5,14 @@ cd cpp mkdir build-dir cd build-dir +EXTRA_CMAKE_ARGS="" + +# Include g++'s system headers +if [ "$(uname)" == "Linux" ]; then + SYSTEM_INLUDES=$(echo | ${CXX} -E -Wp,-v - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';') + EXTRA_CMAKE_ARGS=" -DARROW_GANDIVA_PC_CXX_FLAGS=\"${SYSTEM_INLUDES}\"" +fi + cmake \ -DCMAKE_BUILD_TYPE=release \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ @@ -26,6 +34,7 @@ cmake \ -DCMAKE_AR=${AR} \ -DCMAKE_RANLIB=${RANLIB} \ -GNinja \ + ${EXTRA_CMAKE_ARGS} \ .. ninja install diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index ff367383b59..877bfc6021c 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -14,8 +14,8 @@ build: requirements: build: - - autoconf # [unix] - cmake + - autoconf # [unix] - ninja - {{ compiler('c') }} - {{ compiler('cxx') }} @@ -44,6 +44,7 @@ requirements: - {{ pin_compatible('numpy', lower_bound='1.14') }} - boost-cpp - brotli + - double-conversion - gflags - glog - lz4-c From bdf705ff0e64744a43d0cf2a0aa20116ae16be5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 24 Jun 2019 11:33:26 +0200 Subject: [PATCH 16/39] OSX configuration [skip ci] --- dev/tasks/conda-recipes/azure.linux.yml | 11 +++-- dev/tasks/conda-recipes/azure.osx.yml | 66 ++++++++++++------------- 2 files changed, 38 insertions(+), 39 deletions(-) diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index c5932daf4d8..decb1b86be5 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -13,17 +13,18 @@ jobs: steps: # configure qemu binfmt-misc running. This allows us to run docker containers # embedded qemu-static - - script: | + - displayName: Configure binfmt_misc + script: | docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes ls /proc/sys/fs/binfmt_misc/ condition: not(startsWith(variables['CONFIG'], 'linux_64')) - displayName: Configure binfmt_misc - - script: | + - displayName: Clone arrow + script: | git clone --no-checkout {{ arrow.remote }} arrow git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} git -C arrow checkout {{ arrow.head }} - - script: CI=azure ./run_docker_build.sh - displayName: Run docker build + - displayName: Run docker build + script: CI=azure ./run_docker_build.sh workingDirectory: arrow/dev/tasks/conda-recipes diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index d51d70222d7..867453f8898 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -3,67 +3,65 @@ jobs: pool: vmImage: macOS-10.13 timeoutInMinutes: 360 - ############ instead of strategy ############### variables: + CONFIG: {{ config }} + ARROW_VERSION: {{ arrow.no_rc_version }} UPLOAD_PACKAGES: False steps: - ############### clone arrow and change directory ################# - - template: azure-template-clone-arrow.yml - - # TODO: Fast finish on azure pipelines? - - script: | - echo "Fast Finish" - - - script: | + - displayName: Remove homebrew + script: | echo "Removing homebrew from Azure to avoid conflicts." curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/uninstall > ~/uninstall_homebrew chmod +x ~/uninstall_homebrew ~/uninstall_homebrew -fq rm ~/uninstall_homebrew - displayName: Remove homebrew - - bash: | + - displayName: Add conda to PATH + bash: | echo "##vso[task.prependpath]$CONDA/bin" sudo chown -R $USER $CONDA - displayName: Add conda to PATH - - script: | + - displayName: 'Add conda-forge-ci-setup=2' + script: | source activate base conda install -n base -c conda-forge --quiet --yes conda-forge-ci-setup=2 conda-build - displayName: 'Add conda-forge-ci-setup=2' - - script: | + - displayName: Clone arrow + script: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout {{ arrow.head }} + + - displayName: Configure conda and conda-build + script: | source activate base echo "Configuring conda." - setup_conda_rc ./ ./recipe ./.ci_support/${CONFIG}.yaml + setup_conda_rc ./ ./ ./.ci_support/${CONFIG}.yaml export CI=azure source run_conda_forge_build_setup conda update --yes --quiet --override-channels -c conda-forge -c defaults --all + workingDirectory: arrow/dev/tasks/conda-recipes env: { OSX_FORCE_SDK_DOWNLOAD: "1" } - displayName: Configure conda and conda-build - - - script: | - source activate base - mangle_compiler ./ ./recipe ./.ci_support/${CONFIG}.yaml - displayName: Mangle compiler - - script: | + - displayName: Mangle compiler + workingDirectory: arrow/dev/tasks/conda-recipes + script: | source activate base - make_build_number ./ ./recipe ./.ci_support/${CONFIG}.yaml - displayName: Generate build number clobber file + mangle_compiler ./ ./ ./.ci_support/${CONFIG}.yaml - - script: | + - displayName: Generate build number clobber file + workingDirectory: arrow/dev/tasks/conda-recipes + script: | source activate base - conda build ./recipe -m ./.ci_support/${CONFIG}.yaml --clobber-file ./.ci_support/clobber_${CONFIG}.yaml - displayName: Build recipe + make_build_number ./ ./ ./.ci_support/${CONFIG}.yaml - - script: | + - displayName: Build recipes + workingDirectory: arrow/dev/tasks/conda-recipes + script: | source activate base - upload_package ./ ./recipe ./.ci_support/${CONFIG}.yaml - displayName: Upload recipe - env: - BINSTAR_TOKEN: $(BINSTAR_TOKEN) - condition: not(eq(variables['UPLOAD_PACKAGES'], 'False')) + conda build arrow-cpp parquet-cpp pyarrow \ + -m ./.ci_support/${CONFIG}.yaml \ + --clobber-file ./.ci_support/clobber_${CONFIG}.yaml From 8abd347790cd4fdbe1907e2bb08a06fe609737b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 24 Jun 2019 11:46:30 +0200 Subject: [PATCH 17/39] move displayName after the script [skip ci] --- dev/tasks/conda-recipes/azure.osx.yml | 38 +++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index 867453f8898..2582668eb10 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -8,32 +8,31 @@ jobs: ARROW_VERSION: {{ arrow.no_rc_version }} UPLOAD_PACKAGES: False steps: - - displayName: Remove homebrew - script: | + - script: | echo "Removing homebrew from Azure to avoid conflicts." curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/uninstall > ~/uninstall_homebrew chmod +x ~/uninstall_homebrew ~/uninstall_homebrew -fq rm ~/uninstall_homebrew + displayName: Remove homebrew - - displayName: Add conda to PATH - bash: | + - bash: | echo "##vso[task.prependpath]$CONDA/bin" sudo chown -R $USER $CONDA + displayName: Add conda to PATH - - displayName: 'Add conda-forge-ci-setup=2' - script: | + - script: | source activate base conda install -n base -c conda-forge --quiet --yes conda-forge-ci-setup=2 conda-build + displayName: 'Add conda-forge-ci-setup=2' - - displayName: Clone arrow - script: | + - script: | git clone --no-checkout {{ arrow.remote }} arrow git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} git -C arrow checkout {{ arrow.head }} + displayName: Clone arrow - - displayName: Configure conda and conda-build - script: | + - script: | source activate base echo "Configuring conda." @@ -41,27 +40,28 @@ jobs: export CI=azure source run_conda_forge_build_setup conda update --yes --quiet --override-channels -c conda-forge -c defaults --all + displayName: Configure conda and conda-build workingDirectory: arrow/dev/tasks/conda-recipes env: { OSX_FORCE_SDK_DOWNLOAD: "1" } - - displayName: Mangle compiler - workingDirectory: arrow/dev/tasks/conda-recipes - script: | + - script: | source activate base mangle_compiler ./ ./ ./.ci_support/${CONFIG}.yaml - - - displayName: Generate build number clobber file workingDirectory: arrow/dev/tasks/conda-recipes - script: | + displayName: Mangle compiler + + - script: | source activate base make_build_number ./ ./ ./.ci_support/${CONFIG}.yaml - - - displayName: Build recipes workingDirectory: arrow/dev/tasks/conda-recipes - script: | + displayName: Generate build number clobber file + + - script: | source activate base conda build arrow-cpp parquet-cpp pyarrow \ -m ./.ci_support/${CONFIG}.yaml \ --clobber-file ./.ci_support/clobber_${CONFIG}.yaml + workingDirectory: arrow/dev/tasks/conda-recipes + displayName: Build recipes From c42ebf5953c2d5c4751d86dda2613f9f3a8f125d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 24 Jun 2019 13:20:05 +0200 Subject: [PATCH 18/39] quoting gandiva flags [skip ci] --- dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml | 1 + dev/tasks/conda-recipes/arrow-cpp/build.sh | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml b/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml index 55685d02c32..37369d72ff1 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml @@ -24,3 +24,4 @@ python: - '3.7' + diff --git a/dev/tasks/conda-recipes/arrow-cpp/build.sh b/dev/tasks/conda-recipes/arrow-cpp/build.sh index 6feb0a72ecd..88ca6194daf 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/build.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build.sh @@ -10,7 +10,7 @@ EXTRA_CMAKE_ARGS="" # Include g++'s system headers if [ "$(uname)" == "Linux" ]; then SYSTEM_INLUDES=$(echo | ${CXX} -E -Wp,-v - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';') - EXTRA_CMAKE_ARGS=" -DARROW_GANDIVA_PC_CXX_FLAGS=\"${SYSTEM_INLUDES}\"" + EXTRA_CMAKE_ARGS=" -DARROW_GANDIVA_PC_CXX_FLAGS=${SYSTEM_INLUDES}" fi cmake \ @@ -30,7 +30,6 @@ cmake \ -DARROW_PARQUET=ON \ -DARROW_GANDIVA=ON \ -DARROW_ORC=ON \ - -DORC_HOME=$PREFIX \ -DCMAKE_AR=${AR} \ -DCMAKE_RANLIB=${RANLIB} \ -GNinja \ From aecc2b19e3c732bfba4f337b4b94e89724743715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 24 Jun 2019 13:26:33 +0200 Subject: [PATCH 19/39] displayName order [skip ci] --- dev/tasks/conda-recipes/azure.linux.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index decb1b86be5..d4fe2eb62e2 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -13,18 +13,18 @@ jobs: steps: # configure qemu binfmt-misc running. This allows us to run docker containers # embedded qemu-static - - displayName: Configure binfmt_misc - script: | + - script: | docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes ls /proc/sys/fs/binfmt_misc/ + displayName: Configure binfmt_misc condition: not(startsWith(variables['CONFIG'], 'linux_64')) - - displayName: Clone arrow - script: | + - script: | git clone --no-checkout {{ arrow.remote }} arrow git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} git -C arrow checkout {{ arrow.head }} + displayName: Clone arrow - - displayName: Run docker build - script: CI=azure ./run_docker_build.sh + - script: CI=azure ./run_docker_build.sh + displayName: Run docker build workingDirectory: arrow/dev/tasks/conda-recipes From 0c8464a4b55cc4547e3c2c7ae0a13f5f6ac70f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 24 Jun 2019 13:48:08 +0200 Subject: [PATCH 20/39] parquet-cpp depend on exact arrow-cpp version [skip ci] --- dev/tasks/conda-recipes/parquet-cpp/meta.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/parquet-cpp/meta.yaml b/dev/tasks/conda-recipes/parquet-cpp/meta.yaml index 5eb999b5bd7..9bf0c9c856c 100644 --- a/dev/tasks/conda-recipes/parquet-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/parquet-cpp/meta.yaml @@ -13,9 +13,10 @@ build: requirements: host: - - arrow-cpp >={{ ARROW_VERSION }} + # NOTE: in the upstream feedstock use >= instead of = + - arrow-cpp ={{ ARROW_VERSION }} run: - - arrow-cpp >={{ ARROW_VERSION }} + - arrow-cpp ={{ ARROW_VERSION }} test: commands: From 0d76f13648aaa6dcf620c0b0081202c14ae130eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 24 Jun 2019 14:10:14 +0200 Subject: [PATCH 21/39] win [skip ci] --- .../.ci_support/linux_python3.7.yaml | 1 + dev/tasks/conda-recipes/azure.win.yml | 32 ++++++++----------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml b/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml index 37369d72ff1..c3bad089be4 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml @@ -25,3 +25,4 @@ python: + diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml index 01b0eb80aed..ed8048c19cc 100755 --- a/dev/tasks/conda-recipes/azure.win.yml +++ b/dev/tasks/conda-recipes/azure.win.yml @@ -3,18 +3,12 @@ jobs: pool: vmImage: vs2017-win2016 timeoutInMinutes: 360 - ############ instead of strategy ############### variables: + CONFIG: {{ config }} + ARROW_VERSION: {{ arrow.no_rc_version }} CONDA_BLD_PATH: D:\\bld\\ UPLOAD_PACKAGES: False steps: - ############### clone arrow and change directory ################# - - template: azure-template-clone-arrow.yml - - # TODO: Fast finish on azure pipelines? - - script: | - ECHO ON - - script: | choco install vcpython27 -fdv -y --debug condition: contains(variables['CONFIG'], 'vs2008') @@ -58,8 +52,15 @@ jobs: - script: set PYTHONUNBUFFERED=1 + - script: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout {{ arrow.head }} + displayName: Clone arrow + # Configure the VM - - script: setup_conda_rc .\ .\recipe .\.ci_support\%CONFIG%.yaml + - script: setup_conda_rc .\ .\ .\.ci_support\%CONFIG%.yaml + workingDirectory: arrow\dev\tasks\conda-recipes # Configure the VM. - script: | @@ -67,7 +68,6 @@ jobs: run_conda_forge_build_setup displayName: conda-forge build setup - - script: | rmdir C:\strawberry /s /q continueOnError: true @@ -75,22 +75,18 @@ jobs: # Special cased version setting some more things! - script: | - conda.exe build recipe -m .ci_support\%CONFIG%.yaml + conda.exe build arrow-cpp parquet-cpp pyarrow -m .ci_support\%CONFIG%.yaml displayName: Build recipe (vs2008) + workingDirectory: arrow\dev\tasks\conda-recipes env: VS90COMNTOOLS: "C:\\Program Files (x86)\\Common Files\\Microsoft\\Visual C++ for Python\\9.0\\VC\\bin" PYTHONUNBUFFERED: 1 condition: contains(variables['CONFIG'], 'vs2008') - script: | - conda.exe build recipe -m .ci_support\%CONFIG%.yaml + conda.exe build arrow-cpp parquet-cpp pyarrow -m .ci_support\%CONFIG%.yaml displayName: Build recipe + workingDirectory: arrow\dev\tasks\conda-recipes env: PYTHONUNBUFFERED: 1 condition: not(contains(variables['CONFIG'], 'vs2008')) - - - script: | - upload_package .\ .\recipe .ci_support\%CONFIG%.yaml - env: - BINSTAR_TOKEN: $(BINSTAR_TOKEN) - condition: not(eq(variables['UPLOAD_PACKAGES'], 'False')) From ba0e88cce008639cc2ba8433155e566f1fe8bc29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 24 Jun 2019 17:57:24 +0200 Subject: [PATCH 22/39] update old templates; query cxx include paths [skip ci] --- dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml | 4 ---- dev/tasks/conda-recipes/appveyor.yml | 4 ++-- dev/tasks/conda-recipes/arrow-cpp/build.sh | 4 ++-- dev/tasks/conda-recipes/azure.win.yml | 4 ---- dev/tasks/conda-recipes/travis.linux.yml | 4 ++-- dev/tasks/conda-recipes/travis.osx.yml | 6 +----- 6 files changed, 7 insertions(+), 19 deletions(-) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml b/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml index c3bad089be4..9022d934233 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml @@ -22,7 +22,3 @@ pin_run_as_build: max_pin: x.x python: - '3.7' - - - - diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.yml index 8ad63f48af4..121595a6793 100644 --- a/dev/tasks/conda-recipes/appveyor.yml +++ b/dev/tasks/conda-recipes/appveyor.yml @@ -33,11 +33,11 @@ test_script: - pushd arrow\dev\tasks\conda-recipes # Configure conda - - cmd: setup_conda_rc .\ .\ variants\{{ config }}.yaml + - cmd: setup_conda_rc .\ .\ .ci_support\{{ config }}.yaml - cmd: run_conda_forge_build_setup # Build the recipes - - conda.exe build --output-folder . -m variants\{{ config }}.yaml parquet-cpp arrow-cpp pyarrow + - conda.exe build --output-folder . -m .ci_support\{{ config }}.yaml parquet-cpp arrow-cpp pyarrow # Rename artifacts - pushd win-64 diff --git a/dev/tasks/conda-recipes/arrow-cpp/build.sh b/dev/tasks/conda-recipes/arrow-cpp/build.sh index 88ca6194daf..b18862a83fd 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/build.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build.sh @@ -9,8 +9,8 @@ EXTRA_CMAKE_ARGS="" # Include g++'s system headers if [ "$(uname)" == "Linux" ]; then - SYSTEM_INLUDES=$(echo | ${CXX} -E -Wp,-v - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';') - EXTRA_CMAKE_ARGS=" -DARROW_GANDIVA_PC_CXX_FLAGS=${SYSTEM_INLUDES}" + SYSTEM_INCLUDES=$(echo | ${CXX} -E -Wp,-v -xc++ - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';') + EXTRA_CMAKE_ARGS=" -DARROW_GANDIVA_PC_CXX_FLAGS=${SYSTEM_INCLUDES}" fi cmake \ diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml index ed8048c19cc..9f160bd1137 100755 --- a/dev/tasks/conda-recipes/azure.win.yml +++ b/dev/tasks/conda-recipes/azure.win.yml @@ -14,10 +14,6 @@ jobs: condition: contains(variables['CONFIG'], 'vs2008') displayName: Install vcpython27.msi (if needed) - # Cygwin's git breaks conda-build. (See https://github.com/conda-forge/conda-smithy-feedstock/pull/2.) - # - script: rmdir C:\cygwin /s /q - # continueOnError: true - - powershell: | Set-PSDebug -Trace 1 diff --git a/dev/tasks/conda-recipes/travis.linux.yml b/dev/tasks/conda-recipes/travis.linux.yml index ba5143f210c..899e699fd0e 100644 --- a/dev/tasks/conda-recipes/travis.linux.yml +++ b/dev/tasks/conda-recipes/travis.linux.yml @@ -37,14 +37,14 @@ before_script: - pushd arrow/dev/tasks/conda-recipes # Configure conda - - setup_conda_rc ./ ./ variants/{{ config }}.yaml + - setup_conda_rc ./ ./ .ci_support/{{ config }}.yaml - source run_conda_forge_build_setup script: - | conda build --croot $TRAVIS_HOME/conda_build_root \ --output-folder . \ - -m variants/{{ config }}.yaml \ + -m .ci_support/{{ config }}.yaml \ parquet-cpp arrow-cpp pyarrow deploy: diff --git a/dev/tasks/conda-recipes/travis.osx.yml b/dev/tasks/conda-recipes/travis.osx.yml index 956034db894..f32a611511b 100644 --- a/dev/tasks/conda-recipes/travis.osx.yml +++ b/dev/tasks/conda-recipes/travis.osx.yml @@ -36,11 +36,7 @@ before_script: - pushd arrow/dev/tasks/conda-recipes # Configure conda - - setup_conda_rc ./ ./ variants/{{ config }}.yaml - - # XXX: workaround, see run_conda_forge_build_setup_osx#L33 - - mkdir -p ./.ci_support - - cp variants/{{ config }}.yaml ./.ci_support/{{ config }}.yaml + - setup_conda_rc ./ ./ .ci_support/{{ config }}.yaml - CONFIG={{ config }} source run_conda_forge_build_setup # Compiler cleanup From 611222e281a6bf17a41f19dd1d7c7ea8b2282eef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 24 Jun 2019 18:35:12 +0200 Subject: [PATCH 23/39] docker shm [skip ci] --- dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml | 1 + dev/tasks/conda-recipes/run_docker_build.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml b/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml index 9022d934233..1a27da23c50 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_python3.7.yaml @@ -22,3 +22,4 @@ pin_run_as_build: max_pin: x.x python: - '3.7' + diff --git a/dev/tasks/conda-recipes/run_docker_build.sh b/dev/tasks/conda-recipes/run_docker_build.sh index 735bc1ea34b..99476c59e9a 100755 --- a/dev/tasks/conda-recipes/run_docker_build.sh +++ b/dev/tasks/conda-recipes/run_docker_build.sh @@ -56,6 +56,7 @@ fi export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" docker run ${DOCKER_RUN_ARGS} \ + --shm-size=2G \ -v "${ARROW_ROOT}":/arrow:rw,z \ -e FEEDSTOCK_ROOT="/arrow/dev/tasks/conda-recipes" \ -e CONFIG \ From d6c4ce9fa129af6374c6080b1e8451f2ac531880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Mon, 24 Jun 2019 20:46:42 +0200 Subject: [PATCH 24/39] touch done_canary [skip ci] --- dev/tasks/conda-recipes/build_steps.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh index 89f860c75b4..889574d17bb 100755 --- a/dev/tasks/conda-recipes/build_steps.sh +++ b/dev/tasks/conda-recipes/build_steps.sh @@ -34,3 +34,5 @@ conda build \ "${FEEDSTOCK_ROOT}/pyarrow" \ -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" + +touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" From 7fbce5df8fd0447df4a98e0c755b99090cd56166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 25 Jun 2019 00:38:50 +0200 Subject: [PATCH 25/39] use appveyor for the win packages; upload assets scripts [skip ci] --- .../{appveyor.yml => appveyor.win.yml} | 0 dev/tasks/tasks.yml | 8 +-- dev/tasks/upload-assets.py | 63 +++++++++++++++++++ 3 files changed, 67 insertions(+), 4 deletions(-) rename dev/tasks/conda-recipes/{appveyor.yml => appveyor.win.yml} (100%) create mode 100755 dev/tasks/upload-assets.py diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.win.yml similarity index 100% rename from dev/tasks/conda-recipes/appveyor.yml rename to dev/tasks/conda-recipes/appveyor.win.yml diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 1dfe95f3c18..6aead4d378b 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -136,9 +136,9 @@ tasks: ############################## Conda Windows ################################ conda-win-vs2015-py36: - ci: azure + ci: appveyor platform: win - template: conda-recipes/azure.win.yml + template: conda-recipes/appveyor.win.yml params: config: win_c_compilervs2015cxx_compilervs2015python3.6 artifacts: @@ -146,9 +146,9 @@ tasks: - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0.tar.bz2 conda-win-vs2015-py37: - ci: azure + ci: appveyor platform: win - template: conda-recipes/azure.win.yml + template: conda-recipes/appveyor.win.yml params: config: win_c_compilervs2015cxx_compilervs2015python3.7 artifacts: diff --git a/dev/tasks/upload-assets.py b/dev/tasks/upload-assets.py new file mode 100755 index 00000000000..103300278a3 --- /dev/null +++ b/dev/tasks/upload-assets.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import mimetypes +from glob import glob + +import click +import github3 + + +# TODO(kszucs): move it to crossbow.py + + +@click.command() +@click.option('--sha', help='Target committish') +@click.option('--tag', help='Target tag') +@click.option('--pattern', help='File pattern') +def upload_assets(tag, sha, pattern): + token = os.environ['CROSSBOW_GITHUB_TOKEN'] + owner, repository = os.environ['CROSSBOW_GITHUB_REPO'].split('/') + + gh = github3.login(token=token) + repo = gh.repository(owner, repository) + + try: + release = repo.release_from_tag(tag) + except github3.exceptions.NotFoundError: + pass + else: + click.echo('Removing release `{}`'.format(release.tag_name)) + release.delete() + + click.echo('Creating release `{}`'.format(tag)) + release = repo.create_release(tag, sha) + + for path in glob(pattern): + name = os.path.basename(path) + mime = mimetypes.guess_type(name)[0] or 'application/octet-stream' + + click.echo('Uploading asset `{}`...'.format(name)) + with open(path, 'rb') as fp: + release.upload_asset(name=name, asset=fp, content_type=mime) + + +if __name__ == '__main__': + upload_assets() From e0d8fb9b2b4bd98e7bd13fbfc9c3ebae0fb1510f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 25 Jun 2019 01:10:46 +0200 Subject: [PATCH 26/39] git commit additional log [skip ci] --- dev/tasks/upload-assets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/upload-assets.py b/dev/tasks/upload-assets.py index 103300278a3..24356335991 100755 --- a/dev/tasks/upload-assets.py +++ b/dev/tasks/upload-assets.py @@ -38,6 +38,7 @@ def upload_assets(tag, sha, pattern): gh = github3.login(token=token) repo = gh.repository(owner, repository) + click.echo('Selected repository: {}/{}'.format(owner, repository)) try: release = repo.release_from_tag(tag) From 750f624c105d17fed7fb2791d0ef1918ca1ec218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 25 Jun 2019 01:15:39 +0200 Subject: [PATCH 27/39] asset uploading script [skip ci] --- dev/tasks/conda-recipes/azure.linux.yml | 21 ++++++++ dev/tasks/conda-recipes/azure.osx.yml | 21 ++++++++ dev/tasks/conda-recipes/azure.win.yml | 21 ++++++++ dev/tasks/conda-recipes/travis.linux.yml | 61 ----------------------- dev/tasks/conda-recipes/travis.osx.yml | 62 ------------------------ 5 files changed, 63 insertions(+), 123 deletions(-) delete mode 100644 dev/tasks/conda-recipes/travis.linux.yml delete mode 100644 dev/tasks/conda-recipes/travis.osx.yml diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index d4fe2eb62e2..d34c8806b3a 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -28,3 +28,24 @@ jobs: - script: CI=azure ./run_docker_build.sh displayName: Run docker build workingDirectory: arrow/dev/tasks/conda-recipes + + # Using github release tries to find a common ancestor between the currently + # pushed tag and the latest tag of the github repository (don't know why). + # The tag upload took 43 minutes because of this scanning, so use an + # alternative upload script. + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.6' + architecture: 'x64' + + - script: | + pip install github3-py click + python upload-assets.py \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + --pattern "conda-recipes/build_artifacts/*.tar.bz2" + env: + CROSSBOW_GITHUB_REPO: $(Build.Repository.Name) + CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN) + displayName: Upload packages as a GitHub release + workingDirectory: arrow/dev/tasks diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index 2582668eb10..2f38e2e7a9d 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -65,3 +65,24 @@ jobs: --clobber-file ./.ci_support/clobber_${CONFIG}.yaml workingDirectory: arrow/dev/tasks/conda-recipes displayName: Build recipes + + # Using github release tries to find a common ancestor between the currently + # pushed tag and the latest tag of the github repository (don't know why). + # The tag upload took 43 minutes because of this scanning, so use an + # alternative upload script. + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.6' + architecture: 'x64' + + - script: | + pip install github3-py click + python upload-assets.py \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + --pattern "conda-recipes/build_artifacts/*.tar.bz2" + env: + CROSSBOW_GITHUB_REPO: $(Build.Repository.Name) + CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN) + displayName: Upload packages as a GitHub release + workingDirectory: arrow/dev/tasks diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml index 9f160bd1137..5f76fd6fc35 100755 --- a/dev/tasks/conda-recipes/azure.win.yml +++ b/dev/tasks/conda-recipes/azure.win.yml @@ -86,3 +86,24 @@ jobs: env: PYTHONUNBUFFERED: 1 condition: not(contains(variables['CONFIG'], 'vs2008')) + + # Using github release tries to find a common ancestor between the currently + # pushed tag and the latest tag of the github repository (don't know why). + # The tag upload took 43 minutes because of this scanning, so use an + # alternative upload script. + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.6' + architecture: 'x64' + + - script: | + pip install github3-py click + python upload-assets.py \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + --pattern "conda-recipes/build_artifacts/*.tar.bz2" + env: + CROSSBOW_GITHUB_REPO: $(Build.Repository.Name) + CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN) + displayName: Upload packages as a GitHub release + workingDirectory: arrow/dev/tasks diff --git a/dev/tasks/conda-recipes/travis.linux.yml b/dev/tasks/conda-recipes/travis.linux.yml deleted file mode 100644 index 899e699fd0e..00000000000 --- a/dev/tasks/conda-recipes/travis.linux.yml +++ /dev/null @@ -1,61 +0,0 @@ -os: linux -dist: trusty -language: generic - -# don't build twice -if: tag IS blank - -env: - global: - - TRAVIS_TAG={{ task.tag }} - - ARROW_VERSION={{ arrow.no_rc_version }} - - PYTHONUNBUFFERED=1 - -install: - # Install Miniconda. - - echo `pwd` - - | - echo "" - echo "Installing a fresh version of Miniconda." - MINICONDA_URL="https://repo.continuum.io/miniconda" - MINICONDA_FILE="Miniconda3-latest-Linux-x86_64.sh" - curl -L -O "${MINICONDA_URL}/${MINICONDA_FILE}" - bash $MINICONDA_FILE -b - - # Install conda build dependency - - | - echo "" - echo "Configuring conda." - source /home/travis/miniconda3/bin/activate root - conda install -n root -c conda-forge --quiet --yes conda-forge-ci-setup=2 conda-build - -before_script: - - git clone --no-checkout {{ arrow.remote }} arrow - - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - - git -C arrow checkout FETCH_HEAD - - - pushd arrow/dev/tasks/conda-recipes - - # Configure conda - - setup_conda_rc ./ ./ .ci_support/{{ config }}.yaml - - source run_conda_forge_build_setup - -script: - - | - conda build --croot $TRAVIS_HOME/conda_build_root \ - --output-folder . \ - -m .ci_support/{{ config }}.yaml \ - parquet-cpp arrow-cpp pyarrow - -deploy: - provider: releases - api_key: $CROSSBOW_GITHUB_TOKEN - file_glob: true - file: $TRAVIS_BUILD_DIR/arrow/dev/tasks/conda-recipes/linux-64/*.tar.bz2 - skip_cleanup: true - on: - tags: true - -notifications: - email: - - {{ job.email }} diff --git a/dev/tasks/conda-recipes/travis.osx.yml b/dev/tasks/conda-recipes/travis.osx.yml deleted file mode 100644 index f32a611511b..00000000000 --- a/dev/tasks/conda-recipes/travis.osx.yml +++ /dev/null @@ -1,62 +0,0 @@ -os: osx -osx_image: xcode9.4 -language: generic - -# don't build twice -if: tag IS blank - -env: - global: - - TRAVIS_TAG={{ task.tag }} - - ARROW_VERSION={{ arrow.no_rc_version }} - - PYTHONUNBUFFERED=1 - -install: - # Install Miniconda. - - | - echo "" - echo "Installing a fresh version of Miniconda." - MINICONDA_URL="https://repo.continuum.io/miniconda" - MINICONDA_FILE="Miniconda3-latest-MacOSX-x86_64.sh" - curl -L -O "${MINICONDA_URL}/${MINICONDA_FILE}" - bash $MINICONDA_FILE -b - # Install conda build dependency - - | - echo "" - echo "Configuring conda." - source /Users/travis/miniconda3/bin/activate root - conda install -n root -c conda-forge --quiet --yes conda-forge-ci-setup=2 conda-build - - -before_script: - - git clone --no-checkout {{ arrow.remote }} arrow - - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - - git -C arrow checkout FETCH_HEAD - - - pushd arrow/dev/tasks/conda-recipes - - # Configure conda - - setup_conda_rc ./ ./ .ci_support/{{ config }}.yaml - - CONFIG={{ config }} source run_conda_forge_build_setup - - # Compiler cleanup - - mangle_compiler ./ ./ ./.ci_support/{{ config }}.yaml - -script: - - | - conda build --output-folder . \ - -m ./.ci_support/{{ config }}.yaml \ - parquet-cpp arrow-cpp pyarrow - -deploy: - provider: releases - api_key: $CROSSBOW_GITHUB_TOKEN - file_glob: true - file: $TRAVIS_BUILD_DIR/arrow/dev/tasks/conda-recipes/osx-64/*.tar.bz2 - skip_cleanup: true - on: - tags: true - -notifications: - email: - - {{ job.email }} From 2aa497748343384181c33c98cb2558a59e08b3f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 25 Jun 2019 01:21:34 +0200 Subject: [PATCH 28/39] azure template for docker tests --- dev/tasks/docker-tests/azure.linux.yml | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 dev/tasks/docker-tests/azure.linux.yml diff --git a/dev/tasks/docker-tests/azure.linux.yml b/dev/tasks/docker-tests/azure.linux.yml new file mode 100644 index 00000000000..b3f1f9b0ca7 --- /dev/null +++ b/dev/tasks/docker-tests/azure.linux.yml @@ -0,0 +1,28 @@ +jobs: +- job: linux + pool: + vmImage: ubuntu-16.04 + timeoutInMinutes: 360 + + steps: + - task: DockerInstaller@0 + displayName: Docker Installer + inputs: + dockerVersion: 17.09.0-ce + releaseType: stable + + - script: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout {{ arrow.head }} + git -C arrow submodule update --init --recursive + displayName: Clone arrow + + - script: | + pushd arrow + {%- for command in commands %} + {{ command }} + {%- endfor %} + popd + workingDirectory: arrow + displayName: Run docker test From 9db3db1dda9e10ff63a2c01b9b87f4e8c6e9e82b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 25 Jun 2019 01:48:14 +0200 Subject: [PATCH 29/39] use pip3 --- dev/tasks/conda-recipes/azure.linux.yml | 2 +- dev/tasks/conda-recipes/azure.osx.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index d34c8806b3a..43142236ef1 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -39,7 +39,7 @@ jobs: architecture: 'x64' - script: | - pip install github3-py click + pip3 install github3-py click python upload-assets.py \ --sha {{ task.branch }} \ --tag {{ task.tag }} \ diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index 2f38e2e7a9d..4a668a7cf1b 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -76,7 +76,7 @@ jobs: architecture: 'x64' - script: | - pip install github3-py click + pip3 install github3-py click python upload-assets.py \ --sha {{ task.branch }} \ --tag {{ task.tag }} \ From 2a598945d7fde54ac375bd0a7b4a71956a430fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 25 Jun 2019 02:15:55 +0200 Subject: [PATCH 30/39] tabulate win template --- dev/tasks/conda-recipes/azure.linux.yml | 4 +- dev/tasks/conda-recipes/azure.osx.yml | 4 +- dev/tasks/conda-recipes/azure.win.yml | 195 ++++++++++++------------ 3 files changed, 101 insertions(+), 102 deletions(-) diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index 43142236ef1..ac25f42fea1 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -31,8 +31,8 @@ jobs: # Using github release tries to find a common ancestor between the currently # pushed tag and the latest tag of the github repository (don't know why). - # The tag upload took 43 minutes because of this scanning, so use an - # alternative upload script. + # The tag upload took 43 minutes because of this scan, so use an alternative + # upload script. - task: UsePythonVersion@0 inputs: versionSpec: '3.6' diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index 4a668a7cf1b..f3c19c5b120 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -68,8 +68,8 @@ jobs: # Using github release tries to find a common ancestor between the currently # pushed tag and the latest tag of the github repository (don't know why). - # The tag upload took 43 minutes because of this scanning, so use an - # alternative upload script. + # The tag upload took 43 minutes because of this scan, so use an alternative + # upload script. - task: UsePythonVersion@0 inputs: versionSpec: '3.6' diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml index 5f76fd6fc35..0b957258403 100755 --- a/dev/tasks/conda-recipes/azure.win.yml +++ b/dev/tasks/conda-recipes/azure.win.yml @@ -9,101 +9,100 @@ jobs: CONDA_BLD_PATH: D:\\bld\\ UPLOAD_PACKAGES: False steps: - - script: | - choco install vcpython27 -fdv -y --debug - condition: contains(variables['CONFIG'], 'vs2008') - displayName: Install vcpython27.msi (if needed) - - - powershell: | - Set-PSDebug -Trace 1 - - $batchcontent = @" - ECHO ON - SET vcpython=C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0 - - DIR "%vcpython%" - - CALL "%vcpython%\vcvarsall.bat" %* - "@ - - $batchDir = "C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\VC" - $batchPath = "$batchDir" + "\vcvarsall.bat" - New-Item -Path $batchPath -ItemType "file" -Force - - Set-Content -Value $batchcontent -Path $batchPath - - Get-ChildItem -Path $batchDir - - Get-ChildItem -Path ($batchDir + '\..') - - condition: contains(variables['CONFIG'], 'vs2008') - displayName: Patch vs2008 (if needed) - - - task: CondaEnvironment@1 - inputs: - packageSpecs: 'python=3.6 conda-build conda conda-forge::conda-forge-ci-setup=2' # Optional - installOptions: "-c conda-forge" - updateConda: false - displayName: Install conda-build and activate environment - - - script: set PYTHONUNBUFFERED=1 - - - script: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout {{ arrow.head }} - displayName: Clone arrow - - # Configure the VM - - script: setup_conda_rc .\ .\ .\.ci_support\%CONFIG%.yaml - workingDirectory: arrow\dev\tasks\conda-recipes - - # Configure the VM. - - script: | - set "CI=azure" - run_conda_forge_build_setup - displayName: conda-forge build setup - - - script: | - rmdir C:\strawberry /s /q - continueOnError: true - displayName: remove strawberryperl - - # Special cased version setting some more things! - - script: | - conda.exe build arrow-cpp parquet-cpp pyarrow -m .ci_support\%CONFIG%.yaml - displayName: Build recipe (vs2008) - workingDirectory: arrow\dev\tasks\conda-recipes - env: - VS90COMNTOOLS: "C:\\Program Files (x86)\\Common Files\\Microsoft\\Visual C++ for Python\\9.0\\VC\\bin" - PYTHONUNBUFFERED: 1 - condition: contains(variables['CONFIG'], 'vs2008') - - - script: | - conda.exe build arrow-cpp parquet-cpp pyarrow -m .ci_support\%CONFIG%.yaml - displayName: Build recipe - workingDirectory: arrow\dev\tasks\conda-recipes - env: - PYTHONUNBUFFERED: 1 - condition: not(contains(variables['CONFIG'], 'vs2008')) - - # Using github release tries to find a common ancestor between the currently - # pushed tag and the latest tag of the github repository (don't know why). - # The tag upload took 43 minutes because of this scanning, so use an - # alternative upload script. - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.6' - architecture: 'x64' - - - script: | - pip install github3-py click - python upload-assets.py \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} \ - --pattern "conda-recipes/build_artifacts/*.tar.bz2" - env: - CROSSBOW_GITHUB_REPO: $(Build.Repository.Name) - CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN) - displayName: Upload packages as a GitHub release - workingDirectory: arrow/dev/tasks + - script: | + choco install vcpython27 -fdv -y --debug + condition: contains(variables['CONFIG'], 'vs2008') + displayName: Install vcpython27.msi (if needed) + + - powershell: | + Set-PSDebug -Trace 1 + + $batchcontent = @" + ECHO ON + SET vcpython=C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0 + + DIR "%vcpython%" + + CALL "%vcpython%\vcvarsall.bat" %* + "@ + + $batchDir = "C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\VC" + $batchPath = "$batchDir" + "\vcvarsall.bat" + New-Item -Path $batchPath -ItemType "file" -Force + + Set-Content -Value $batchcontent -Path $batchPath + + Get-ChildItem -Path $batchDir + + Get-ChildItem -Path ($batchDir + '\..') + + condition: contains(variables['CONFIG'], 'vs2008') + displayName: Patch vs2008 (if needed) + + - task: CondaEnvironment@1 + inputs: + packageSpecs: 'python=3.6 conda-build conda conda-forge::conda-forge-ci-setup=2' # Optional + installOptions: "-c conda-forge" + updateConda: false + displayName: Install conda-build and activate environment + + - script: set PYTHONUNBUFFERED=1 + + - script: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout {{ arrow.head }} + displayName: Clone arrow + + # Configure the VM + - script: setup_conda_rc .\ .\ .\.ci_support\%CONFIG%.yaml + workingDirectory: arrow\dev\tasks\conda-recipes + + # Configure the VM. + - script: | + set "CI=azure" + run_conda_forge_build_setup + displayName: conda-forge build setup + + - script: | + rmdir C:\strawberry /s /q + continueOnError: true + displayName: remove strawberryperl + + # Special cased version setting some more things! + - script: | + conda.exe build arrow-cpp parquet-cpp pyarrow -m .ci_support\%CONFIG%.yaml + displayName: Build recipe (vs2008) + workingDirectory: arrow\dev\tasks\conda-recipes + env: + VS90COMNTOOLS: "C:\\Program Files (x86)\\Common Files\\Microsoft\\Visual C++ for Python\\9.0\\VC\\bin" + PYTHONUNBUFFERED: 1 + condition: contains(variables['CONFIG'], 'vs2008') + + - script: | + conda.exe build arrow-cpp parquet-cpp pyarrow -m .ci_support\%CONFIG%.yaml + displayName: Build recipe + workingDirectory: arrow\dev\tasks\conda-recipes + env: + PYTHONUNBUFFERED: 1 + condition: not(contains(variables['CONFIG'], 'vs2008')) + + # Using github release tries to find a common ancestor between the currently + # pushed tag and the latest tag of the github repository (don't know why). + # The tag upload took 43 minutes because of this scan, so use an alternative upload script. + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.6' + architecture: 'x64' + + - script: | + pip install github3-py click + python upload-assets.py \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + --pattern "conda-recipes/build_artifacts/*.tar.bz2" + env: + CROSSBOW_GITHUB_REPO: $(Build.Repository.Name) + CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN) + displayName: Upload packages as a GitHub release + workingDirectory: arrow/dev/tasks From f75efae18c609746343a96cf6ead60be689ec725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 25 Jun 2019 02:23:39 +0200 Subject: [PATCH 31/39] use pip module for installing dependencies --- dev/tasks/conda-recipes/azure.linux.yml | 2 +- dev/tasks/conda-recipes/azure.osx.yml | 3 ++- dev/tasks/conda-recipes/azure.win.yml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index ac25f42fea1..0c5127c7343 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -39,7 +39,7 @@ jobs: architecture: 'x64' - script: | - pip3 install github3-py click + python -m pip install github3-py click python upload-assets.py \ --sha {{ task.branch }} \ --tag {{ task.tag }} \ diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index f3c19c5b120..6b30f191b06 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -63,6 +63,7 @@ jobs: conda build arrow-cpp parquet-cpp pyarrow \ -m ./.ci_support/${CONFIG}.yaml \ --clobber-file ./.ci_support/clobber_${CONFIG}.yaml + source deactivate workingDirectory: arrow/dev/tasks/conda-recipes displayName: Build recipes @@ -76,7 +77,7 @@ jobs: architecture: 'x64' - script: | - pip3 install github3-py click + python -m pip install github3-py click python upload-assets.py \ --sha {{ task.branch }} \ --tag {{ task.tag }} \ diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml index 0b957258403..b5b5cfe9775 100755 --- a/dev/tasks/conda-recipes/azure.win.yml +++ b/dev/tasks/conda-recipes/azure.win.yml @@ -96,7 +96,7 @@ jobs: architecture: 'x64' - script: | - pip install github3-py click + python -m pip install github3-py click python upload-assets.py \ --sha {{ task.branch }} \ --tag {{ task.tag }} \ From 8f58839e1786b5812138f1fba1bb704444944239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 25 Jun 2019 11:10:51 +0200 Subject: [PATCH 32/39] use the default python on osx --- dev/tasks/conda-recipes/azure.osx.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index 6b30f191b06..dafb45c1c2b 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -63,7 +63,6 @@ jobs: conda build arrow-cpp parquet-cpp pyarrow \ -m ./.ci_support/${CONFIG}.yaml \ --clobber-file ./.ci_support/clobber_${CONFIG}.yaml - source deactivate workingDirectory: arrow/dev/tasks/conda-recipes displayName: Build recipes @@ -71,11 +70,6 @@ jobs: # pushed tag and the latest tag of the github repository (don't know why). # The tag upload took 43 minutes because of this scan, so use an alternative # upload script. - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.6' - architecture: 'x64' - - script: | python -m pip install github3-py click python upload-assets.py \ From cbb9c9ce79932e982d68bab869c4231495bea2b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 25 Jun 2019 12:05:44 +0200 Subject: [PATCH 33/39] rat --- dev/tasks/docker-tests/azure.linux.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/dev/tasks/docker-tests/azure.linux.yml b/dev/tasks/docker-tests/azure.linux.yml index b3f1f9b0ca7..dea73a0d1ca 100644 --- a/dev/tasks/docker-tests/azure.linux.yml +++ b/dev/tasks/docker-tests/azure.linux.yml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + jobs: - job: linux pool: From 95cb442170401788b698c891df1bf4af375e564f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 25 Jun 2019 12:57:57 +0200 Subject: [PATCH 34/39] use FETCH_HEAD in the CI templates --- dev/tasks/conda-recipes/azure.linux.yml | 2 +- dev/tasks/conda-recipes/azure.osx.yml | 2 +- dev/tasks/conda-recipes/azure.win.yml | 2 +- dev/tasks/docker-tests/azure.linux.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index 0c5127c7343..4b7539e8cb0 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -22,7 +22,7 @@ jobs: - script: | git clone --no-checkout {{ arrow.remote }} arrow git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout {{ arrow.head }} + git -C arrow checkout FETCH_HEAD displayName: Clone arrow - script: CI=azure ./run_docker_build.sh diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index dafb45c1c2b..b3e85e63d94 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -29,7 +29,7 @@ jobs: - script: | git clone --no-checkout {{ arrow.remote }} arrow git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout {{ arrow.head }} + git -C arrow checkout FETCH_HEAD displayName: Clone arrow - script: | diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml index b5b5cfe9775..02fc91ae63f 100755 --- a/dev/tasks/conda-recipes/azure.win.yml +++ b/dev/tasks/conda-recipes/azure.win.yml @@ -51,7 +51,7 @@ jobs: - script: | git clone --no-checkout {{ arrow.remote }} arrow git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout {{ arrow.head }} + git -C arrow checkout FETCH_HEAD displayName: Clone arrow # Configure the VM diff --git a/dev/tasks/docker-tests/azure.linux.yml b/dev/tasks/docker-tests/azure.linux.yml index dea73a0d1ca..8d54b6977af 100644 --- a/dev/tasks/docker-tests/azure.linux.yml +++ b/dev/tasks/docker-tests/azure.linux.yml @@ -31,7 +31,7 @@ jobs: - script: | git clone --no-checkout {{ arrow.remote }} arrow git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout {{ arrow.head }} + git -C arrow checkout FETCH_HEAD git -C arrow submodule update --init --recursive displayName: Clone arrow From 111964957971aae0edcc1e2c1ae9187dbf35a5cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 25 Jun 2019 13:01:24 +0200 Subject: [PATCH 35/39] fix artifact patterns --- dev/tasks/conda-recipes/azure.linux.yml | 2 +- dev/tasks/conda-recipes/azure.osx.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index 4b7539e8cb0..355f3b18b84 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -43,7 +43,7 @@ jobs: python upload-assets.py \ --sha {{ task.branch }} \ --tag {{ task.tag }} \ - --pattern "conda-recipes/build_artifacts/*.tar.bz2" + --pattern "conda-recipes/build_artifacts/linux-64/*.tar.bz2" env: CROSSBOW_GITHUB_REPO: $(Build.Repository.Name) CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN) diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index b3e85e63d94..4226a131810 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -75,7 +75,7 @@ jobs: python upload-assets.py \ --sha {{ task.branch }} \ --tag {{ task.tag }} \ - --pattern "conda-recipes/build_artifacts/*.tar.bz2" + --pattern "conda-recipes/build_artifacts/osx-64/*.tar.bz2" env: CROSSBOW_GITHUB_REPO: $(Build.Repository.Name) CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN) From 68d88a83334844a9897a6b4bda85ed985d5e4ef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 25 Jun 2019 15:42:12 +0200 Subject: [PATCH 36/39] combine status api and checks api --- dev/tasks/crossbow.py | 98 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 83 insertions(+), 15 deletions(-) diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py index db626552fa4..ab1b30bc5e2 100755 --- a/dev/tasks/crossbow.py +++ b/dev/tasks/crossbow.py @@ -20,18 +20,19 @@ import os import re import time -import click import hashlib +from io import StringIO +from pathlib import Path +from textwrap import dedent +from datetime import datetime +from collections import namedtuple + +import click import gnupg import toolz import pygit2 import github3 import jira.client - -from io import StringIO -from pathlib import Path -from textwrap import dedent -from datetime import datetime from jinja2 import Template, StrictUndefined from setuptools_scm.git import parse as parse_git_version from ruamel.yaml import YAML @@ -73,6 +74,7 @@ def unflatten(mapping): # - travis reads from the master branch and applies the rules # - circle requires the configuration to be present on all branch, even ones # that are configured to be skipped +# - azure skips branches without azure-pipelines.yml by default _default_travis_yml = """ branches: @@ -106,7 +108,6 @@ def unflatten(mapping): _default_tree = { '.travis.yml': _default_travis_yml, '.circleci/config.yml': _default_circle_yml - # azure-pipelines.yml' TODO? } @@ -227,6 +228,10 @@ def credentials(self, url, username_from_url, allowed_types): return None +def _git_ssh_to_https(url): + return url.replace('git@github.com:', 'https://github.com/') + + class Repo: """Base class for interaction with local git repositories @@ -244,6 +249,7 @@ def __init__(self, path, github_token=None, require_https=False): self.repo = pygit2.Repository(str(self.path)) self.github_token = github_token self.require_https = require_https + self._github_repo = None # set by as_github_repo() self._updated_refs = [] def __str__(self): @@ -389,13 +395,14 @@ def _parse_github_user_repo(self): def as_github_repo(self): """Converts it to a repository object which wraps the GitHub API""" - username, reponame = self._parse_github_user_repo() - gh = github3.login(token=self.github_token) - return gh.repository(username, reponame) + if self._github_repo is None: + username, reponame = self._parse_github_user_repo() + gh = github3.login(token=self.github_token) + return gh.repository(username, reponame) + return self._github_repo -def _git_ssh_to_https(url): - return url.replace('git@github.com:', 'https://github.com/') +CombinedStatus = namedtuple('CombinedStatus', ('state', 'total_count')) class Queue(Repo): @@ -440,10 +447,71 @@ def put(self, job, prefix='build'): # create job's branch with its description return self.create_branch(job.branch, files=job.render_files()) - def github_statuses(self, job): + def combined_status(self, task): + """Combine the results from status and checks API to a single state. + + Azure pipelines uses checks API which doesn't provide a combined + interface like status API does, so we need to manually combine + both the commit statuses and the commit checks coming from + different API endpoint + + Status.state: error, failure, pending or success, default pending + CheckRun.status: queued, in_progress or completed, default: queued + CheckRun.conclusion: success, failure, neutral, cancelled, timed_out + or action_required, only set if + CheckRun.status == 'completed' + + 1. Convert CheckRun's status and conslusion to one of Status.state + 2. Merge the states based on the following rules: + - failure if any of the contexts report as error or failure + - pending if there are no statuses or a context is pending + - success if the latest status for all contexts is success + error otherwise. + + Parameters + ---------- + task : Task + Task to query the combined status for. + + Returns + ------- + combined_state: CombinedStatus( + state='error|failure|pending|success', + total_count='number of statuses and checks' + ) + """ repo = self.as_github_repo() - return {name: repo.commit(task.commit).status() - for name, task in job.tasks.items()} + commit = repo.commit(task.commit) + states = [] + + for status in commit.status().statuses: + states.append(status.state) + + for check in commit.check_runs(): + if check.status == 'completed': + if check.conclusion in {'success', 'failure'}: + states.append(check.conclusion) + elif check.conclusion in {'cancelled', 'timed_out', + 'action_required'}: + states.append('error') + # omit `neutral` conslusion + else: + states.append('pending') + + # it could be more effective, but the following is more descriptive + if any(state in {'error', 'failure'} for state in states): + combined_state = 'failure' + elif any(state == 'pending' for state in states): + combined_state = 'pending' + elif all(state == 'success' for state in states): + combined_state = 'success' + else: + combined_state = 'error' + + return CombinedStatus(state=combined_state, total_count=len(states)) + + def github_statuses(self, job): + return toolz.valmap(self.combined_status, job.tasks) def github_assets(self, task): repo = self.as_github_repo() From b6a86076b6098efa3aa4f9d6954e6dbe4f44ee3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 25 Jun 2019 15:49:42 +0200 Subject: [PATCH 37/39] configure output folder for the artifacts --- dev/tasks/conda-recipes/azure.osx.yml | 3 ++- dev/tasks/conda-recipes/build_steps.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index 4226a131810..5f5c8d05324 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -62,7 +62,8 @@ jobs: source activate base conda build arrow-cpp parquet-cpp pyarrow \ -m ./.ci_support/${CONFIG}.yaml \ - --clobber-file ./.ci_support/clobber_${CONFIG}.yaml + --clobber-file ./.ci_support/clobber_${CONFIG}.yaml \ + --output-folder ./build_artifacts workingDirectory: arrow/dev/tasks/conda-recipes displayName: Build recipes diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh index 889574d17bb..9f243f3a763 100755 --- a/dev/tasks/conda-recipes/build_steps.sh +++ b/dev/tasks/conda-recipes/build_steps.sh @@ -33,6 +33,7 @@ conda build \ "${FEEDSTOCK_ROOT}/parquet-cpp" \ "${FEEDSTOCK_ROOT}/pyarrow" \ -m "${CI_SUPPORT}/${CONFIG}.yaml" \ - --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" + --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" \ + --output-folder "${FEEDSTOCK_ROOT}/build_artifacts" touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}" From 0c470bb6a44fbfd75b546d3a105df12c9f618532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 25 Jun 2019 17:10:50 +0200 Subject: [PATCH 38/39] readme fixes --- dev/tasks/conda-recipes/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/tasks/conda-recipes/README.md b/dev/tasks/conda-recipes/README.md index 2041fb7ec26..d5cfe3e6979 100644 --- a/dev/tasks/conda-recipes/README.md +++ b/dev/tasks/conda-recipes/README.md @@ -33,11 +33,11 @@ latest releases. ### Backporting from the upstream feedstocks -So most of the cases these recipes are more accurate, altough the upstream -feedstock regurarly receive automatic updates by the conda-forge team, and -we need to backport those changes to the crossbow recipes. Most of these -updates are touching the version pinning files (``.ci_support`) and other CI -related configuration files. +In most of the cases these recipes are more accurate, then the upstream +feedstocks. Altough the upstream feedstocks regurarly receive automatic updates +by the conda-forge team so we need to backport those changes to the crossbow +recipes. Most of these updates are touching the version pinning files +(under `.ci_support`) and other CI related configuration files. Because all three recipes must be built in the same continuous integration job prefer porting from the [pyarrow feedstock][pyarrow-feedstock]. @@ -66,4 +66,4 @@ copied to the upstream feedstocks. [arrow-cpp-feedstock]: https://github.com/conda-forge/arrow-cpp-feedstock [parquet-cpp-feedstock]: https://github.com/conda-forge/parquet-cpp-feedstock [pyarrow-cpp-feedstock]: https://github.com/conda-forge/pyarrow-feedstock -[matric-definition]: https://github.com/conda-forge/pyarrow-feedstock/blob/master/.azure-pipelines/azure-pipelines-linux.yml#L12 +[matrix-definition]: https://github.com/conda-forge/pyarrow-feedstock/blob/master/.azure-pipelines/azure-pipelines-linux.yml#L12 From 20f6cecd3e8a8e476387a6441d6572ff94fa6e5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 25 Jun 2019 18:09:12 +0200 Subject: [PATCH 39/39] update conda-win artifact patterns --- dev/tasks/tasks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 6aead4d378b..9030aef9918 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -142,7 +142,7 @@ tasks: params: config: win_c_compilervs2015cxx_compilervs2015python3.6 artifacts: - - arrow-cpp-{no_rc_version}-py36_vc14(h[a-z0-9]+)_0.tar.bz2 + - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0.tar.bz2 - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0.tar.bz2 conda-win-vs2015-py37: @@ -152,7 +152,7 @@ tasks: params: config: win_c_compilervs2015cxx_compilervs2015python3.7 artifacts: - - arrow-cpp-{no_rc_version}-py37_vc14(h[a-z0-9]+)_0.tar.bz2 + - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0.tar.bz2 - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0.tar.bz2 ############################## Wheel Linux ##################################