ARROW-6937: [Packaging][Python] Fix conda linux and OSX wheel nightly builds

dev/tasks/conda-recipes/azure.linux.yml

@@ -26,9 +26,10 @@ jobs:
       git -C arrow submodule update --init --recursive
     displayName: Clone arrow
 
-  - script: CI=azure ./run_docker_build.sh
+  - script: |
+      mkdir build_artifacts
+      CI=azure arrow/dev/tasks/conda-recipes/run_docker_build.sh $(pwd)/build_artifacts
     displayName: Run docker build
-    workingDirectory: arrow/dev/tasks/conda-recipes
 
   # Using github release tries to find a common ancestor between the
   # currently pushed tag and the latest tag of the github repository
@@ -47,7 +48,7 @@ jobs:
         upload-artifacts \
         --sha {{ task.branch }} \
         --tag {{ task.tag }} \
-        --pattern "arrow/dev/tasks/conda-recipes/build_artifacts/linux-64/*.tar.bz2"
+        --pattern "build_artifacts/linux-64/*.tar.bz2"
     env:
       CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN)
     displayName: Upload packages as a GitHub release

dev/tasks/conda-recipes/build_steps.sh

@@ -6,6 +6,9 @@
 # benefit from the improvement.
 
 set -xeuo pipefail
+
+output_dir=${1}
+
 export PYTHONUNBUFFERED=1
 export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}"
 export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support"
@@ -14,7 +17,7 @@ export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml"
 cat >~/.condarc <<CONDARC
 
 conda-build:
- root-dir: ${FEEDSTOCK_ROOT}/build_artifacts
+ root-dir: ${output_dir}
 
 CONDARC
 
@@ -28,12 +31,14 @@ source run_conda_forge_build_setup
 # make the build number clobber
 make_build_number "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}"
 
+export CONDA_BLD_PATH="${output_dir}"
+
 conda build \
     "${FEEDSTOCK_ROOT}/arrow-cpp" \
     "${FEEDSTOCK_ROOT}/parquet-cpp" \
     "${FEEDSTOCK_ROOT}/pyarrow" \
     -m "${CI_SUPPORT}/${CONFIG}.yaml" \
     --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" \
-    --output-folder "${FEEDSTOCK_ROOT}/build_artifacts"
+    --output-folder "${output_dir}"
 
-touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done-${CONFIG}"
+touch "${output_dir}/conda-forge-build-done-${CONFIG}"

dev/tasks/conda-recipes/run_docker_build.sh

@@ -7,6 +7,8 @@
 
 set -xeo pipefail
 
+build_dir=${1}
+
 THISDIR="$( cd "$( dirname "$0" )" >/dev/null && pwd )"
 ARROW_ROOT=$(cd "$THISDIR/../../.."; pwd;)
 FEEDSTOCK_ROOT=$THISDIR
@@ -23,8 +25,6 @@ if hash docker-machine 2> /dev/null && docker-machine active > /dev/null; then
     export HOST_USER_ID=$(docker-machine ssh $(docker-machine active) id -u)
 fi
 
-ARTIFACTS="$FEEDSTOCK_ROOT/build_artifacts"
-
 if [ -z "$CONFIG" ]; then
     set +x
     FILES=`ls .ci_support/linux_*`
@@ -46,8 +46,8 @@ if [ -z "${DOCKER_IMAGE}" ]; then
     fi
 fi
 
-mkdir -p "$ARTIFACTS"
-DONE_CANARY="$ARTIFACTS/conda-forge-build-done-${CONFIG}"
+mkdir -p "${build_dir}"
+DONE_CANARY="${build_dir}/conda-forge-build-done-${CONFIG}"
 rm -f "$DONE_CANARY"
 
 if [ -z "${CI}" ]; then
@@ -58,14 +58,15 @@ export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}"
 docker run ${DOCKER_RUN_ARGS} \
            --shm-size=2G \
            -v "${ARROW_ROOT}":/arrow:rw,z \
+           -v "${build_dir}":/build:rw \
            -e FEEDSTOCK_ROOT="/arrow/dev/tasks/conda-recipes" \
            -e CONFIG \
            -e HOST_USER_ID \
            -e UPLOAD_PACKAGES \
            -e ARROW_VERSION \
            -e CI \
            $DOCKER_IMAGE \
-           bash /arrow/dev/tasks/conda-recipes/build_steps.sh
+           bash /arrow/dev/tasks/conda-recipes/build_steps.sh /build
 
 # verify that the end of the script was reached
 test -f "$DONE_CANARY"

dev/tasks/python-wheels/travis.osx.yml

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 os: osx
-osx_image: xcode8.3
+osx_image: xcode9.3
 language: generic
 
 # don't build twice
@@ -47,7 +47,7 @@ before_install:
   - brew uninstall boost cgal postgis sfcgal
   - brew update
   - brew upgrade cmake
-  - brew install bison flex grpc openssl@1.1 llvm@7 zlib gperftools
+  - travis_wait 30 brew install bison flex grpc openssl@1.1 llvm@7 zlib gperftools
   # Remove shared grpc libraries installed by brew to make sure
   # we are linked against the static ones.
   - rm -f /usr/local/opt/grpc/lib/*.dylib
@@ -81,7 +81,7 @@ install:
   # move built wheels to a top level directory
   - mv -v arrow/python/dist/* dist/
   # reinstall openssl because travis' deployment script depends on it
-  - brew install openssl
+  - brew install openssl@1.1
 
 deploy:
   provider: releases