From 490ceacfedf5c14a0b3c078339571b335472f969 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 3 Feb 2020 15:06:14 -0600 Subject: [PATCH 1/3] pip3 no longer in miniconda --- dev/release/verify-release-candidate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index bae762d3274..7ca6703c3fc 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -503,7 +503,7 @@ test_integration() { export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/release - pip3 install -e dev/archery + pip install -e dev/archery INTEGRATION_TEST_ARGS="" From 26da759823ee943f324d7f046482c913492ac1d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 4 Feb 2020 14:04:21 +0100 Subject: [PATCH 2/3] always set up miniconda --- dev/release/verify-release-candidate.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 7ca6703c3fc..0e907ffff08 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -30,7 +30,8 @@ # LD_LIBRARY_PATH. # # To reuse build artifacts between runs set TMPDIR environment variable to -# a directory where the temporary files should be placed to. +# a directory where the temporary files should be placed to, note that this +# directory is not cleaned up automatically. case $# in 3) ARTIFACT="$1" @@ -126,7 +127,7 @@ test_binary() { local download_dir=binaries mkdir -p ${download_dir} - python3 $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER --dest=${download_dir} + python $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER --dest=${download_dir} verify_dir_artifact_signatures ${download_dir} } @@ -207,11 +208,13 @@ setup_tempdir() { echo "Failed to verify release candidate. See ${TMPDIR} for details." fi } - trap cleanup EXIT if [ -z "${TMPDIR}" ]; then + # clean up automatically if TMPDIR is not defined TMPDIR=$(mktemp -d -t "$1.XXXXX") + trap cleanup EXIT else + # don't clean up automatically mkdir -p "${TMPDIR}" fi } @@ -552,7 +555,6 @@ test_source_distribution() { test_package_java fi if [ ${TEST_CPP} -gt 0 ]; then - setup_miniconda test_and_install_cpp fi if [ ${TEST_CSHARP} -gt 0 ]; then @@ -686,9 +688,9 @@ test_wheels() { conda create -yq -n py3-base python=3.7 conda activate py3-base - python3 $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \ - --regex=${filter_regex} \ - --dest=${download_dir} + python $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \ + --regex=${filter_regex} \ + --dest=${download_dir} verify_dir_artifact_signatures ${download_dir} @@ -751,6 +753,9 @@ setup_tempdir "arrow-${VERSION}" echo "Working in sandbox ${TMPDIR}" cd ${TMPDIR} +setup_miniconda +echo "Using miniconda environment ${MINICONDA}" + if [ "${ARTIFACT}" == "source" ]; then dist_name="apache-arrow-${VERSION}" if [ ${TEST_SOURCE} -gt 0 ]; then @@ -766,7 +771,6 @@ if [ "${ARTIFACT}" == "source" ]; then popd elif [ "${ARTIFACT}" == "wheels" ]; then import_gpg_keys - setup_miniconda test_wheels else import_gpg_keys From fcf3ae643fb5a2316401c72270d51d23d392038d Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 4 Feb 2020 11:23:37 -0600 Subject: [PATCH 3/3] conda environment must be activated for other steps to work if C++ is disabled. Fix selective disabling of integration test components in archery --- dev/archery/archery/integration/runner.py | 12 ++++++------ dev/release/verify-release-candidate.sh | 22 +++++++++++++--------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/dev/archery/archery/integration/runner.py b/dev/archery/archery/integration/runner.py index 9fa3f6194c7..60ee8f3b856 100644 --- a/dev/archery/archery/integration/runner.py +++ b/dev/archery/archery/integration/runner.py @@ -227,23 +227,23 @@ def get_static_json_files(): ] -def run_all_tests(enable_cpp=True, enable_java=True, enable_js=True, - enable_go=True, run_flight=False, +def run_all_tests(with_cpp=True, with_java=True, with_js=True, + with_go=True, run_flight=False, tempdir=None, **kwargs): tempdir = tempdir or tempfile.mkdtemp() testers = [] - if enable_cpp: + if with_cpp: testers.append(CPPTester(kwargs)) - if enable_java: + if with_java: testers.append(JavaTester(kwargs)) - if enable_js: + if with_js: testers.append(JSTester(kwargs)) - if enable_go: + if with_go: testers.append(GoTester(kwargs)) static_json_files = get_static_json_files() diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 0e907ffff08..88974529ecf 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -238,6 +238,15 @@ setup_miniconda() { fi . $MINICONDA/etc/profile.d/conda.sh + + conda create -n arrow-test -y -q -c conda-forge \ + python=3.6 \ + nomkl \ + numpy \ + pandas \ + six \ + cython + conda activate arrow-test } # Build and test Java (Requires newer Maven -- I used 3.3.9) @@ -254,15 +263,6 @@ test_package_java() { # Build and test C++ test_and_install_cpp() { - conda create -n arrow-test -y -q -c conda-forge \ - python=3.6 \ - nomkl \ - numpy \ - pandas \ - six \ - cython - conda activate arrow-test - mkdir -p cpp/build pushd cpp/build @@ -764,6 +764,10 @@ if [ "${ARTIFACT}" == "source" ]; then tar xf ${dist_name}.tar.gz else mkdir -p ${dist_name} + if [ ! -f ${TEST_ARCHIVE} ]; then + echo "${TEST_ARCHIVE} not found, did you mean to pass TEST_SOURCE=1?" + exit 1 + fi tar xf ${TEST_ARCHIVE} -C ${dist_name} --strip-components=1 fi pushd ${dist_name}