From 0601c4c9f9c21c420ba875899faae1fc9d2f1316 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 22 Jun 2021 17:56:11 +0200 Subject: [PATCH 1/3] Rename conda requirements files to have txt extension instead of yml --- .github/workflows/python.yml | 6 ------ ci/appveyor-cpp-setup.bat | 6 +++--- ci/{conda_env_archery.yml => conda_env_archery.txt} | 0 ci/{conda_env_cpp.yml => conda_env_cpp.txt} | 0 ci/{conda_env_gandiva.yml => conda_env_gandiva.txt} | 0 ...onda_env_gandiva_win.yml => conda_env_gandiva_win.txt} | 0 ci/{conda_env_python.yml => conda_env_python.txt} | 0 ci/{conda_env_r.yml => conda_env_r.txt} | 0 ci/{conda_env_sphinx.yml => conda_env_sphinx.txt} | 0 ci/{conda_env_unix.yml => conda_env_unix.txt} | 0 ci/docker/conda-cpp.dockerfile | 8 ++++---- ci/docker/conda-integration.dockerfile | 6 +++--- ci/docker/conda-python.dockerfile | 4 ++-- ci/docker/conda.dockerfile | 4 ++-- 14 files changed, 14 insertions(+), 20 deletions(-) rename ci/{conda_env_archery.yml => conda_env_archery.txt} (100%) rename ci/{conda_env_cpp.yml => conda_env_cpp.txt} (100%) rename ci/{conda_env_gandiva.yml => conda_env_gandiva.txt} (100%) rename ci/{conda_env_gandiva_win.yml => conda_env_gandiva_win.txt} (100%) rename ci/{conda_env_python.yml => conda_env_python.txt} (100%) rename ci/{conda_env_r.yml => conda_env_r.txt} (100%) rename ci/{conda_env_sphinx.yml => conda_env_sphinx.txt} (100%) rename ci/{conda_env_unix.yml => conda_env_unix.txt} (100%) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index c6781a4b149..59b14dc3287 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -140,12 +140,6 @@ jobs: - name: Fetch Submodules and Tags shell: bash run: ci/scripts/util_checkout.sh - - name: Show available RAM size - shell: bash - run: | - hwmemsize=$(sysctl -n hw.memsize) - ramsize=$(expr $hwmemsize / $((1024**3))) - echo "System Memory: ${ramsize} GB" - name: Install Dependencies shell: bash run: | diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat index 261b1183f5b..4f790f05f00 100644 --- a/ci/appveyor-cpp-setup.bat +++ b/ci/appveyor-cpp-setup.bat @@ -50,16 +50,16 @@ set CONDA_PACKAGES= if "%ARROW_BUILD_GANDIVA%" == "ON" ( @rem Install llvmdev in the toolchain if building gandiva.dll - set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_gandiva_win.yml + set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_gandiva_win.txt ) if "%JOB%" == "Toolchain" ( @rem Install pre-built "toolchain" packages for faster builds - set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.yml + set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.txt ) if "%JOB%" NEQ "Build_Debug" ( @rem Arrow conda environment is only required for the Build and Toolchain jobs conda create -n arrow -q -y -c conda-forge ^ - --file=ci\conda_env_python.yml ^ + --file=ci\conda_env_python.txt ^ %CONDA_PACKAGES% ^ "cmake=3.17" ^ "ninja" ^ diff --git a/ci/conda_env_archery.yml b/ci/conda_env_archery.txt similarity index 100% rename from ci/conda_env_archery.yml rename to ci/conda_env_archery.txt diff --git a/ci/conda_env_cpp.yml b/ci/conda_env_cpp.txt similarity index 100% rename from ci/conda_env_cpp.yml rename to ci/conda_env_cpp.txt diff --git a/ci/conda_env_gandiva.yml b/ci/conda_env_gandiva.txt similarity index 100% rename from ci/conda_env_gandiva.yml rename to ci/conda_env_gandiva.txt diff --git a/ci/conda_env_gandiva_win.yml b/ci/conda_env_gandiva_win.txt similarity index 100% rename from ci/conda_env_gandiva_win.yml rename to ci/conda_env_gandiva_win.txt diff --git a/ci/conda_env_python.yml b/ci/conda_env_python.txt similarity index 100% rename from ci/conda_env_python.yml rename to ci/conda_env_python.txt diff --git a/ci/conda_env_r.yml b/ci/conda_env_r.txt similarity index 100% rename from ci/conda_env_r.yml rename to ci/conda_env_r.txt diff --git a/ci/conda_env_sphinx.yml b/ci/conda_env_sphinx.txt similarity index 100% rename from ci/conda_env_sphinx.yml rename to ci/conda_env_sphinx.txt diff --git a/ci/conda_env_unix.yml b/ci/conda_env_unix.txt similarity index 100% rename from ci/conda_env_unix.yml rename to ci/conda_env_unix.txt diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile index 660ad821655..ff31930c06c 100644 --- a/ci/docker/conda-cpp.dockerfile +++ b/ci/docker/conda-cpp.dockerfile @@ -20,12 +20,12 @@ ARG arch FROM ${repo}:${arch}-conda # install the required conda packages into the test environment -COPY ci/conda_env_cpp.yml \ - ci/conda_env_gandiva.yml \ +COPY ci/conda_env_cpp.txt \ + ci/conda_env_gandiva.txt \ /arrow/ci/ RUN conda install \ - --file arrow/ci/conda_env_cpp.yml \ - --file arrow/ci/conda_env_gandiva.yml \ + --file arrow/ci/conda_env_cpp.txt \ + --file arrow/ci/conda_env_gandiva.txt \ compilers \ doxygen \ valgrind && \ diff --git a/ci/docker/conda-integration.dockerfile b/ci/docker/conda-integration.dockerfile index 1f2c9ac5da2..ff82814f6c6 100644 --- a/ci/docker/conda-integration.dockerfile +++ b/ci/docker/conda-integration.dockerfile @@ -26,10 +26,10 @@ ARG jdk=8 ARG go=1.15 # Install Archery and integration dependencies -COPY ci/conda_env_archery.yml /arrow/ci/ +COPY ci/conda_env_archery.txt /arrow/ci/ RUN conda install -q \ - --file arrow/ci/conda_env_cpp.yml \ - --file arrow/ci/conda_env_archery.yml \ + --file arrow/ci/conda_env_cpp.txt \ + --file arrow/ci/conda_env_archery.txt \ numpy \ compilers \ maven=${maven} \ diff --git a/ci/docker/conda-python.dockerfile b/ci/docker/conda-python.dockerfile index a7e76974825..ab3f77be1b6 100644 --- a/ci/docker/conda-python.dockerfile +++ b/ci/docker/conda-python.dockerfile @@ -21,9 +21,9 @@ FROM ${repo}:${arch}-conda-cpp # install python specific packages ARG python=3.6 -COPY ci/conda_env_python.yml /arrow/ci/ +COPY ci/conda_env_python.txt /arrow/ci/ RUN conda install -q \ - --file arrow/ci/conda_env_python.yml \ + --file arrow/ci/conda_env_python.txt \ $([ "$python" == "3.6" -o "$python" == "3.7" ] && echo "pickle5") \ python=${python} \ nomkl && \ diff --git a/ci/docker/conda.dockerfile b/ci/docker/conda.dockerfile index 3ea393d3fa9..2e773b5437e 100644 --- a/ci/docker/conda.dockerfile +++ b/ci/docker/conda.dockerfile @@ -38,8 +38,8 @@ RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix} RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix} # create a conda environment -ADD ci/conda_env_unix.yml /arrow/ci/ -RUN conda create -n arrow --file arrow/ci/conda_env_unix.yml git && \ +ADD ci/conda_env_unix.txt /arrow/ci/ +RUN conda create -n arrow --file arrow/ci/conda_env_unix.txt git && \ conda clean --all # activate the created environment by default From 8f1913ff8ef799345d4e06c4a159d7ab4b1e70d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 29 Jun 2021 12:53:23 +0200 Subject: [PATCH 2/3] Fix conda isntegration dockerfile --- ci/docker/conda-integration.dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/docker/conda-integration.dockerfile b/ci/docker/conda-integration.dockerfile index ff82814f6c6..8a7dd48b947 100644 --- a/ci/docker/conda-integration.dockerfile +++ b/ci/docker/conda-integration.dockerfile @@ -28,7 +28,6 @@ ARG go=1.15 # Install Archery and integration dependencies COPY ci/conda_env_archery.txt /arrow/ci/ RUN conda install -q \ - --file arrow/ci/conda_env_cpp.txt \ --file arrow/ci/conda_env_archery.txt \ numpy \ compilers \ From 0bbd7480f6cd0f3cdba2a4223f73263c7ca1b73d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 29 Jun 2021 14:34:59 +0200 Subject: [PATCH 3/3] Pin aws sdk --- ci/conda_env_cpp.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt index 1a8ae6d7edc..def40deb98a 100644 --- a/ci/conda_env_cpp.txt +++ b/ci/conda_env_cpp.txt @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +# workaround for https://issues.apache.org/jira/browse/ARROW-13134 aws-sdk-cpp<1.9 benchmark=1.5.2 boost-cpp>=1.68.0