From d592a1c975dfb9bef84cde41c86273303b95464e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Dec 2024 12:02:54 +0100 Subject: [PATCH 01/43] GH-44950: [C++] Bump minimum CMake version to 3.25 Try installing CMake 3.25 manually on Ubuntu Install cmake in linux-apt-lint.dockerfile and update install_cmake.sh to not require platform argument Install CMake 3.25.0 on several Ubuntu jobs and Windows jobs that require the existing version to be bumped Several fixes to install_cmake Some more fixes and upgrade minimal examples to use Ubuntu 24.04 Some more fixes for R Update from CMAKE to CMAKE_VERSION Ensure that versions returned are strings, bump cmake download Try forcing MINGW package to pull a newer CMake Revert trying to pull newer CMake Try to completely remove mingw CMake from CRAN /c/cmake/bin/cmake.exe should be present and in path, try with .exe Try with full path for CMake Try removing manual installation of CMake and installing CMake via pacman before setting up RTools Remove manual Installation of CMake from pacman not present More tries Temporarily try using msys2 GH action to install cmake Remove CMake from PKGBUILD CMake should be in the path Debug new CMake location Try with C:\Program Files\CMake\bin\cmake.exe Just testing at this point Bundle AWSSDK under RTools (#90) Check for mingw Check Windows compiler Try modifying patch due to redefinition Try fixing patch AWS is bundled on Windows R now, do not use MSYS libraries Fix license header on patch + add CMake condition for AWS patch Update configure.win to remove specific AWS CPP libs Update r/configure.win Co-authored-by: Bryce Mecum Update r/configure.win Test turning ccache back on Update ci/scripts/r_windows_build.sh Co-authored-by: Neal Richardson Update r/configure.win Co-authored-by: Neal Richardson Review comments Remove MSYS2_ARG_CONV_EXCL Testing with manually adding missing rtools40 Add correct slash for path Add comment on why MSYS2_ARG_CONV_EXCL is required Use MSYS2_ARG_CONV_EXCL on the same CMake command Update tutorial examples to use a newer Ubuntu Add CMAKE_VERSION to cuda jobs Upgrade minimal Ubuntu to use 24.04 instead of 22.04 Fix minimal build for Ubuntu 24.04 ubuntu-cuda-cpp requires cmake argument Add cmake to docker-compose.yml missing images Add missing cmake to some more images Some more fixing mainly for R binary jobs Set CMAKE to newly installed path Revert "Try modifying patch due to redefinition" This reverts commit b772b23878fb29a8752e82a6214b7c29a3b80352. Reapply "Try modifying patch due to redefinition" This reverts commit 1d713753c9e296a58fa101817332d81b250855da. Hardcode enum value I tried to find a proper way to include-guard this but everything I tried didn't work. Remove outdated refs to aws libs Since we are bundling these we don't have these to copy Move _WIN32_WINNT def into {c,cxx}flags Ignore warnings-as-errors in cmake --- .env | 1 + .github/workflows/cpp.yml | 10 + .github/workflows/dev.yml | 5 + .github/workflows/r.yml | 6 + ci/docker/centos-7-cpp.dockerfile | 7 +- ci/docker/linux-r.dockerfile | 4 + ci/docker/python-wheel-manylinux.dockerfile | 2 +- ci/docker/ubuntu-20.04-cpp-minimal.dockerfile | 5 +- ci/docker/ubuntu-20.04-cpp.dockerfile | 5 +- ci/docker/ubuntu-22.04-cpp-minimal.dockerfile | 4 + ci/docker/ubuntu-22.04-cpp.dockerfile | 5 +- ci/rtools/BuildAwsCCommon.patch | 54 +++++ ci/rtools/aws_c_io_ep.patch | 88 ++++++++ ci/rtools/awssdk_ep.patch | 213 ++++++++++++++++++ ci/scripts/PKGBUILD | 12 +- ci/scripts/install_cmake.sh | 46 ++-- ci/scripts/r_windows_build.sh | 13 +- cpp/CMakeLists.txt | 2 +- cpp/cmake_modules/ThirdpartyToolchain.cmake | 31 +++ cpp/examples/minimal_build/CMakeLists.txt | 2 +- cpp/examples/minimal_build/minimal.dockerfile | 2 +- .../system_dependency.dockerfile | 2 +- .../parquet/parquet_arrow/CMakeLists.txt | 2 +- cpp/examples/tutorial_examples/CMakeLists.txt | 2 +- .../tutorial_examples/tutorial.dockerfile | 2 +- dev/tasks/python-wheels/github.osx.yml | 2 +- dev/tasks/tasks.yml | 4 +- dev/tasks/vcpkg-tests/github.windows.yml | 2 +- docker-compose.yml | 12 + docs/source/cpp/build_system.rst | 2 +- matlab/CMakeLists.txt | 2 +- python/CMakeLists.txt | 2 +- .../examples/minimal_build/Dockerfile.ubuntu | 5 +- python/examples/minimal_build/README.md | 2 +- r/configure.win | 7 +- r/tools/nixlibs.R | 10 +- 36 files changed, 516 insertions(+), 59 deletions(-) create mode 100644 ci/rtools/BuildAwsCCommon.patch create mode 100644 ci/rtools/aws_c_io_ep.patch create mode 100644 ci/rtools/awssdk_ep.patch diff --git a/.env b/.env index a94803cc327..d42a79f6264 100644 --- a/.env +++ b/.env @@ -54,6 +54,7 @@ UBUNTU=22.04 # Default versions for various dependencies CLANG_TOOLS=14 +CMAKE_VERSION=3.25.0 CUDA=11.2.2 DASK=latest DOTNET=8.0 diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index b0adbbfa0a4..dcfdce36eb7 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -165,6 +165,11 @@ jobs: with: fetch-depth: 0 submodules: recursive + - name: Install CMake 3.25.0 + shell: bash + run: | + ci/scripts/install_cmake.sh 3.25.0 /tmp/local/ + echo "/tmp/local/bin" >> $GITHUB_PATH - name: Check CMake presets run: | cd cpp @@ -324,6 +329,11 @@ jobs: with: fetch-depth: 0 submodules: recursive + - name: Install CMake 3.25.0 + shell: bash + run: | + ci/scripts/install_cmake.sh 3.25.0 /c/cmake + echo "c:\\cmake\\bin" >> $GITHUB_PATH - name: Download Timezone Database shell: bash run: ci/scripts/download_tz_database.sh diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 6a8cc05ca68..c81f33629b6 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -53,6 +53,11 @@ jobs: uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: 3.12 + - name: Install CMake 3.25.0 + shell: bash + run: | + ci/scripts/install_cmake.sh 3.25.0 /tmp/local/ + echo "/tmp/local/bin" >> $GITHUB_PATH - name: Install pre-commit run: | python -m pip install pre-commit diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 407cc775c39..2507c4b1133 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -282,6 +282,12 @@ jobs: restore-keys: | r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}- + - name: Install CMake via MSYS2 + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: true + install: mingw-w64-ucrt-x86_64-cmake - uses: r-lib/actions/setup-r@v2 with: # Note: RTools must be 40 here because RTools40 + ucrt is how we build the Arrow C++ diff --git a/ci/docker/centos-7-cpp.dockerfile b/ci/docker/centos-7-cpp.dockerfile index 1f30eed694e..b012a5abed2 100644 --- a/ci/docker/centos-7-cpp.dockerfile +++ b/ci/docker/centos-7-cpp.dockerfile @@ -37,7 +37,6 @@ RUN \ -e 's/mirror\.centos\.org/vault.centos.org/' \ /etc/yum.repos.d/CentOS-SCLo-scl*.repo && \ yum install -y \ - cmake3 \ curl \ devtoolset-8 \ diffutils \ @@ -49,9 +48,13 @@ RUN \ wget \ which +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/ RUN bash /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin ENV \ ARROW_R_DEV=TRUE \ - CMAKE=/usr/bin/cmake3 + CMAKE=/usr/local/bin/cmake diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile index 7b7e989adc0..da378eac430 100644 --- a/ci/docker/linux-r.dockerfile +++ b/ci/docker/linux-r.dockerfile @@ -51,6 +51,10 @@ RUN /arrow/ci/scripts/r_docker_configure.sh COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ COPY r/DESCRIPTION /arrow/r/ RUN /arrow/ci/scripts/r_deps.sh /arrow diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile index 0b5645285b6..ffcaa8c0a07 100644 --- a/ci/docker/python-wheel-manylinux.dockerfile +++ b/ci/docker/python-wheel-manylinux.dockerfile @@ -53,7 +53,7 @@ ENV PATH=/opt/python/${CPYTHON_VERSION}-${CPYTHON_VERSION}/bin:${PATH} # Install CMake ARG cmake=3.29.2 COPY ci/scripts/install_cmake.sh arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_cmake.sh ${arch} linux ${cmake} /usr/local +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local # Install Ninja ARG ninja=1.10.2 diff --git a/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile index 1b342df596c..79a0c786aaf 100644 --- a/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile @@ -27,7 +27,6 @@ RUN apt-get update -y -q && \ apt-get install -y -q \ build-essential \ ccache \ - cmake \ curl \ gdb \ git \ @@ -68,6 +67,10 @@ RUN latest_system_llvm=10 && \ apt-get clean && \ rm -rf /var/lib/apt/lists* +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index 259c5fb77fa..db7e03b8bfb 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -67,7 +67,6 @@ RUN apt-get update -y -q && \ autoconf \ ca-certificates \ ccache \ - cmake \ curl \ g++ \ gcc \ @@ -121,6 +120,10 @@ RUN apt-get update -y -q && \ apt-get clean && \ rm -rf /var/lib/apt/lists* +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local diff --git a/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile index ce31c457e90..2a90a5637d4 100644 --- a/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile @@ -68,6 +68,10 @@ RUN latest_system_llvm=14 && \ apt-get clean && \ rm -rf /var/lib/apt/lists* +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile index 721b37dcae8..4a3d311acbf 100644 --- a/ci/docker/ubuntu-22.04-cpp.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp.dockerfile @@ -71,7 +71,6 @@ RUN apt-get update -y -q && \ ceph \ ceph-fuse \ ceph-mds \ - cmake \ curl \ gdb \ git \ @@ -168,6 +167,10 @@ RUN if [ "${gcc}" = "" ]; then \ # make sure zlib is cached in the EMSDK folder RUN source ~/emsdk/emsdk_env.sh && embuilder --pic build zlib +ARG cmake +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local diff --git a/ci/rtools/BuildAwsCCommon.patch b/ci/rtools/BuildAwsCCommon.patch new file mode 100644 index 00000000000..f129b6f1234 --- /dev/null +++ b/ci/rtools/BuildAwsCCommon.patch @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +diff --git a/cmake/AwsCFlags.cmake b/cmake/AwsCFlags.cmake +index b717bca..5aa8ac9 100644 +--- a/cmake/AwsCFlags.cmake ++++ b/cmake/AwsCFlags.cmake +@@ -120,6 +120,10 @@ function(aws_set_common_properties target) + list(APPEND AWS_C_FLAGS -Wno-strict-aliasing) + endif() + ++ if(CMAKE_C_IMPLICIT_LINK_LIBRARIES MATCHES "mingw32") ++ list(APPEND AWS_C_FLAGS -D__USE_MINGW_ANSI_STDIO=1 -Wno-unused-local-typedefs) ++ endif() ++ + # -moutline-atomics generates code for both older load/store exclusive atomics and also + # Arm's Large System Extensions (LSE) which scale substantially better on large core count systems. + # +diff --git a/include/aws/common/byte_order.inl b/include/aws/common/byte_order.inl +index 1204be0..0abd9cb 100644 +--- a/include/aws/common/byte_order.inl ++++ b/include/aws/common/byte_order.inl +@@ -13,7 +13,7 @@ + # include + #else + # include +-#endif /* _MSC_VER */ ++#endif /* _WIN32 */ + + AWS_EXTERN_C_BEGIN + +@@ -39,7 +39,7 @@ AWS_STATIC_IMPL uint64_t aws_hton64(uint64_t x) { + uint64_t v; + __asm__("bswap %q0" : "=r"(v) : "0"(x)); + return v; +-#elif defined(_MSC_VER) ++#elif defined(_WIN32) + return _byteswap_uint64(x); + #else + uint32_t low = x & UINT32_MAX; diff --git a/ci/rtools/aws_c_io_ep.patch b/ci/rtools/aws_c_io_ep.patch new file mode 100644 index 00000000000..960b7d0d5e4 --- /dev/null +++ b/ci/rtools/aws_c_io_ep.patch @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +diff --git a/source/windows/host_resolver.c b/source/windows/host_resolver.c +index 59fbb85..ad4a99e 100644 +--- a/source/windows/host_resolver.c ++++ b/source/windows/host_resolver.c +@@ -4,6 +4,7 @@ + */ + + /* don't move this below the Windows.h include!!!!*/ ++#define _WIN32_WINNT 0x0601 + #include + #include + +diff --git a/source/windows/iocp/iocp_event_loop.c b/source/windows/iocp/iocp_event_loop.c +index 9ccce30..5cbbef7 100644 +--- a/source/windows/iocp/iocp_event_loop.c ++++ b/source/windows/iocp/iocp_event_loop.c +@@ -12,6 +12,7 @@ + + #include + ++#define _WIN32_WINNT 0x0601 + #include + + /* The next set of struct definitions are taken directly from the +diff --git a/source/windows/secure_channel_tls_handler.c b/source/windows/secure_channel_tls_handler.c +index 50caf02..29fe850 100644 +--- a/source/windows/secure_channel_tls_handler.c ++++ b/source/windows/secure_channel_tls_handler.c +@@ -2,6 +2,7 @@ + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ ++#define _WIN32_WINNT 0x0601 + #define SECURITY_WIN32 + + #include +@@ -35,6 +36,25 @@ + # pragma warning(disable : 4306) /* Identifier is type cast to a larger pointer. */ + #endif + ++#ifndef SP_PROT_TLS1_0_SERVER ++#define SP_PROT_TLS1_0_SERVER SP_PROT_TLS1_SERVER ++#endif ++#ifndef SP_PROT_TLS1_0_CLIENT ++#define SP_PROT_TLS1_0_CLIENT SP_PROT_TLS1_CLIENT ++#endif ++#ifndef SP_PROT_TLS1_1_SERVER ++#define SP_PROT_TLS1_1_SERVER 0x00000100 ++#endif ++#ifndef SP_PROT_TLS1_1_CLIENT ++#define SP_PROT_TLS1_1_CLIENT 0x00000200 ++#endif ++#ifndef SCH_USE_STRONG_CRYPTO ++#define SCH_USE_STRONG_CRYPTO 0x00400000 ++#endif ++#ifndef SECBUFFER_ALERT ++#define SECBUFFER_ALERT 0x11 ++#endif ++ + #define KB_1 1024 + #define READ_OUT_SIZE (16 * KB_1) + #define READ_IN_SIZE READ_OUT_SIZE +@@ -456,7 +476,7 @@ static int s_fillin_alpn_data( + + *extension_length += sizeof(uint32_t) + sizeof(uint16_t); + +- *extension_name = SecApplicationProtocolNegotiationExt_ALPN; ++ *extension_name = 2; + /*now add the protocols*/ + for (size_t i = 0; i < protocols_count; ++i) { + struct aws_byte_cursor *protocol_ptr = NULL; diff --git a/ci/rtools/awssdk_ep.patch b/ci/rtools/awssdk_ep.patch new file mode 100644 index 00000000000..5920ace5a49 --- /dev/null +++ b/ci/rtools/awssdk_ep.patch @@ -0,0 +1,213 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +diff --git a/aws-cpp-sdk-core/include/aws/core/utils/Array.h b/aws-cpp-sdk-core/include/aws/core/utils/Array.h +index 2b5bbc566..7cb93bdf0 100644 +--- a/aws-cpp-sdk-core/include/aws/core/utils/Array.h ++++ b/aws-cpp-sdk-core/include/aws/core/utils/Array.h +@@ -54,7 +54,7 @@ namespace Aws + { + m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); + +-#ifdef _WIN32 ++#ifdef _MSC_VER + std::copy(arrayToCopy, arrayToCopy + arraySize, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); + #else + std::copy(arrayToCopy, arrayToCopy + arraySize, m_data.get()); +@@ -82,7 +82,7 @@ namespace Aws + if(arr->m_size > 0 && arr->m_data) + { + size_t arraySize = arr->m_size; +-#ifdef _WIN32 ++#ifdef _MSC_VER + std::copy(arr->m_data.get(), arr->m_data.get() + arraySize, stdext::checked_array_iterator< T * >(m_data.get() + location, m_size)); + #else + std::copy(arr->m_data.get(), arr->m_data.get() + arraySize, m_data.get() + location); +@@ -101,7 +101,7 @@ namespace Aws + { + m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); + +-#ifdef _WIN32 ++#ifdef _MSC_VER + std::copy(other.m_data.get(), other.m_data.get() + other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); + #else + std::copy(other.m_data.get(), other.m_data.get() + other.m_size, m_data.get()); +@@ -134,7 +134,7 @@ namespace Aws + { + m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); + +-#ifdef _WIN32 ++#ifdef _MSC_VER + std::copy(other.m_data.get(), other.m_data.get() + other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); + #else + std::copy(other.m_data.get(), other.m_data.get() + other.m_size, m_data.get()); +diff --git a/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h b/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h +index e26e36b60..3e7189b70 100644 +--- a/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h ++++ b/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h +@@ -29,7 +29,14 @@ namespace Aws + { + namespace Crypto + { +- static const char* SecureRandom_BCrypt_Tag = "SecureRandom_BCrypt"; ++#ifdef __MINGW32__ ++#pragma GCC diagnostic push ++#pragma GCC diagnostic ignored "-Wunused-variable" ++#endif ++ static const char* SecureRandom_BCrypt_Tag = "SecureRandom_BCrypt"; ++#ifdef __MINGW32__ ++#pragma GCC diagnostic pop ++#endif + + class SecureRandomBytes_BCrypt : public SecureRandomBytes + { +diff --git a/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp b/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp +index 4dade6489..a0456cf8e 100644 +--- a/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp ++++ b/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp +@@ -22,6 +22,16 @@ + #include + #include + ++#ifndef WINHTTP_OPTION_WEB_SOCKET_KEEPALIVE_INTERVAL ++#define WINHTTP_OPTION_WEB_SOCKET_KEEPALIVE_INTERVAL 116 ++#endif ++#ifndef WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 ++#define WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 0x00000200 ++#endif ++#ifndef WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2 ++#define WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2 0x00000800 ++#endif ++ + using namespace Aws::Client; + using namespace Aws::Http; + using namespace Aws::Http::Standard; +@@ -272,7 +282,7 @@ bool WinHttpSyncHttpClient::DoQueryHeaders(void* hHttpRequest, std::shared_ptr(dwSize / sizeof(wchar_t))); + + WinHttpQueryHeaders(hHttpRequest, WINHTTP_QUERY_CONTENT_TYPE, nullptr, &contentTypeStr, &dwSize, 0); +- if (contentTypeStr[0] != NULL) ++ if (contentTypeStr[0]) + { + Aws::String contentStr = StringUtils::FromWString(contentTypeStr); + response->SetContentType(contentStr); +@@ -303,7 +313,7 @@ bool WinHttpSyncHttpClient::DoQueryHeaders(void* hHttpRequest, std::shared_ptrSetContentType(contentTypeStr); + AWS_LOGSTREAM_DEBUG(GetLogTag(), "Received content type " << contentTypeStr); +diff --git a/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp b/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp +index d7513cc3c..e390a8d4e 100644 +--- a/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp ++++ b/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp +@@ -349,7 +349,7 @@ std::shared_ptr WinSyncHttpClient::MakeRequest(const std::shared_p + } + } + +- if (!success && !IsRequestProcessingEnabled() || !ContinueRequest(*request)) ++ if (!success && (!IsRequestProcessingEnabled() || !ContinueRequest(*request))) + { + response->SetClientErrorType(CoreErrors::USER_CANCELLED); + response->SetClientErrorMessage("Request processing disabled or continuation cancelled by user's continuation handler."); +diff --git a/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp b/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp +index f6e36077e..13e349aef 100644 +--- a/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp ++++ b/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp +@@ -3,6 +3,7 @@ + * SPDX-License-Identifier: Apache-2.0. + */ + ++#define _WIN32_WINNT 0x0601 + #include + #include + #include +diff --git a/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp b/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp +index 2ea82de6f..bc423441e 100644 +--- a/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp ++++ b/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp +@@ -11,7 +11,9 @@ + #include + #include + ++#ifdef _MSC_VER + #pragma warning( disable : 4996) ++#endif + + using namespace Aws::Utils; + namespace Aws +@@ -304,6 +306,9 @@ Aws::String CreateTempFilePath() + { + #ifdef _MSC_VER + #pragma warning(disable: 4996) // _CRT_SECURE_NO_WARNINGS ++#elif !defined(L_tmpnam_s) ++ // Definition from the MSVC stdio.h ++ #define L_tmpnam_s (sizeof("\\") + 16) + #endif + char s_tempName[L_tmpnam_s+1]; + +diff --git a/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp b/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp +index 0180f7fbf..3adbab313 100644 +--- a/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp ++++ b/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp +@@ -9,7 +9,9 @@ + + #include + ++#ifdef _MSC_VER + #pragma warning(disable: 4996) ++#endif + #include + #include + namespace Aws +diff --git a/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp b/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp +index 2ee517b48..3b0dce665 100644 +--- a/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp ++++ b/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp +@@ -939,7 +939,7 @@ std::shared_ptr Aws::Utils::Crypto::CreateSha256HMACIm + return GetSha256HMACFactory()->CreateImplementation(); + } + +-#ifdef _WIN32 ++#ifdef _MSC_VER + #pragma warning( push ) + #pragma warning( disable : 4702 ) + #endif +@@ -1032,7 +1032,7 @@ std::shared_ptr Aws::Utils::Crypto::CreateAES_KeyWrapImplementa + return GetAES_KeyWrapFactory()->CreateImplementation(key); + } + +-#ifdef _WIN32 ++#ifdef _MSC_VER + #pragma warning(pop) + #endif diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index efeed954006..56a463f6422 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -24,8 +24,7 @@ pkgdesc="Apache Arrow is a cross-language development platform for in-memory dat arch=("any") url="https://arrow.apache.org/" license=("Apache-2.0") -depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" - "${MINGW_PACKAGE_PREFIX}-bzip2" +depends=("${MINGW_PACKAGE_PREFIX}-bzip2" "${MINGW_PACKAGE_PREFIX}-curl" # for google-cloud-cpp bundled build "${MINGW_PACKAGE_PREFIX}-libutf8proc" "${MINGW_PACKAGE_PREFIX}-re2" @@ -36,7 +35,6 @@ depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" "${MINGW_PACKAGE_PREFIX}-zstd" "${MINGW_PACKAGE_PREFIX}-brotli") makedepends=("${MINGW_PACKAGE_PREFIX}-ccache" - "${MINGW_PACKAGE_PREFIX}-cmake" "${MINGW_PACKAGE_PREFIX}-gcc") options=("staticlibs" "strip" "!buildflags") @@ -82,8 +80,13 @@ build() { # CMAKE_UNITY_BUILD is set to OFF as otherwise some compute functionality # segfaults in tests + # MSYS2_ARG_CONV_EXCL is needed to prevent autoconverting CMAKE_INSTALL_PREFIX + # to Windows paths. See https://www.msys2.org/docs/filesystem-paths/#process-arguments + + # We install CMake via the msys2 GitHub action. We have to use the installed + # CMake to build Arrow which is in the Program Files directory. MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ - ${MINGW_PREFIX}/bin/cmake.exe \ + "${PROGRAMFILES}\CMake\bin\cmake.exe" \ ${ARROW_CPP_DIR} \ -G "MSYS Makefiles" \ -DARROW_ACERO=ON \ @@ -116,6 +119,7 @@ build() { -DARROW_WITH_BZ2=ON \ -DARROW_ZSTD_USE_SHARED=OFF \ -DARROW_CXXFLAGS="${CPPFLAGS}" \ + -DAWSSDK_SOURCE=BUNDLED \ -DCMAKE_BUILD_TYPE="release" \ -DCMAKE_INSTALL_PREFIX=${MINGW_PREFIX} \ -DCMAKE_UNITY_BUILD=OFF \ diff --git a/ci/scripts/install_cmake.sh b/ci/scripts/install_cmake.sh index 7fdb06d90f0..e283288f14a 100755 --- a/ci/scripts/install_cmake.sh +++ b/ci/scripts/install_cmake.sh @@ -17,29 +17,49 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi declare -A archs -archs=([amd64]=x86_64 - [arch64]=aarch64 +archs=([x86_64]=x86_64 [arm64]=aarch64 - [arm64v8]=aarch64 - [x86_64]=x86_64) + [aarch64]=aarch64) + +arch=$(uname -m) +if [ -z ${archs[$arch]} ]; then + echo "Unsupported architecture: ${arch}" + exit 0 +fi +arch=${archs[$arch]} declare -A platforms platforms=([linux]=linux [macos]=macos [windows]=windows) -if [ "$#" -ne 4 ]; then - echo "Usage: $0 " - exit 1 -fi +version=$1 +prefix=$2 -arch=${archs[$1]} -platform=${platforms[$2]} -version=$3 -prefix=$4 +platform=$(uname) +case ${platform} in + Linux) + platform=linux + ;; + Darwin) + platform=macos + ;; + MSYS_NT*|MINGW64_NT*) + platform=windows + ;; + *) + echo "Unsupported platform: ${platform}" + exit 0 + ;; +esac mkdir -p ${prefix} url="https://github.com/Kitware/CMake/releases/download/v${version}/cmake-${version}-${platform}-" diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index c9395eb243f..af29858bfe8 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -23,13 +23,8 @@ set -ex # Make sure it is absolute and exported export ARROW_HOME="$(cd "${ARROW_HOME}" && pwd)" -# Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN -# curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf -# curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" -# pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz -# pacman --noconfirm -Scc - pacman --noconfirm -Syy + RWINLIB_LIB_DIR="lib" : ${MINGW_ARCH:="mingw32 mingw64 ucrt64"} @@ -71,7 +66,7 @@ if [ -d mingw64/lib/ ]; then # Move the 64-bit versions of libarrow into the expected location mv mingw64/lib/*.a $DST_DIR/lib/x64 # These are from https://dl.bintray.com/rtools/mingw{32,64}/ - cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,aws*,nghttp2}.a $DST_DIR/lib/x64 + cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a $DST_DIR/lib/x64 fi # Same for the 32-bit versions @@ -79,7 +74,7 @@ if [ -d mingw32/lib/ ]; then ls $MSYS_LIB_DIR/mingw32/lib/ mkdir -p $DST_DIR/lib/i386 mv mingw32/lib/*.a $DST_DIR/lib/i386 - cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,aws*,nghttp2}.a $DST_DIR/lib/i386 + cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a $DST_DIR/lib/i386 fi # Do the same also for ucrt64 @@ -87,7 +82,7 @@ if [ -d ucrt64/lib/ ]; then ls $MSYS_LIB_DIR/ucrt64/lib/ mkdir -p $DST_DIR/lib/x64-ucrt mv ucrt64/lib/*.a $DST_DIR/lib/x64-ucrt - cp $MSYS_LIB_DIR/ucrt64/lib/lib{thrift,snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,aws*,nghttp2}.a $DST_DIR/lib/x64-ucrt + cp $MSYS_LIB_DIR/ucrt64/lib/lib{thrift,snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a $DST_DIR/lib/x64-ucrt fi # Create build artifact diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a7d80c2e96c..f2500b3a72f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.25) message(STATUS "Building using CMake version: ${CMAKE_VERSION}") # https://www.cmake.org/cmake/help/latest/policy/CMP0025.html diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 3f169900b16..1ca61e0eee9 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5050,6 +5050,12 @@ macro(build_awssdk) string(APPEND AWS_C_FLAGS " -Wno-deprecated") string(APPEND AWS_CXX_FLAGS " -Wno-deprecated") endif() + # GH-44950: This is required to build under Rtools40 and we may be able to + # remove it if/when we no longer need to build under Rtools40 + if(WIN32 AND NOT MSVC) + string(APPEND AWS_C_FLAGS " -D_WIN32_WINNT=0x0601 -Wno-error -Wno-error=format= -Wno-error=format-extra-args") + string(APPEND AWS_CXX_FLAGS " -D_WIN32_WINNT=0x0601 -Wno-error -Wno-error=format= -Wno-error=format-extra-args") + endif() set(AWSSDK_COMMON_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} @@ -5087,6 +5093,29 @@ macro(build_awssdk) endif() list(APPEND AWSSDK_PATCH_COMMAND ${AWSSDK_UNUSED_DIRECTORIES}) + # WIP GH-44950 + # Patch parts of the AWSSDK EP so it builds cleanly under Rtools40 + if(WIN32 AND NOT MSVC) + find_program(PATCH patch REQUIRED) + # Patch aws_c_common to build under Rtools40 + set(AWS_C_COMMON_PATCH_COMMAND ${PATCH} -p1 -i + ${CMAKE_SOURCE_DIR}/../ci/rtools/BuildAwsCCommon.patch) + message(STATUS "Hello ${AWS_C_COMMON_PATCH_COMMAND}") + # aws_c_io_ep to build under Rtools40 + set(AWS_C_IO_PATCH_COMMAND ${PATCH} -p1 -i + ${CMAKE_SOURCE_DIR}/../ci/rtools/aws_c_io_ep.patch) + message(STATUS "Hello ${AWS_C_IO_PATCH_COMMAND}") + # awssdk_ep to build under Rtools40 + list(APPEND + AWSSDK_PATCH_COMMAND + && + ${PATCH} + -p1 + -i + ${CMAKE_SOURCE_DIR}/../ci/rtools/awssdk_ep.patch) + message(STATUS "Hello ${AWSSDK_PATCH_COMMAND}") + endif() + if(UNIX) # on Linux and macOS curl seems to be required find_curl() @@ -5181,6 +5210,7 @@ macro(build_awssdk) ${EP_COMMON_OPTIONS} URL ${AWS_C_COMMON_SOURCE_URL} URL_HASH "SHA256=${ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM}" + PATCH_COMMAND ${AWS_C_COMMON_PATCH_COMMAND} CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_COMMON_STATIC_LIBRARY}) add_dependencies(AWS::aws-c-common aws_c_common_ep) @@ -5276,6 +5306,7 @@ macro(build_awssdk) ${EP_COMMON_OPTIONS} URL ${AWS_C_IO_SOURCE_URL} URL_HASH "SHA256=${ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM}" + PATCH_COMMAND ${AWS_C_IO_PATCH_COMMAND} CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} BUILD_BYPRODUCTS ${AWS_C_IO_STATIC_LIBRARY} DEPENDS ${AWS_C_IO_DEPENDS}) diff --git a/cpp/examples/minimal_build/CMakeLists.txt b/cpp/examples/minimal_build/CMakeLists.txt index 95dad34221a..689dba437e0 100644 --- a/cpp/examples/minimal_build/CMakeLists.txt +++ b/cpp/examples/minimal_build/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.25) project(ArrowMinimalExample) diff --git a/cpp/examples/minimal_build/minimal.dockerfile b/cpp/examples/minimal_build/minimal.dockerfile index 9361fc5e81d..8062e9b6984 100644 --- a/cpp/examples/minimal_build/minimal.dockerfile +++ b/cpp/examples/minimal_build/minimal.dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:focal +FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive diff --git a/cpp/examples/minimal_build/system_dependency.dockerfile b/cpp/examples/minimal_build/system_dependency.dockerfile index 926fcaf6f4b..84a16c4902f 100644 --- a/cpp/examples/minimal_build/system_dependency.dockerfile +++ b/cpp/examples/minimal_build/system_dependency.dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:focal +FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive diff --git a/cpp/examples/parquet/parquet_arrow/CMakeLists.txt b/cpp/examples/parquet/parquet_arrow/CMakeLists.txt index 0480391e380..189d17914d6 100644 --- a/cpp/examples/parquet/parquet_arrow/CMakeLists.txt +++ b/cpp/examples/parquet/parquet_arrow/CMakeLists.txt @@ -16,7 +16,7 @@ # under the License. # Require cmake that supports BYPRODUCTS in add_custom_command, ExternalProject_Add [1]. -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.25) project(parquet_arrow_example) diff --git a/cpp/examples/tutorial_examples/CMakeLists.txt b/cpp/examples/tutorial_examples/CMakeLists.txt index 8788501484c..a6f8350c41d 100644 --- a/cpp/examples/tutorial_examples/CMakeLists.txt +++ b/cpp/examples/tutorial_examples/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.25) project(ArrowTutorialExamples) diff --git a/cpp/examples/tutorial_examples/tutorial.dockerfile b/cpp/examples/tutorial_examples/tutorial.dockerfile index 9361fc5e81d..8062e9b6984 100644 --- a/cpp/examples/tutorial_examples/tutorial.dockerfile +++ b/cpp/examples/tutorial_examples/tutorial.dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:focal +FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml index 031bad94227..b353d7f06f2 100644 --- a/dev/tasks/python-wheels/github.osx.yml +++ b/dev/tasks/python-wheels/github.osx.yml @@ -63,7 +63,7 @@ jobs: - name: Install CMake 3.29.0 shell: bash run: | - arrow/ci/scripts/install_cmake.sh $(arch) macos 3.29.0 ${PWD}/local + arrow/ci/scripts/install_cmake.sh 3.29.0 ${PWD}/local echo "${PWD}/local/bin" >> $GITHUB_PATH - name: Retrieve VCPKG version from arrow/.env diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 6c57d918eb2..ba0cdbabff1 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1344,8 +1344,8 @@ tasks: template: docker-tests/github.cuda.yml params: env: - UBUNTU: 22.04 CUDA: 11.7.1 + UBUNTU: 22.04 image: ubuntu-cuda-cpp test-cuda-python-ubuntu-22.04-cuda-11.7.1: @@ -1353,8 +1353,8 @@ tasks: template: docker-tests/github.cuda.yml params: env: - UBUNTU: 22.04 CUDA: 11.7.1 + UBUNTU: 22.04 image: ubuntu-cuda-python ############################## Fuzz tests ################################# diff --git a/dev/tasks/vcpkg-tests/github.windows.yml b/dev/tasks/vcpkg-tests/github.windows.yml index af12db59528..0fd4892c0a6 100644 --- a/dev/tasks/vcpkg-tests/github.windows.yml +++ b/dev/tasks/vcpkg-tests/github.windows.yml @@ -32,7 +32,7 @@ jobs: - name: Install CMake 3.29.0 shell: bash run: | - arrow/ci/scripts/install_cmake.sh amd64 windows 3.29.0 /c/cmake + arrow/ci/scripts/install_cmake.sh 3.29.0 /c/cmake echo "c:\\cmake\\bin" >> $GITHUB_PATH - name: Download Timezone Database shell: bash diff --git a/docker-compose.yml b/docker-compose.yml index 468872dc964..495a0b7ad70 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -390,6 +390,7 @@ services: arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" clang_tools: ${CLANG_TOOLS} + cmake: ${CMAKE_VERSION} gcc: ${GCC} llvm: ${LLVM} shm_size: *shm-size @@ -426,6 +427,7 @@ services: arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" clang_tools: ${CLANG_TOOLS} + cmake: ${CMAKE_VERSION} gcc: ${GCC} llvm: ${LLVM} shm_size: *shm-size @@ -455,6 +457,8 @@ services: dockerfile: ci/docker/centos-7-cpp.dockerfile cache_from: - ${REPO}:centos-7-cpp-static + args: + cmake: ${CMAKE_VERSION} shm_size: *shm-size volumes: - .:/arrow:delegated @@ -477,6 +481,7 @@ services: args: arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" + cmake: ${CMAKE_VERSION} llvm: ${LLVM} shm_size: *shm-size ulimits: *ulimits @@ -571,6 +576,7 @@ services: arch: ${ARCH} base: nvidia/cuda:${CUDA}-devel-ubuntu${UBUNTU} clang_tools: ${CLANG_TOOLS} + cmake: ${CMAKE_VERSION} llvm: ${LLVM} shm_size: *shm-size ulimits: *ulimits @@ -621,6 +627,7 @@ services: args: arch: ${ARCH} clang_tools: ${CLANG_TOOLS} + cmake: ${CMAKE_VERSION} llvm: ${LLVM} shm_size: *shm-size volumes: *ubuntu-volumes @@ -700,6 +707,7 @@ services: args: arch: ${ARCH} clang_tools: ${CLANG_TOOLS} + cmake: ${CMAKE_VERSION} llvm: ${LLVM} shm_size: *shm-size volumes: *ubuntu-volumes @@ -1587,6 +1595,7 @@ services: tz: ${TZ} r_prune_deps: ${R_PRUNE_DEPS} r_custom_ccache: ${R_CUSTOM_CCACHE} + cmake: ${CMAKE_VERSION} shm_size: *shm-size environment: <<: [*common, *sccache] @@ -1616,6 +1625,7 @@ services: - ${REPO}:amd64-ubuntu-20.04-r-sanitizer args: base: wch1/r-debug:latest + cmake: ${CMAKE_VERSION} r_bin: RDsan tz: ${TZ} r_prune_deps: ${R_PRUNE_DEPS} @@ -1636,6 +1646,7 @@ services: - ${REPO}:r-rhub-clang-devel-latest args: base: rhub/clang-asan + cmake: ${CMAKE_VERSION} r_dev: ${ARROW_R_DEV} r_bin: R tz: ${TZ} @@ -1667,6 +1678,7 @@ services: - ${REPO}:amd64-ubuntu-20.04-r-valgrind args: base: wch1/r-debug:latest + cmake: ${CMAKE_VERSION} r_bin: RDvalgrind tz: ${TZ} environment: diff --git a/docs/source/cpp/build_system.rst b/docs/source/cpp/build_system.rst index e80bca4c949..5f89866a16b 100644 --- a/docs/source/cpp/build_system.rst +++ b/docs/source/cpp/build_system.rst @@ -46,7 +46,7 @@ file into an executable linked with the Arrow C++ shared library: .. code-block:: cmake - cmake_minimum_required(VERSION 3.16) + cmake_minimum_required(VERSION 3.25) project(MyExample) diff --git a/matlab/CMakeLists.txt b/matlab/CMakeLists.txt index cd6d8768d03..07de5a612b3 100644 --- a/matlab/CMakeLists.txt +++ b/matlab/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.25) # Build the Arrow C++ libraries using ExternalProject_Add. function(build_arrow) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 0b0275f200d..ac497e844e7 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -18,7 +18,7 @@ # Includes code assembled from BSD/MIT/Apache-licensed code from some 3rd-party # projects, including Kudu, Impala, and libdynd. See python/LICENSE.txt -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.25) project(pyarrow) # This is needed for 3.13 free-threading. CMake used to add Python diff --git a/python/examples/minimal_build/Dockerfile.ubuntu b/python/examples/minimal_build/Dockerfile.ubuntu index 07cd69c0824..5c6b1131684 100644 --- a/python/examples/minimal_build/Dockerfile.ubuntu +++ b/python/examples/minimal_build/Dockerfile.ubuntu @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:jammy +FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive @@ -33,7 +33,6 @@ RUN apt-get update -y -q && \ python3-pip \ python3-venv \ tzdata \ + tzdata-legacy \ && \ apt-get clean && rm -rf /var/lib/apt/lists* - -RUN pip3 install --no-cache-dir -U pip setuptools diff --git a/python/examples/minimal_build/README.md b/python/examples/minimal_build/README.md index e9d595b6031..2ed7a653e7b 100644 --- a/python/examples/minimal_build/README.md +++ b/python/examples/minimal_build/README.md @@ -51,7 +51,7 @@ docker compose run --rm minimal-fedora-conda docker compose run --rm minimal-fedora-venv ``` -## Ubuntu 20.04 +## Ubuntu 24.04 First, build the Docker image using: ``` diff --git a/r/configure.win b/r/configure.win index b6ac19faea2..a8f8422478b 100755 --- a/r/configure.win +++ b/r/configure.win @@ -61,10 +61,9 @@ function configure_binaries() { OPENSSL_LIBS="-lcrypto -lcrypt32" MIMALLOC_LIBS="-lbcrypt -lpsapi" BROTLI_LIBS="-lbrotlienc -lbrotlidec -lbrotlicommon" # Common goes last since dec and enc depend on it - AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-identity-management \ - -laws-cpp-sdk-cognito-identity -laws-cpp-sdk-sts -laws-cpp-sdk-s3 \ - -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common \ - -luserenv -lversion -lws2_32 -lbcrypt -lwininet -lwinhttp" + # We build aws-sdk-cpp bundled now, so the AWS libs are included in arrow_bundled_dependencies + # but we also need to include these Windows system libraries + AWS_LIBS="-luserenv -lversion -lws2_32 -lbcrypt -lwininet -lwinhttp -lsecur32 -lshlwapi -lncrypt" # pkg-config --libs libcurl GCS_LIBS="-lcurl -lnormaliz -lssh2 -lgdi32 -lssl -lcrypto -lcrypt32 -lwldap32 \ -lz -lws2_32 -lnghttp2 -ldbghelp" diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 6c68133cd89..d3b9d2ce12e 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -635,12 +635,12 @@ build_libarrow <- function(src_dir, dst_dir) { invisible(status) } -ensure_cmake <- function(cmake_minimum_required = "3.16") { +ensure_cmake <- function(cmake_minimum_required = "3.25") { cmake <- find_cmake(version_required = cmake_minimum_required) if (is.null(cmake)) { # If not found, download it - CMAKE_VERSION <- Sys.getenv("CMAKE_VERSION", "3.26.4") + CMAKE_VERSION <- Sys.getenv("CMAKE_VERSION", "3.31.2") if (on_macos) { postfix <- "-macos-universal.tar.gz" } else if (tolower(Sys.info()[["machine"]]) %in% c("arm64", "aarch64")) { @@ -698,7 +698,7 @@ find_cmake <- function(paths = c( if (on_macos) "/Applications/CMake.app/Contents/bin/cmake", Sys.which("cmake3") ), - version_required = "3.16") { + version_required) { # Given a list of possible cmake paths, return the first one that exists and is new enough # version_required should be a string or packageVersion; numeric version # can be misleading (e.g. 3.10 is actually 3.1) @@ -715,7 +715,7 @@ find_cmake <- function(paths = c( } else { # Keep trying lg("Not using cmake found at %s", path, .indent = "****") - if (found_version > 0) { + if (found_version > "0") { lg("Version >= %s required; found %s", version_required, found_version, .indent = "*****") } else { # If cmake_version() couldn't determine version, it returns 0 @@ -737,7 +737,7 @@ cmake_version <- function(cmd = "cmake") { package_version(sub(pat, "\\1", raw_version[which_line])) }, error = function(e) { - return(0) + return("0") } ) } From 5bc6ed50a9c943620e67e7add03a61779b38bf0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 21 Jan 2025 18:15:31 +0100 Subject: [PATCH 02/43] Fix some review comments Rever CMAKE_VERSION back to CMAKE on .env Fix typo on file name --- .env | 2 +- ...AwsCCommon.patch => aws_c_common_ep.patch} | 2 +- ci/scripts/install_cmake.sh | 5 ----- cpp/cmake_modules/ThirdpartyToolchain.cmake | 12 +++++++--- docker-compose.yml | 22 +++++++++---------- 5 files changed, 22 insertions(+), 21 deletions(-) rename ci/rtools/{BuildAwsCCommon.patch => aws_c_common_ep.patch} (96%) diff --git a/.env b/.env index d42a79f6264..ce10b3e199d 100644 --- a/.env +++ b/.env @@ -54,7 +54,7 @@ UBUNTU=22.04 # Default versions for various dependencies CLANG_TOOLS=14 -CMAKE_VERSION=3.25.0 +CMAKE=3.25.0 CUDA=11.2.2 DASK=latest DOTNET=8.0 diff --git a/ci/rtools/BuildAwsCCommon.patch b/ci/rtools/aws_c_common_ep.patch similarity index 96% rename from ci/rtools/BuildAwsCCommon.patch rename to ci/rtools/aws_c_common_ep.patch index f129b6f1234..f1357070429 100644 --- a/ci/rtools/BuildAwsCCommon.patch +++ b/ci/rtools/aws_c_common_ep.patch @@ -23,7 +23,7 @@ index b717bca..5aa8ac9 100644 list(APPEND AWS_C_FLAGS -Wno-strict-aliasing) endif() -+ if(CMAKE_C_IMPLICIT_LINK_LIBRARIES MATCHES "mingw32") ++ if (CMAKE_C_IMPLICIT_LINK_LIBRARIES MATCHES "mingw32") + list(APPEND AWS_C_FLAGS -D__USE_MINGW_ANSI_STDIO=1 -Wno-unused-local-typedefs) + endif() + diff --git a/ci/scripts/install_cmake.sh b/ci/scripts/install_cmake.sh index e283288f14a..d01a7a744dc 100755 --- a/ci/scripts/install_cmake.sh +++ b/ci/scripts/install_cmake.sh @@ -36,11 +36,6 @@ if [ -z ${archs[$arch]} ]; then fi arch=${archs[$arch]} -declare -A platforms -platforms=([linux]=linux - [macos]=macos - [windows]=windows) - version=$1 prefix=$2 diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 1ca61e0eee9..d039a83c0b9 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5053,8 +5053,14 @@ macro(build_awssdk) # GH-44950: This is required to build under Rtools40 and we may be able to # remove it if/when we no longer need to build under Rtools40 if(WIN32 AND NOT MSVC) - string(APPEND AWS_C_FLAGS " -D_WIN32_WINNT=0x0601 -Wno-error -Wno-error=format= -Wno-error=format-extra-args") - string(APPEND AWS_CXX_FLAGS " -D_WIN32_WINNT=0x0601 -Wno-error -Wno-error=format= -Wno-error=format-extra-args") + string(APPEND + AWS_C_FLAGS + " -D_WIN32_WINNT=0x0601 -Wno-error -Wno-error=format= -Wno-error=format-extra-args" + ) + string(APPEND + AWS_CXX_FLAGS + " -D_WIN32_WINNT=0x0601 -Wno-error -Wno-error=format= -Wno-error=format-extra-args" + ) endif() set(AWSSDK_COMMON_CMAKE_ARGS @@ -5099,7 +5105,7 @@ macro(build_awssdk) find_program(PATCH patch REQUIRED) # Patch aws_c_common to build under Rtools40 set(AWS_C_COMMON_PATCH_COMMAND ${PATCH} -p1 -i - ${CMAKE_SOURCE_DIR}/../ci/rtools/BuildAwsCCommon.patch) + ${CMAKE_SOURCE_DIR}/../ci/rtools/aws_c_common_ep.patch) message(STATUS "Hello ${AWS_C_COMMON_PATCH_COMMAND}") # aws_c_io_ep to build under Rtools40 set(AWS_C_IO_PATCH_COMMAND ${PATCH} -p1 -i diff --git a/docker-compose.yml b/docker-compose.yml index 495a0b7ad70..dbba6b104f2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -390,7 +390,7 @@ services: arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" clang_tools: ${CLANG_TOOLS} - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} gcc: ${GCC} llvm: ${LLVM} shm_size: *shm-size @@ -427,7 +427,7 @@ services: arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" clang_tools: ${CLANG_TOOLS} - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} gcc: ${GCC} llvm: ${LLVM} shm_size: *shm-size @@ -458,7 +458,7 @@ services: cache_from: - ${REPO}:centos-7-cpp-static args: - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} shm_size: *shm-size volumes: - .:/arrow:delegated @@ -481,7 +481,7 @@ services: args: arch: ${ARCH} base: "${ARCH}/ubuntu:${UBUNTU}" - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} llvm: ${LLVM} shm_size: *shm-size ulimits: *ulimits @@ -576,7 +576,7 @@ services: arch: ${ARCH} base: nvidia/cuda:${CUDA}-devel-ubuntu${UBUNTU} clang_tools: ${CLANG_TOOLS} - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} llvm: ${LLVM} shm_size: *shm-size ulimits: *ulimits @@ -627,7 +627,7 @@ services: args: arch: ${ARCH} clang_tools: ${CLANG_TOOLS} - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} llvm: ${LLVM} shm_size: *shm-size volumes: *ubuntu-volumes @@ -707,7 +707,7 @@ services: args: arch: ${ARCH} clang_tools: ${CLANG_TOOLS} - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} llvm: ${LLVM} shm_size: *shm-size volumes: *ubuntu-volumes @@ -1595,7 +1595,7 @@ services: tz: ${TZ} r_prune_deps: ${R_PRUNE_DEPS} r_custom_ccache: ${R_CUSTOM_CCACHE} - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} shm_size: *shm-size environment: <<: [*common, *sccache] @@ -1625,7 +1625,7 @@ services: - ${REPO}:amd64-ubuntu-20.04-r-sanitizer args: base: wch1/r-debug:latest - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} r_bin: RDsan tz: ${TZ} r_prune_deps: ${R_PRUNE_DEPS} @@ -1646,7 +1646,7 @@ services: - ${REPO}:r-rhub-clang-devel-latest args: base: rhub/clang-asan - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} r_dev: ${ARROW_R_DEV} r_bin: R tz: ${TZ} @@ -1678,7 +1678,7 @@ services: - ${REPO}:amd64-ubuntu-20.04-r-valgrind args: base: wch1/r-debug:latest - cmake: ${CMAKE_VERSION} + cmake: ${CMAKE} r_bin: RDvalgrind tz: ${TZ} environment: From e3291cd50950e51cb5b1b58e3285e1ad2bc8b981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 11:06:28 +0100 Subject: [PATCH 03/43] Update some workflows to use Ubuntu 24.04 in order to remove manual CMake installation --- .github/workflows/cpp.yml | 12 +----------- .github/workflows/dev.yml | 7 +------ 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index dcfdce36eb7..1a7ca18d7fb 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -156,7 +156,7 @@ jobs: build-example: name: C++ Minimal Build Example - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 45 steps: @@ -165,11 +165,6 @@ jobs: with: fetch-depth: 0 submodules: recursive - - name: Install CMake 3.25.0 - shell: bash - run: | - ci/scripts/install_cmake.sh 3.25.0 /tmp/local/ - echo "/tmp/local/bin" >> $GITHUB_PATH - name: Check CMake presets run: | cd cpp @@ -329,11 +324,6 @@ jobs: with: fetch-depth: 0 submodules: recursive - - name: Install CMake 3.25.0 - shell: bash - run: | - ci/scripts/install_cmake.sh 3.25.0 /c/cmake - echo "c:\\cmake\\bin" >> $GITHUB_PATH - name: Download Timezone Database shell: bash run: ci/scripts/download_tz_database.sh diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index c81f33629b6..a62e06a7e29 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -41,7 +41,7 @@ jobs: lint: name: Lint C++, Python, R, Docker, RAT - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 15 steps: @@ -53,11 +53,6 @@ jobs: uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: 3.12 - - name: Install CMake 3.25.0 - shell: bash - run: | - ci/scripts/install_cmake.sh 3.25.0 /tmp/local/ - echo "/tmp/local/bin" >> $GITHUB_PATH - name: Install pre-commit run: | python -m pip install pre-commit From 42380352d972c7aad79da0d5449b1e4cb98eb442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 11:13:34 +0100 Subject: [PATCH 04/43] Try to use newer CMake from base Windows-2019 image on Windows R build instead of installing via MSYS2 --- .github/workflows/r.yml | 6 ------ ci/scripts/PKGBUILD | 4 +--- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 2507c4b1133..407cc775c39 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -282,12 +282,6 @@ jobs: restore-keys: | r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}- - - name: Install CMake via MSYS2 - uses: msys2/setup-msys2@v2 - with: - msystem: UCRT64 - update: true - install: mingw-w64-ucrt-x86_64-cmake - uses: r-lib/actions/setup-r@v2 with: # Note: RTools must be 40 here because RTools40 + ucrt is how we build the Arrow C++ diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 56a463f6422..9ecdc2ad2e9 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -83,10 +83,8 @@ build() { # MSYS2_ARG_CONV_EXCL is needed to prevent autoconverting CMAKE_INSTALL_PREFIX # to Windows paths. See https://www.msys2.org/docs/filesystem-paths/#process-arguments - # We install CMake via the msys2 GitHub action. We have to use the installed - # CMake to build Arrow which is in the Program Files directory. MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ - "${PROGRAMFILES}\CMake\bin\cmake.exe" \ + cmake \ ${ARROW_CPP_DIR} \ -G "MSYS Makefiles" \ -DARROW_ACERO=ON \ From 09e06aa5c2944c3153af6c8cc66868522b4cdbf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 11:26:53 +0100 Subject: [PATCH 05/43] Locate CMake --- .github/workflows/r.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 407cc775c39..bbad7ab5d17 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -290,6 +290,8 @@ jobs: r-version: "4.1" rtools-version: 40 Ncpus: 2 + - name: Locate CMAKE + run: where cmake - name: Build Arrow C++ shell: bash env: From 87d2cf5c2f7ebc65655aa25a6042c64d310d2296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 11:27:07 +0100 Subject: [PATCH 06/43] Try installing cmake on Linux packages --- .../linux-packages/apache-arrow/yum/centos-7/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile index b10c40937d3..3b467a87486 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile @@ -50,7 +50,6 @@ RUN \ brotli-devel \ bzip2-devel \ ccache \ - cmake3 \ curl-devel \ flex \ gflags-devel \ @@ -75,6 +74,10 @@ RUN \ zlib-devel && \ yum clean ${quiet} all +ARG cmake=3.25.0 +COPY ../../../../../ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ + ENV \ BOOST_INCLUDEDIR=/usr/include/boost169 \ BOOST_LIBRARYDIR=/usr/lib64/boost169 From 75ac2412c919bd1d03490358c16eee9a0486841d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 11:32:49 +0100 Subject: [PATCH 07/43] Locate CMake v2 --- .github/workflows/r.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index bbad7ab5d17..8bc339a9981 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -291,7 +291,10 @@ jobs: rtools-version: 40 Ncpus: 2 - name: Locate CMAKE - run: where cmake + shell: bash + run: | + cmake --version + which cmake - name: Build Arrow C++ shell: bash env: From 3ae63ab05aa14bf6bec79ed05fc67d18e5c46531 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 11:44:24 +0100 Subject: [PATCH 08/43] Use full path to CMake on Program Files --- ci/scripts/PKGBUILD | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 9ecdc2ad2e9..c09efe3ee12 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -83,8 +83,10 @@ build() { # MSYS2_ARG_CONV_EXCL is needed to prevent autoconverting CMAKE_INSTALL_PREFIX # to Windows paths. See https://www.msys2.org/docs/filesystem-paths/#process-arguments + # We require the full path to the CMake executable in order to build Arrow + # which is in the Program Files directory: "/c/Program Files/CMake/bin/cmake" MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ - cmake \ + "${PROGRAMFILES}\CMake\bin\cmake.exe" \ ${ARROW_CPP_DIR} \ -G "MSYS Makefiles" \ -DARROW_ACERO=ON \ From 257945c2226ce2b5f85bb02c9f67fc4cb38d66dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 11:45:11 +0100 Subject: [PATCH 09/43] Try centos7 path (this is just testing at this point) --- dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile index 3b467a87486..3e687519efd 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile @@ -75,7 +75,7 @@ RUN \ yum clean ${quiet} all ARG cmake=3.25.0 -COPY ../../../../../ci/scripts/install_cmake.sh /arrow/ci/scripts/ +COPY ../../../../../../../ci/scripts/install_cmake.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ ENV \ From 1e2208b7c375fc3ae126d02777e4182f7fb89011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jan 2025 12:28:35 +0100 Subject: [PATCH 10/43] Try using link for context on Dockerfile --- dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile | 4 ++-- dev/tasks/linux-packages/scripts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 120000 dev/tasks/linux-packages/scripts diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile index 3e687519efd..db43f56a489 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile @@ -75,8 +75,8 @@ RUN \ yum clean ${quiet} all ARG cmake=3.25.0 -COPY ../../../../../../../ci/scripts/install_cmake.sh /arrow/ci/scripts/ -RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ +COPY scripts/install_cmake.sh /ci/scripts/ +RUN /ci/scripts/install_cmake.sh ${cmake} /usr/local/ ENV \ BOOST_INCLUDEDIR=/usr/include/boost169 \ diff --git a/dev/tasks/linux-packages/scripts b/dev/tasks/linux-packages/scripts new file mode 120000 index 00000000000..1762358f1e0 --- /dev/null +++ b/dev/tasks/linux-packages/scripts @@ -0,0 +1 @@ +../../../ci/scripts \ No newline at end of file From c977f359b49d347f6bf67f62db73839e5b911272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 24 Jan 2025 16:04:36 +0100 Subject: [PATCH 11/43] Try adding install_cmake.sh link to centos7 folder --- dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile | 4 ++-- .../linux-packages/apache-arrow/yum/centos-7/install_cmake.sh | 1 + dev/tasks/linux-packages/scripts | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) create mode 120000 dev/tasks/linux-packages/apache-arrow/yum/centos-7/install_cmake.sh delete mode 120000 dev/tasks/linux-packages/scripts diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile index db43f56a489..af6271805a9 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile @@ -75,8 +75,8 @@ RUN \ yum clean ${quiet} all ARG cmake=3.25.0 -COPY scripts/install_cmake.sh /ci/scripts/ -RUN /ci/scripts/install_cmake.sh ${cmake} /usr/local/ +COPY install_cmake.sh /tmp +RUN /tmp/install_cmake.sh ${cmake} /usr/local/ ENV \ BOOST_INCLUDEDIR=/usr/include/boost169 \ diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/install_cmake.sh b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/install_cmake.sh new file mode 120000 index 00000000000..72c41c44cb7 --- /dev/null +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/install_cmake.sh @@ -0,0 +1 @@ +../../../../../../ci/scripts/install_cmake.sh \ No newline at end of file diff --git a/dev/tasks/linux-packages/scripts b/dev/tasks/linux-packages/scripts deleted file mode 120000 index 1762358f1e0..00000000000 --- a/dev/tasks/linux-packages/scripts +++ /dev/null @@ -1 +0,0 @@ -../../../ci/scripts \ No newline at end of file From f71fc8178a82a0755c29d5030eaa3f2425e6acac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 24 Jan 2025 16:20:21 +0100 Subject: [PATCH 12/43] Use script --- .../linux-packages/apache-arrow/yum/centos-7/Dockerfile | 5 +++-- .../apache-arrow/yum/centos-7/install_cmake.sh | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) delete mode 120000 dev/tasks/linux-packages/apache-arrow/yum/centos-7/install_cmake.sh diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile index af6271805a9..19923001569 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile @@ -75,8 +75,9 @@ RUN \ yum clean ${quiet} all ARG cmake=3.25.0 -COPY install_cmake.sh /tmp -RUN /tmp/install_cmake.sh ${cmake} /usr/local/ +RUN curl -L \ + "https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-linux-$(uname -m).tar.gz" | \ + tar -xzf - --directory /usr/local --strip-components=1 ENV \ BOOST_INCLUDEDIR=/usr/include/boost169 \ diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/install_cmake.sh b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/install_cmake.sh deleted file mode 120000 index 72c41c44cb7..00000000000 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/install_cmake.sh +++ /dev/null @@ -1 +0,0 @@ -../../../../../../ci/scripts/install_cmake.sh \ No newline at end of file From e610341d35a23d6cfff5c7756a9bc242d0966a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 24 Jan 2025 16:36:27 +0100 Subject: [PATCH 13/43] Try using cmake instead of cmake3 --- dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 3766073bef5..c29d5213ca8 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -52,7 +52,7 @@ %define arrow_cmake_install %cmake_install %else %define arrow_cmake_builddir build -%define arrow_cmake %cmake3 -S . -B %{arrow_cmake_builddir} +%define arrow_cmake %cmake -S . -B %{arrow_cmake_builddir} %define arrow_cmake_build make -C %{arrow_cmake_builddir} %{?_smp_mflags} %define arrow_cmake_install DESTDIR="%{buildroot}" make -C %{arrow_cmake_builddir} install %endif From 033f8ed43a8a77a080d06d04fa0e3635c5a79f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 24 Jan 2025 16:48:06 +0100 Subject: [PATCH 14/43] Remove cmake3 requirement for rhel 7 and remove BuildRequires --- dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index c29d5213ca8..733b88b277f 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -35,10 +35,6 @@ if [ %{_rhel} -eq 7 ]; then \ echo 169; \ fi) -%define cmake_version %( \ - if [ %{_rhel} -eq 7 ]; then \ - echo 3; \ - fi) %define lz4_requirement %( \ if [ %{_amzn} -eq 0 ]; then \ @@ -104,7 +100,6 @@ BuildRequires: bzip2-devel %if %{use_flight} BuildRequires: c-ares-devel %endif -BuildRequires: cmake%{cmake_version} BuildRequires: curl-devel BuildRequires: flex BuildRequires: %{gcc_package}-c++ From 406a4e9b67c9eec92524ed68bdac872931fb0dc2 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 25 Jan 2025 07:24:34 +0900 Subject: [PATCH 15/43] Ensure using downloaded CMake on CentOS-7 --- dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in | 8 ++++++-- .../linux-packages/apache-arrow/yum/centos-7/Dockerfile | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 733b88b277f..d289eda4aa9 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -35,6 +35,10 @@ if [ %{_rhel} -eq 7 ]; then \ echo 169; \ fi) +%define cmake_version %( \ + if [ %{_rhel} -eq 7 ]; then \ + echo 3; \ + fi) %define lz4_requirement %( \ if [ %{_amzn} -eq 0 ]; then \ @@ -48,7 +52,7 @@ %define arrow_cmake_install %cmake_install %else %define arrow_cmake_builddir build -%define arrow_cmake %cmake -S . -B %{arrow_cmake_builddir} +%define arrow_cmake %cmake3 -S . -B %{arrow_cmake_builddir} %define arrow_cmake_build make -C %{arrow_cmake_builddir} %{?_smp_mflags} %define arrow_cmake_install DESTDIR="%{buildroot}" make -C %{arrow_cmake_builddir} install %endif @@ -100,6 +104,7 @@ BuildRequires: bzip2-devel %if %{use_flight} BuildRequires: c-ares-devel %endif +BuildRequires: cmake%{cmake_version} BuildRequires: curl-devel BuildRequires: flex BuildRequires: %{gcc_package}-c++ @@ -158,7 +163,6 @@ Apache Arrow is a data processing library for analysis. %build cpp_build_type=release -cd cpp %arrow_cmake \ %if %{use_azure} -DARROW_AZURE=ON \ diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile index 19923001569..56e2f52ff3c 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile @@ -50,6 +50,7 @@ RUN \ brotli-devel \ bzip2-devel \ ccache \ + cmake3 \ curl-devel \ flex \ gflags-devel \ @@ -77,7 +78,8 @@ RUN \ ARG cmake=3.25.0 RUN curl -L \ "https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-linux-$(uname -m).tar.gz" | \ - tar -xzf - --directory /usr/local --strip-components=1 + tar -xzf - --directory /usr/local --strip-components=1 && \ + ln -fs /usr/local/bin/cmake /usr/bin/cmake3 ENV \ BOOST_INCLUDEDIR=/usr/include/boost169 \ From 6a91db297dcc88aaf1d0b122c3f7bb3dfc696e54 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 25 Jan 2025 07:51:15 +0900 Subject: [PATCH 16/43] Revert needless change --- dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index d289eda4aa9..3766073bef5 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -163,6 +163,7 @@ Apache Arrow is a data processing library for analysis. %build cpp_build_type=release +cd cpp %arrow_cmake \ %if %{use_azure} -DARROW_AZURE=ON \ From 3a85d65f18e53d177040180e22c6ae72a0f32c34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 28 Jan 2025 16:34:24 +0100 Subject: [PATCH 17/43] Revert Required CMake on minimal build example and install cmake manually to amazon-linux-2023, ubuntu-focal and ubuntu-jammy --- cpp/examples/minimal_build/CMakeLists.txt | 2 +- .../linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile | 6 ++++++ .../apache-arrow/yum/amazon-linux-2023/Dockerfile | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cpp/examples/minimal_build/CMakeLists.txt b/cpp/examples/minimal_build/CMakeLists.txt index 689dba437e0..95dad34221a 100644 --- a/cpp/examples/minimal_build/CMakeLists.txt +++ b/cpp/examples/minimal_build/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 3.25) +cmake_minimum_required(VERSION 3.16) project(ArrowMinimalExample) diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile index 1d9065d6b2e..c8bc067f56b 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile @@ -82,3 +82,9 @@ RUN \ fi && \ apt clean && \ rm -rf /var/lib/apt/lists/* + +ARG cmake=3.25.0 +RUN curl -L \ + "https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-linux-$(uname -m).tar.gz" | \ + tar -xzf - --directory /usr/local --strip-components=1 && \ + ln -fs /usr/local/bin/cmake /usr/bin/cmake diff --git a/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2023/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2023/Dockerfile index a19641f8f96..a5664dbbbe9 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2023/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2023/Dockerfile @@ -60,3 +60,9 @@ RUN \ which \ zlib-devel && \ dnf clean ${quiet} all + +ARG cmake=3.25.0 +RUN curl -L \ + "https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-linux-$(uname -m).tar.gz" | \ + tar -xzf - --directory /usr/local --strip-components=1 && \ + ln -fs /usr/local/bin/cmake /usr/bin/cmake From 87a8eea96ba3476bd196833f1e28dbb70680c794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 28 Jan 2025 17:02:42 +0100 Subject: [PATCH 18/43] Install required curl to download newer cmake --- .../linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile index c8bc067f56b..09c0436fcc8 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile @@ -34,6 +34,7 @@ RUN \ build-essential \ clang \ cmake \ + curl \ debhelper \ devscripts \ gi-docgen \ From 53a1d2afdb6523057a74c4e07d2e6d243e38dbe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 29 Jan 2025 10:39:44 +0100 Subject: [PATCH 19/43] Add CMake debug logs for fetchcontent_makeavailable(orc) --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index d039a83c0b9..43727348295 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4680,7 +4680,9 @@ function(build_orc) OFF CACHE BOOL "" FORCE) + set(CMAKE_FIND_DEBUG_MODE TRUE) fetchcontent_makeavailable(orc) + set(CMAKE_FIND_DEBUG_MODE FALSE) add_library(orc::orc INTERFACE IMPORTED) target_link_libraries(orc::orc INTERFACE orc) From 4ab92ce34fde988f2b5bf4e8c6c8433518fccd69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 29 Jan 2025 11:47:27 +0100 Subject: [PATCH 20/43] Try adding zlib_ep dependency to ORC if ZLIB_VENDORED --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 43727348295..754c2a030e6 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4679,7 +4679,9 @@ function(build_orc) set(STOP_BUILD_ON_WARNING OFF CACHE BOOL "" FORCE) - + if(ZLIB_VENDORED) + add_dependencies(orc zlib_ep) + endif() set(CMAKE_FIND_DEBUG_MODE TRUE) fetchcontent_makeavailable(orc) set(CMAKE_FIND_DEBUG_MODE FALSE) From 33e155692df1fec594632c46f4c14a43123f9618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 29 Jan 2025 13:10:49 +0100 Subject: [PATCH 21/43] Set old behaviour for policy CMP0126 --- cpp/CMakeLists.txt | 5 +++++ cpp/cmake_modules/ThirdpartyToolchain.cmake | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f2500b3a72f..3c348e699bb 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -18,6 +18,11 @@ cmake_minimum_required(VERSION 3.25) message(STATUS "Building using CMake version: ${CMAKE_VERSION}") +# https://cmake.org/cmake/help/latest/policy/CMP0126.html +# +# set(CACHE) does not remove a normal variable of the same name. +cmake_policy(SET CMP0126 OLD) + # https://www.cmake.org/cmake/help/latest/policy/CMP0025.html # # Compiler id for Apple Clang is now AppleClang. diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 754c2a030e6..59d227ad4bc 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4679,12 +4679,7 @@ function(build_orc) set(STOP_BUILD_ON_WARNING OFF CACHE BOOL "" FORCE) - if(ZLIB_VENDORED) - add_dependencies(orc zlib_ep) - endif() - set(CMAKE_FIND_DEBUG_MODE TRUE) fetchcontent_makeavailable(orc) - set(CMAKE_FIND_DEBUG_MODE FALSE) add_library(orc::orc INTERFACE IMPORTED) target_link_libraries(orc::orc INTERFACE orc) From 8598511d5739fa326138eabdcab47697291ffd47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 29 Jan 2025 15:45:04 +0100 Subject: [PATCH 22/43] Remove unnecessary debugging to locate CMake --- .github/workflows/r.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 8bc339a9981..407cc775c39 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -290,11 +290,6 @@ jobs: r-version: "4.1" rtools-version: 40 Ncpus: 2 - - name: Locate CMAKE - shell: bash - run: | - cmake --version - which cmake - name: Build Arrow C++ shell: bash env: From 9b2b3b937e71cf2eae116181c7f38b4bfbaad5dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 29 Jan 2025 15:46:32 +0100 Subject: [PATCH 23/43] Remove stray comment --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 59d227ad4bc..5283bbbe335 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5098,7 +5098,6 @@ macro(build_awssdk) endif() list(APPEND AWSSDK_PATCH_COMMAND ${AWSSDK_UNUSED_DIRECTORIES}) - # WIP GH-44950 # Patch parts of the AWSSDK EP so it builds cleanly under Rtools40 if(WIN32 AND NOT MSVC) find_program(PATCH patch REQUIRED) From d0e5cf9ca3fcc07248f09dc9bbd261783bdb51cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 29 Jan 2025 16:52:20 +0100 Subject: [PATCH 24/43] Require newer CMake for verification tasks on old Ubuntu --- ci/docker/ubuntu-22.04-verify-rc.dockerfile | 4 ++++ docker-compose.yml | 1 + 2 files changed, 5 insertions(+) diff --git a/ci/docker/ubuntu-22.04-verify-rc.dockerfile b/ci/docker/ubuntu-22.04-verify-rc.dockerfile index 8bc6f39b67a..537120f6560 100644 --- a/ci/docker/ubuntu-22.04-verify-rc.dockerfile +++ b/ci/docker/ubuntu-22.04-verify-rc.dockerfile @@ -24,3 +24,7 @@ RUN /setup-ubuntu.sh && \ rm /setup-ubuntu.sh && \ apt-get clean && \ rm -rf /var/lib/apt/lists* + +ARG cmake=3.25.0 +COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ diff --git a/docker-compose.yml b/docker-compose.yml index dbba6b104f2..ec4419a0faa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2047,6 +2047,7 @@ services: args: repo: ${REPO} arch: ${ARCH} + cmake: ${CMAKE} volumes: - .:/arrow:delegated - ${DOCKER_VOLUME_PREFIX}ubuntu-ccache:/ccache:delegated From 2f956eae1bc502746f3b271acc7f794d9e1f5a46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 30 Jan 2025 17:26:32 +0100 Subject: [PATCH 25/43] Apply some suggestions and improvements --- ci/docker/ubuntu-22.04-verify-rc.dockerfile | 2 +- ci/scripts/PKGBUILD | 5 +++++ .../linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile | 4 +--- python/examples/minimal_build/Dockerfile.ubuntu | 2 ++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ci/docker/ubuntu-22.04-verify-rc.dockerfile b/ci/docker/ubuntu-22.04-verify-rc.dockerfile index 537120f6560..b9f130d24ea 100644 --- a/ci/docker/ubuntu-22.04-verify-rc.dockerfile +++ b/ci/docker/ubuntu-22.04-verify-rc.dockerfile @@ -25,6 +25,6 @@ RUN /setup-ubuntu.sh && \ apt-get clean && \ rm -rf /var/lib/apt/lists* -ARG cmake=3.25.0 +ARG cmake COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/ diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index c09efe3ee12..a4b4162d137 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -80,6 +80,11 @@ build() { # CMAKE_UNITY_BUILD is set to OFF as otherwise some compute functionality # segfaults in tests + # We use the bundled AWS SDK instead of the MINGW one because the upstream + # one on rtools packages is unmaintained, uses an old version (1.7.365) + # and does not work with newer versions of CMake. See comments: + # https://github.com/apache/arrow/pull/44989/files#r1901428998 + # MSYS2_ARG_CONV_EXCL is needed to prevent autoconverting CMAKE_INSTALL_PREFIX # to Windows paths. See https://www.msys2.org/docs/filesystem-paths/#process-arguments diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile index 09c0436fcc8..ea082234d75 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile @@ -33,7 +33,6 @@ RUN \ apt install -y -V ${quiet} \ build-essential \ clang \ - cmake \ curl \ debhelper \ devscripts \ @@ -87,5 +86,4 @@ RUN \ ARG cmake=3.25.0 RUN curl -L \ "https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-linux-$(uname -m).tar.gz" | \ - tar -xzf - --directory /usr/local --strip-components=1 && \ - ln -fs /usr/local/bin/cmake /usr/bin/cmake + tar -xzf - --directory /usr/local --strip-components=1 diff --git a/python/examples/minimal_build/Dockerfile.ubuntu b/python/examples/minimal_build/Dockerfile.ubuntu index 5c6b1131684..4b4afa0ddbb 100644 --- a/python/examples/minimal_build/Dockerfile.ubuntu +++ b/python/examples/minimal_build/Dockerfile.ubuntu @@ -36,3 +36,5 @@ RUN apt-get update -y -q && \ tzdata-legacy \ && \ apt-get clean && rm -rf /var/lib/apt/lists* + +RUN pip3 install --no-cache-dir -U pip setuptools From 7020fb5dc3503a9862432ba1d6ef89c2d7ee8c86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 30 Jan 2025 17:27:54 +0100 Subject: [PATCH 26/43] Removing CMP0126 OLD policy behavior and force unset ZLIB_LIBRARY --- cpp/CMakeLists.txt | 5 ----- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3c348e699bb..f2500b3a72f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -18,11 +18,6 @@ cmake_minimum_required(VERSION 3.25) message(STATUS "Building using CMake version: ${CMAKE_VERSION}") -# https://cmake.org/cmake/help/latest/policy/CMP0126.html -# -# set(CACHE) does not remove a normal variable of the same name. -cmake_policy(SET CMP0126 OLD) - # https://www.cmake.org/cmake/help/latest/policy/CMP0025.html # # Compiler id for Apple Clang is now AppleClang. diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 5283bbbe335..e4c69616b7c 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4646,6 +4646,7 @@ function(build_orc) set(ZLIB_HOME ${ZLIB_ROOT} CACHE STRING "" FORCE) + set(unset ZLIB_LIBRARY) set(ZLIB_LIBRARY ZLIB::ZLIB CACHE STRING "" FORCE) From 3b9999599c8b280d9ca035259fb7ce2953d253cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 30 Jan 2025 17:54:51 +0100 Subject: [PATCH 27/43] Remove system wide pip upgrade --- python/examples/minimal_build/Dockerfile.ubuntu | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/examples/minimal_build/Dockerfile.ubuntu b/python/examples/minimal_build/Dockerfile.ubuntu index 4b4afa0ddbb..5c6b1131684 100644 --- a/python/examples/minimal_build/Dockerfile.ubuntu +++ b/python/examples/minimal_build/Dockerfile.ubuntu @@ -36,5 +36,3 @@ RUN apt-get update -y -q && \ tzdata-legacy \ && \ apt-get clean && rm -rf /var/lib/apt/lists* - -RUN pip3 install --no-cache-dir -U pip setuptools From 5e25f01938640af5f30b20369404c193d6c96228 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Thu, 30 Jan 2025 09:24:48 -0800 Subject: [PATCH 28/43] Add readme to ci/rtools dir --- ci/rtools/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 ci/rtools/README.md diff --git a/ci/rtools/README.md b/ci/rtools/README.md new file mode 100644 index 00000000000..41c33ed75a3 --- /dev/null +++ b/ci/rtools/README.md @@ -0,0 +1,16 @@ +# rtools40 patches for AWS SDK and related libs + +The patches in this directory are solely for the purpose of building Arrow C++ +under [Rtools40](https://cran.r-project.org/bin/windows/Rtools/rtools40.html) +and not used elsewhere. Once we've dropped support for Rtools40, we can consider +removing these patches. + +The larger reason these patches are needed is that Rtools provides their own +packages and their versions of the AWS libraries weren't compatible with CMake +3.25. Our solution was to bundle the AWS libs instead and these patches were +required to get them building under the Rtools40 environment. + +The patches were added while upgrading the minimum required CMake version to +3.25 in [GH-44950](https://github.com/apache/arrow/issues/44950). Please see the +associated PR, [GH-44989](https://github.com/apache/arrow/pull/44989), for more +context. From ca25b71e94f5ff477e963d119b010793394560fb Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Thu, 30 Jan 2025 10:20:29 -0800 Subject: [PATCH 29/43] Fix bug in patch --- ci/rtools/awssdk_ep.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/rtools/awssdk_ep.patch b/ci/rtools/awssdk_ep.patch index 5920ace5a49..f91cc447291 100644 --- a/ci/rtools/awssdk_ep.patch +++ b/ci/rtools/awssdk_ep.patch @@ -136,7 +136,7 @@ index d7513cc3c..e390a8d4e 100644 } - if (!success && !IsRequestProcessingEnabled() || !ContinueRequest(*request)) -+ if (!success && (!IsRequestProcessingEnabled() || !ContinueRequest(*request))) ++ if ((!success && !IsRequestProcessingEnabled()) || !ContinueRequest(*request)) { response->SetClientErrorType(CoreErrors::USER_CANCELLED); response->SetClientErrorMessage("Request processing disabled or continuation cancelled by user's continuation handler."); From d93a55ba5f8a37c214c56298ea0f26968f97921a Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Thu, 30 Jan 2025 10:49:21 -0800 Subject: [PATCH 30/43] Move patch hunks into ThirdpartyToolchain.cmake --- ci/rtools/aws_c_common_ep.patch | 15 --------------- ci/rtools/awssdk_ep.patch | 20 -------------------- cpp/cmake_modules/ThirdpartyToolchain.cmake | 4 ++-- 3 files changed, 2 insertions(+), 37 deletions(-) diff --git a/ci/rtools/aws_c_common_ep.patch b/ci/rtools/aws_c_common_ep.patch index f1357070429..94c84d0fe1b 100644 --- a/ci/rtools/aws_c_common_ep.patch +++ b/ci/rtools/aws_c_common_ep.patch @@ -15,21 +15,6 @@ # specific language governing permissions and limitations # under the License. -diff --git a/cmake/AwsCFlags.cmake b/cmake/AwsCFlags.cmake -index b717bca..5aa8ac9 100644 ---- a/cmake/AwsCFlags.cmake -+++ b/cmake/AwsCFlags.cmake -@@ -120,6 +120,10 @@ function(aws_set_common_properties target) - list(APPEND AWS_C_FLAGS -Wno-strict-aliasing) - endif() - -+ if (CMAKE_C_IMPLICIT_LINK_LIBRARIES MATCHES "mingw32") -+ list(APPEND AWS_C_FLAGS -D__USE_MINGW_ANSI_STDIO=1 -Wno-unused-local-typedefs) -+ endif() -+ - # -moutline-atomics generates code for both older load/store exclusive atomics and also - # Arm's Large System Extensions (LSE) which scale substantially better on large core count systems. - # diff --git a/include/aws/common/byte_order.inl b/include/aws/common/byte_order.inl index 1204be0..0abd9cb 100644 --- a/include/aws/common/byte_order.inl diff --git a/ci/rtools/awssdk_ep.patch b/ci/rtools/awssdk_ep.patch index f91cc447291..3663fa813d2 100644 --- a/ci/rtools/awssdk_ep.patch +++ b/ci/rtools/awssdk_ep.patch @@ -55,26 +55,6 @@ index 2b5bbc566..7cb93bdf0 100644 std::copy(other.m_data.get(), other.m_data.get() + other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); #else std::copy(other.m_data.get(), other.m_data.get() + other.m_size, m_data.get()); -diff --git a/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h b/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h -index e26e36b60..3e7189b70 100644 ---- a/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h -+++ b/aws-cpp-sdk-core/include/aws/core/utils/crypto/bcrypt/CryptoImpl.h -@@ -29,7 +29,14 @@ namespace Aws - { - namespace Crypto - { -- static const char* SecureRandom_BCrypt_Tag = "SecureRandom_BCrypt"; -+#ifdef __MINGW32__ -+#pragma GCC diagnostic push -+#pragma GCC diagnostic ignored "-Wunused-variable" -+#endif -+ static const char* SecureRandom_BCrypt_Tag = "SecureRandom_BCrypt"; -+#ifdef __MINGW32__ -+#pragma GCC diagnostic pop -+#endif - - class SecureRandomBytes_BCrypt : public SecureRandomBytes - { diff --git a/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp b/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp index 4dade6489..a0456cf8e 100644 --- a/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index e4c69616b7c..af38d65364e 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5055,11 +5055,11 @@ macro(build_awssdk) if(WIN32 AND NOT MSVC) string(APPEND AWS_C_FLAGS - " -D_WIN32_WINNT=0x0601 -Wno-error -Wno-error=format= -Wno-error=format-extra-args" + " -D_WIN32_WINNT=0x0601 -D__USE_MINGW_ANSI_STDIO=1 -Wno-error -Wno-error=format= -Wno-error=format-extra-args -Wno-unused-local-typedefs -Wno-unused-variable" ) string(APPEND AWS_CXX_FLAGS - " -D_WIN32_WINNT=0x0601 -Wno-error -Wno-error=format= -Wno-error=format-extra-args" + " -D_WIN32_WINNT=0x0601 -D__USE_MINGW_ANSI_STDIO=1 -Wno-error -Wno-error=format= -Wno-error=format-extra-args -Wno-unused-local-typedefs -Wno-unused-variable" ) endif() From abf5b84f11bc8b44f54e21008de5d528c6419590 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Thu, 30 Jan 2025 11:16:16 -0800 Subject: [PATCH 31/43] Remove unneeded #define --- ci/rtools/aws_c_io_ep.patch | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ci/rtools/aws_c_io_ep.patch b/ci/rtools/aws_c_io_ep.patch index 960b7d0d5e4..5e55cd4e022 100644 --- a/ci/rtools/aws_c_io_ep.patch +++ b/ci/rtools/aws_c_io_ep.patch @@ -43,14 +43,6 @@ diff --git a/source/windows/secure_channel_tls_handler.c b/source/windows/secure index 50caf02..29fe850 100644 --- a/source/windows/secure_channel_tls_handler.c +++ b/source/windows/secure_channel_tls_handler.c -@@ -2,6 +2,7 @@ - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ -+#define _WIN32_WINNT 0x0601 - #define SECURITY_WIN32 - - #include @@ -35,6 +36,25 @@ # pragma warning(disable : 4306) /* Identifier is type cast to a larger pointer. */ #endif From 4f20c6a08652e5b49ac49ead3eee91f9b199079a Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 31 Jan 2025 09:49:47 +0900 Subject: [PATCH 32/43] Fix wrong command --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index af38d65364e..04500096a17 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4646,7 +4646,7 @@ function(build_orc) set(ZLIB_HOME ${ZLIB_ROOT} CACHE STRING "" FORCE) - set(unset ZLIB_LIBRARY) + unset(ZLIB_LIBRARY) set(ZLIB_LIBRARY ZLIB::ZLIB CACHE STRING "" FORCE) From 74d5606c0fea3c2fcc9113e76f23769b8bee7f43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 5 Feb 2025 15:02:09 +0100 Subject: [PATCH 33/43] Add License header to new README --- ci/rtools/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ci/rtools/README.md b/ci/rtools/README.md index 41c33ed75a3..08b5ea7f513 100644 --- a/ci/rtools/README.md +++ b/ci/rtools/README.md @@ -1,3 +1,22 @@ + + # rtools40 patches for AWS SDK and related libs The patches in this directory are solely for the purpose of building Arrow C++ From 17f502c149d7b6b60ec6d14676a027e62205c411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 5 Feb 2025 15:02:59 +0100 Subject: [PATCH 34/43] Comment Build-Depends on cmake for debian packages until we drop Ubuntu 22.04 support --- dev/tasks/linux-packages/apache-arrow/debian/control.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control.in b/dev/tasks/linux-packages/apache-arrow/debian/control.in index 87d78d843a2..a4836025cfb 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/control.in +++ b/dev/tasks/linux-packages/apache-arrow/debian/control.in @@ -3,7 +3,8 @@ Section: devel Priority: optional Maintainer: Apache Arrow Developers Build-Depends: - cmake, +# TODO: Enable this after we drop support for Ubuntu 22.04. +# cmake, debhelper (>= 12), gi-docgen, git, From fcd35fae04c1c5b988a781de5bb19f55aa163fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 5 Feb 2025 15:30:14 +0100 Subject: [PATCH 35/43] Try using /usr/local/bin/cmake for focal and jammy --- dev/tasks/linux-packages/apache-arrow/debian/rules | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules index 9f044680159..2d802773b46 100755 --- a/dev/tasks/linux-packages/apache-arrow/debian/rules +++ b/dev/tasks/linux-packages/apache-arrow/debian/rules @@ -21,6 +21,10 @@ override_dh_auto_configure: else \ ARROW_AZURE=ON; \ fi; \ + if [ "$${code_name}" = "focal" ] || [ "$${code_name}" = "jammy" ]; then \ + CMAKE=/usr/local/bin/cmake \ + else continue; \ + fi; \ if dpkg -l nvidia-cuda-toolkit > /dev/null 2>&1; then \ ARROW_CUDA=ON; \ else \ From eacab2970c1861595d206d6ae9c39c42c6e0febb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 5 Feb 2025 16:09:59 +0100 Subject: [PATCH 36/43] New attempt to fix debian rules --- dev/tasks/linux-packages/apache-arrow/debian/rules | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules index 2d802773b46..43d86581d3c 100755 --- a/dev/tasks/linux-packages/apache-arrow/debian/rules +++ b/dev/tasks/linux-packages/apache-arrow/debian/rules @@ -23,7 +23,8 @@ override_dh_auto_configure: fi; \ if [ "$${code_name}" = "focal" ] || [ "$${code_name}" = "jammy" ]; then \ CMAKE=/usr/local/bin/cmake \ - else continue; \ + else \ + continue; \ fi; \ if dpkg -l nvidia-cuda-toolkit > /dev/null 2>&1; then \ ARROW_CUDA=ON; \ From 6e72a169df2483e4dc0407e15f5a2c0848a1a83a Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Wed, 5 Feb 2025 07:25:29 -0800 Subject: [PATCH 37/43] Remove remainder of _WIN32_WINNT defines --- ci/rtools/aws_c_io_ep.patch | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/ci/rtools/aws_c_io_ep.patch b/ci/rtools/aws_c_io_ep.patch index 5e55cd4e022..a15d706ba12 100644 --- a/ci/rtools/aws_c_io_ep.patch +++ b/ci/rtools/aws_c_io_ep.patch @@ -15,30 +15,6 @@ # specific language governing permissions and limitations # under the License. -diff --git a/source/windows/host_resolver.c b/source/windows/host_resolver.c -index 59fbb85..ad4a99e 100644 ---- a/source/windows/host_resolver.c -+++ b/source/windows/host_resolver.c -@@ -4,6 +4,7 @@ - */ - - /* don't move this below the Windows.h include!!!!*/ -+#define _WIN32_WINNT 0x0601 - #include - #include - -diff --git a/source/windows/iocp/iocp_event_loop.c b/source/windows/iocp/iocp_event_loop.c -index 9ccce30..5cbbef7 100644 ---- a/source/windows/iocp/iocp_event_loop.c -+++ b/source/windows/iocp/iocp_event_loop.c -@@ -12,6 +12,7 @@ - - #include - -+#define _WIN32_WINNT 0x0601 - #include - - /* The next set of struct definitions are taken directly from the diff --git a/source/windows/secure_channel_tls_handler.c b/source/windows/secure_channel_tls_handler.c index 50caf02..29fe850 100644 --- a/source/windows/secure_channel_tls_handler.c From 2e9856429916ef3a691b141e820710c4a4c94c5c Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Wed, 5 Feb 2025 07:27:02 -0800 Subject: [PATCH 38/43] Remove _WIN32_WINNT define in awssdk_ep.patch --- ci/rtools/awssdk_ep.patch | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/ci/rtools/awssdk_ep.patch b/ci/rtools/awssdk_ep.patch index 3663fa813d2..ad298331e62 100644 --- a/ci/rtools/awssdk_ep.patch +++ b/ci/rtools/awssdk_ep.patch @@ -120,18 +120,6 @@ index d7513cc3c..e390a8d4e 100644 { response->SetClientErrorType(CoreErrors::USER_CANCELLED); response->SetClientErrorMessage("Request processing disabled or continuation cancelled by user's continuation handler."); -diff --git a/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp b/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp -index f6e36077e..13e349aef 100644 ---- a/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp -+++ b/aws-cpp-sdk-core/source/net/windows/SimpleUDP.cpp -@@ -3,6 +3,7 @@ - * SPDX-License-Identifier: Apache-2.0. - */ - -+#define _WIN32_WINNT 0x0601 - #include - #include - #include diff --git a/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp b/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp index 2ea82de6f..bc423441e 100644 --- a/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp From 0395a6a7576dc224ed8194d219a8828851196e1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 5 Feb 2025 17:00:05 +0100 Subject: [PATCH 39/43] Create a link to make debhelper build system happy --- dev/tasks/linux-packages/apache-arrow/debian/rules | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules index 43d86581d3c..6ff8750653b 100755 --- a/dev/tasks/linux-packages/apache-arrow/debian/rules +++ b/dev/tasks/linux-packages/apache-arrow/debian/rules @@ -21,11 +21,9 @@ override_dh_auto_configure: else \ ARROW_AZURE=ON; \ fi; \ - if [ "$${code_name}" = "focal" ] || [ "$${code_name}" = "jammy" ]; then \ - CMAKE=/usr/local/bin/cmake \ - else \ - continue; \ - fi; \ + if [ "$${code_name}" = "focal" ] || [ "$${code_name}" = "jammy" ]; then \ + ln -sf /usr/local/bin/cmake /usr/bin/cmake; \ + fi; \ if dpkg -l nvidia-cuda-toolkit > /dev/null 2>&1; then \ ARROW_CUDA=ON; \ else \ @@ -62,7 +60,7 @@ override_dh_auto_configure: -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ - -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \ + -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \ -DCUDAToolkit_ROOT=/usr \ -DFETCHCONTENT_FULLY_DISCONNECTED=OFF \ -DPARQUET_BUILD_EXECUTABLES=ON \ From 147ecc0c5bcf92b865062e1e71afe5ad21098c42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 5 Feb 2025 17:05:16 +0100 Subject: [PATCH 40/43] Fix extremely weird alignment on debian rules --- dev/tasks/linux-packages/apache-arrow/debian/rules | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules index 6ff8750653b..5b66f7ca9fa 100755 --- a/dev/tasks/linux-packages/apache-arrow/debian/rules +++ b/dev/tasks/linux-packages/apache-arrow/debian/rules @@ -21,9 +21,9 @@ override_dh_auto_configure: else \ ARROW_AZURE=ON; \ fi; \ - if [ "$${code_name}" = "focal" ] || [ "$${code_name}" = "jammy" ]; then \ - ln -sf /usr/local/bin/cmake /usr/bin/cmake; \ - fi; \ + if [ "$${code_name}" = "focal" ] || [ "$${code_name}" = "jammy" ]; then \ + ln -sf /usr/local/bin/cmake /usr/bin/cmake; \ + fi; \ if dpkg -l nvidia-cuda-toolkit > /dev/null 2>&1; then \ ARROW_CUDA=ON; \ else \ @@ -60,7 +60,7 @@ override_dh_auto_configure: -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ - -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \ + -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \ -DCUDAToolkit_ROOT=/usr \ -DFETCHCONTENT_FULLY_DISCONNECTED=OFF \ -DPARQUET_BUILD_EXECUTABLES=ON \ From 6f19834964169684c94e013a606a0c4cb0205168 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Wed, 5 Feb 2025 09:13:42 -0800 Subject: [PATCH 41/43] Update awssdk_ep.patch. See conversation https://github.com/apache/arrow/pull/44989/files#r1934292489. --- ci/rtools/awssdk_ep.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/rtools/awssdk_ep.patch b/ci/rtools/awssdk_ep.patch index ad298331e62..bd26f853290 100644 --- a/ci/rtools/awssdk_ep.patch +++ b/ci/rtools/awssdk_ep.patch @@ -90,7 +90,7 @@ index 4dade6489..a0456cf8e 100644 bool WinHttpSyncHttpClient::DoSendRequest(void* hHttpRequest) const { - return (WinHttpSendRequest(hHttpRequest, NULL, NULL, 0, 0, 0, NULL) != 0); -+ return (WinHttpSendRequest(hHttpRequest, NULL, 0, 0, 0, 0, 0) != 0); ++ return (WinHttpSendRequest(hHttpRequest, NULL, 0, NULL, 0, 0, 0) != 0); } bool WinHttpSyncHttpClient::DoReadData(void* hHttpRequest, char* body, uint64_t size, uint64_t& read) const From 62c89337999851176058b0934309f76725914363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 6 Feb 2025 12:14:57 +0100 Subject: [PATCH 42/43] Add back cmake and link to Dockerfile to avoid having to do the link on debian rules --- .../apache-arrow/apt/ubuntu-jammy/Dockerfile | 7 ++++++- dev/tasks/linux-packages/apache-arrow/debian/control.in | 3 +-- dev/tasks/linux-packages/apache-arrow/debian/rules | 3 --- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile index ea082234d75..64db49c3589 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile @@ -33,6 +33,7 @@ RUN \ apt install -y -V ${quiet} \ build-essential \ clang \ + cmake \ curl \ debhelper \ devscripts \ @@ -83,7 +84,11 @@ RUN \ apt clean && \ rm -rf /var/lib/apt/lists/* +# We install CMake via apt and create a symbolic link to avoid having to: +# Temporarily remove the build-depends check on control.in +# Create the link on debian rules as --buildsystem=cmake expects cmake on /usr/bin ARG cmake=3.25.0 RUN curl -L \ "https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-linux-$(uname -m).tar.gz" | \ - tar -xzf - --directory /usr/local --strip-components=1 + tar -xzf - --directory /usr/local --strip-components=1 && \ + ln -fs /usr/local/bin/cmake /usr/bin/cmake diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control.in b/dev/tasks/linux-packages/apache-arrow/debian/control.in index a4836025cfb..87d78d843a2 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/control.in +++ b/dev/tasks/linux-packages/apache-arrow/debian/control.in @@ -3,8 +3,7 @@ Section: devel Priority: optional Maintainer: Apache Arrow Developers Build-Depends: -# TODO: Enable this after we drop support for Ubuntu 22.04. -# cmake, + cmake, debhelper (>= 12), gi-docgen, git, diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules index 5b66f7ca9fa..9f044680159 100755 --- a/dev/tasks/linux-packages/apache-arrow/debian/rules +++ b/dev/tasks/linux-packages/apache-arrow/debian/rules @@ -21,9 +21,6 @@ override_dh_auto_configure: else \ ARROW_AZURE=ON; \ fi; \ - if [ "$${code_name}" = "focal" ] || [ "$${code_name}" = "jammy" ]; then \ - ln -sf /usr/local/bin/cmake /usr/bin/cmake; \ - fi; \ if dpkg -l nvidia-cuda-toolkit > /dev/null 2>&1; then \ ARROW_CUDA=ON; \ else \ From 7d64cb763838f85f8cc974c385fc801e082e7e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 7 Feb 2025 11:37:40 +0100 Subject: [PATCH 43/43] Some final review comments, sort lists, add comment and add empty line --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 4 ++++ docker-compose.yml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 04500096a17..279d2c0fdfe 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4646,6 +4646,9 @@ function(build_orc) set(ZLIB_HOME ${ZLIB_ROOT} CACHE STRING "" FORCE) + # From CMake 3.21 onwards the set(CACHE) command does not remove any normal + # variable of the same name from the current scope. We have to manually remove + # the variable via unset to avoid ORC not finding the ZLIB_LIBRARY. unset(ZLIB_LIBRARY) set(ZLIB_LIBRARY ZLIB::ZLIB @@ -4680,6 +4683,7 @@ function(build_orc) set(STOP_BUILD_ON_WARNING OFF CACHE BOOL "" FORCE) + fetchcontent_makeavailable(orc) add_library(orc::orc INTERFACE IMPORTED) diff --git a/docker-compose.yml b/docker-compose.yml index ec4419a0faa..52e8558e050 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1591,11 +1591,11 @@ services: - ${REPO}:r-${R_ORG}-${R_IMAGE}-${R_TAG} args: base: ${R_ORG}/${R_IMAGE}:${R_TAG} + cmake: ${CMAKE} r_dev: ${ARROW_R_DEV} tz: ${TZ} r_prune_deps: ${R_PRUNE_DEPS} r_custom_ccache: ${R_CUSTOM_CCACHE} - cmake: ${CMAKE} shm_size: *shm-size environment: <<: [*common, *sccache] @@ -2045,9 +2045,9 @@ services: cache_from: - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-verify-rc args: - repo: ${REPO} arch: ${ARCH} cmake: ${CMAKE} + repo: ${REPO} volumes: - .:/arrow:delegated - ${DOCKER_VOLUME_PREFIX}ubuntu-ccache:/ccache:delegated