diff --git a/ci/rtools/README.md b/ci/rtools/README.md deleted file mode 100644 index 08b5ea7f513..00000000000 --- a/ci/rtools/README.md +++ /dev/null @@ -1,35 +0,0 @@ - - -# rtools40 patches for AWS SDK and related libs - -The patches in this directory are solely for the purpose of building Arrow C++ -under [Rtools40](https://cran.r-project.org/bin/windows/Rtools/rtools40.html) -and not used elsewhere. Once we've dropped support for Rtools40, we can consider -removing these patches. - -The larger reason these patches are needed is that Rtools provides their own -packages and their versions of the AWS libraries weren't compatible with CMake -3.25. Our solution was to bundle the AWS libs instead and these patches were -required to get them building under the Rtools40 environment. - -The patches were added while upgrading the minimum required CMake version to -3.25 in [GH-44950](https://github.com/apache/arrow/issues/44950). Please see the -associated PR, [GH-44989](https://github.com/apache/arrow/pull/44989), for more -context. diff --git a/ci/rtools/aws_c_common_ep.patch b/ci/rtools/aws_c_common_ep.patch deleted file mode 100644 index 94c84d0fe1b..00000000000 --- a/ci/rtools/aws_c_common_ep.patch +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -diff --git a/include/aws/common/byte_order.inl b/include/aws/common/byte_order.inl -index 1204be0..0abd9cb 100644 ---- a/include/aws/common/byte_order.inl -+++ b/include/aws/common/byte_order.inl -@@ -13,7 +13,7 @@ - # include - #else - # include --#endif /* _MSC_VER */ -+#endif /* _WIN32 */ - - AWS_EXTERN_C_BEGIN - -@@ -39,7 +39,7 @@ AWS_STATIC_IMPL uint64_t aws_hton64(uint64_t x) { - uint64_t v; - __asm__("bswap %q0" : "=r"(v) : "0"(x)); - return v; --#elif defined(_MSC_VER) -+#elif defined(_WIN32) - return _byteswap_uint64(x); - #else - uint32_t low = x & UINT32_MAX; diff --git a/ci/rtools/aws_c_io_ep.patch b/ci/rtools/aws_c_io_ep.patch deleted file mode 100644 index a15d706ba12..00000000000 --- a/ci/rtools/aws_c_io_ep.patch +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -diff --git a/source/windows/secure_channel_tls_handler.c b/source/windows/secure_channel_tls_handler.c -index 50caf02..29fe850 100644 ---- a/source/windows/secure_channel_tls_handler.c -+++ b/source/windows/secure_channel_tls_handler.c -@@ -35,6 +36,25 @@ - # pragma warning(disable : 4306) /* Identifier is type cast to a larger pointer. */ - #endif - -+#ifndef SP_PROT_TLS1_0_SERVER -+#define SP_PROT_TLS1_0_SERVER SP_PROT_TLS1_SERVER -+#endif -+#ifndef SP_PROT_TLS1_0_CLIENT -+#define SP_PROT_TLS1_0_CLIENT SP_PROT_TLS1_CLIENT -+#endif -+#ifndef SP_PROT_TLS1_1_SERVER -+#define SP_PROT_TLS1_1_SERVER 0x00000100 -+#endif -+#ifndef SP_PROT_TLS1_1_CLIENT -+#define SP_PROT_TLS1_1_CLIENT 0x00000200 -+#endif -+#ifndef SCH_USE_STRONG_CRYPTO -+#define SCH_USE_STRONG_CRYPTO 0x00400000 -+#endif -+#ifndef SECBUFFER_ALERT -+#define SECBUFFER_ALERT 0x11 -+#endif -+ - #define KB_1 1024 - #define READ_OUT_SIZE (16 * KB_1) - #define READ_IN_SIZE READ_OUT_SIZE -@@ -456,7 +476,7 @@ static int s_fillin_alpn_data( - - *extension_length += sizeof(uint32_t) + sizeof(uint16_t); - -- *extension_name = SecApplicationProtocolNegotiationExt_ALPN; -+ *extension_name = 2; - /*now add the protocols*/ - for (size_t i = 0; i < protocols_count; ++i) { - struct aws_byte_cursor *protocol_ptr = NULL; diff --git a/ci/rtools/awssdk_ep.patch b/ci/rtools/awssdk_ep.patch deleted file mode 100644 index bd26f853290..00000000000 --- a/ci/rtools/awssdk_ep.patch +++ /dev/null @@ -1,181 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -diff --git a/aws-cpp-sdk-core/include/aws/core/utils/Array.h b/aws-cpp-sdk-core/include/aws/core/utils/Array.h -index 2b5bbc566..7cb93bdf0 100644 ---- a/aws-cpp-sdk-core/include/aws/core/utils/Array.h -+++ b/aws-cpp-sdk-core/include/aws/core/utils/Array.h -@@ -54,7 +54,7 @@ namespace Aws - { - m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); - --#ifdef _WIN32 -+#ifdef _MSC_VER - std::copy(arrayToCopy, arrayToCopy + arraySize, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); - #else - std::copy(arrayToCopy, arrayToCopy + arraySize, m_data.get()); -@@ -82,7 +82,7 @@ namespace Aws - if(arr->m_size > 0 && arr->m_data) - { - size_t arraySize = arr->m_size; --#ifdef _WIN32 -+#ifdef _MSC_VER - std::copy(arr->m_data.get(), arr->m_data.get() + arraySize, stdext::checked_array_iterator< T * >(m_data.get() + location, m_size)); - #else - std::copy(arr->m_data.get(), arr->m_data.get() + arraySize, m_data.get() + location); -@@ -101,7 +101,7 @@ namespace Aws - { - m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); - --#ifdef _WIN32 -+#ifdef _MSC_VER - std::copy(other.m_data.get(), other.m_data.get() + other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); - #else - std::copy(other.m_data.get(), other.m_data.get() + other.m_size, m_data.get()); -@@ -134,7 +134,7 @@ namespace Aws - { - m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); - --#ifdef _WIN32 -+#ifdef _MSC_VER - std::copy(other.m_data.get(), other.m_data.get() + other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); - #else - std::copy(other.m_data.get(), other.m_data.get() + other.m_size, m_data.get()); -diff --git a/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp b/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp -index 4dade6489..a0456cf8e 100644 ---- a/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp -+++ b/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp -@@ -22,6 +22,16 @@ - #include - #include - -+#ifndef WINHTTP_OPTION_WEB_SOCKET_KEEPALIVE_INTERVAL -+#define WINHTTP_OPTION_WEB_SOCKET_KEEPALIVE_INTERVAL 116 -+#endif -+#ifndef WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 -+#define WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 0x00000200 -+#endif -+#ifndef WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2 -+#define WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2 0x00000800 -+#endif -+ - using namespace Aws::Client; - using namespace Aws::Http; - using namespace Aws::Http::Standard; -@@ -272,7 +282,7 @@ bool WinHttpSyncHttpClient::DoQueryHeaders(void* hHttpRequest, std::shared_ptr(dwSize / sizeof(wchar_t))); - - WinHttpQueryHeaders(hHttpRequest, WINHTTP_QUERY_CONTENT_TYPE, nullptr, &contentTypeStr, &dwSize, 0); -- if (contentTypeStr[0] != NULL) -+ if (contentTypeStr[0]) - { - Aws::String contentStr = StringUtils::FromWString(contentTypeStr); - response->SetContentType(contentStr); -@@ -303,7 +313,7 @@ bool WinHttpSyncHttpClient::DoQueryHeaders(void* hHttpRequest, std::shared_ptrSetContentType(contentTypeStr); - AWS_LOGSTREAM_DEBUG(GetLogTag(), "Received content type " << contentTypeStr); -diff --git a/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp b/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp -index d7513cc3c..e390a8d4e 100644 ---- a/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp -+++ b/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp -@@ -349,7 +349,7 @@ std::shared_ptr WinSyncHttpClient::MakeRequest(const std::shared_p - } - } - -- if (!success && !IsRequestProcessingEnabled() || !ContinueRequest(*request)) -+ if ((!success && !IsRequestProcessingEnabled()) || !ContinueRequest(*request)) - { - response->SetClientErrorType(CoreErrors::USER_CANCELLED); - response->SetClientErrorMessage("Request processing disabled or continuation cancelled by user's continuation handler."); -diff --git a/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp b/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp -index 2ea82de6f..bc423441e 100644 ---- a/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp -+++ b/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp -@@ -11,7 +11,9 @@ - #include - #include - -+#ifdef _MSC_VER - #pragma warning( disable : 4996) -+#endif - - using namespace Aws::Utils; - namespace Aws -@@ -304,6 +306,9 @@ Aws::String CreateTempFilePath() - { - #ifdef _MSC_VER - #pragma warning(disable: 4996) // _CRT_SECURE_NO_WARNINGS -+#elif !defined(L_tmpnam_s) -+ // Definition from the MSVC stdio.h -+ #define L_tmpnam_s (sizeof("\\") + 16) - #endif - char s_tempName[L_tmpnam_s+1]; - -diff --git a/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp b/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp -index 0180f7fbf..3adbab313 100644 ---- a/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp -+++ b/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp -@@ -9,7 +9,9 @@ - - #include - -+#ifdef _MSC_VER - #pragma warning(disable: 4996) -+#endif - #include - #include - namespace Aws -diff --git a/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp b/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp -index 2ee517b48..3b0dce665 100644 ---- a/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp -+++ b/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp -@@ -939,7 +939,7 @@ std::shared_ptr Aws::Utils::Crypto::CreateSha256HMACIm - return GetSha256HMACFactory()->CreateImplementation(); - } - --#ifdef _WIN32 -+#ifdef _MSC_VER - #pragma warning( push ) - #pragma warning( disable : 4702 ) - #endif -@@ -1032,7 +1032,7 @@ std::shared_ptr Aws::Utils::Crypto::CreateAES_KeyWrapImplementa - return GetAES_KeyWrapFactory()->CreateImplementation(key); - } - --#ifdef _WIN32 -+#ifdef _MSC_VER - #pragma warning(pop) - #endif diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 60a77c815e3..18ff93e965f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -56,6 +56,11 @@ cmake_policy(SET CMP0068 NEW) # find_package() uses _ROOT variables. cmake_policy(SET CMP0074 NEW) +# https://cmake.org/cmake/help/latest/policy/CMP0077.html +# +# option() honors normal variables. +cmake_policy(SET CMP0077 NEW) + # https://cmake.org/cmake/help/latest/policy/CMP0091.html # # MSVC runtime library flags are selected by an abstraction. @@ -387,14 +392,10 @@ endif() # where to put generated archives (.a files) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") -set(ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") - # where to put generated libraries (.so files) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") -set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") - # where to put generated binaries -set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}") +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") if(CMAKE_GENERATOR STREQUAL Xcode) # Xcode projects support multi-configuration builds. This forces a single output directory diff --git a/cpp/cmake_modules/AWSSDKVariables.cmake b/cpp/cmake_modules/AWSSDKVariables.cmake deleted file mode 100644 index 729790dd0f8..00000000000 --- a/cpp/cmake_modules/AWSSDKVariables.cmake +++ /dev/null @@ -1,388 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Generated by: -# $ cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh 1.10.55 - -set(AWSSDK_UNUSED_DIRECTORIES - .github - AndroidSDKTesting - CI - Docs - android-build - android-unified-tests - aws-cpp-sdk-AWSMigrationHub - aws-cpp-sdk-access-management - aws-cpp-sdk-accessanalyzer - aws-cpp-sdk-account - aws-cpp-sdk-acm - aws-cpp-sdk-acm-pca - aws-cpp-sdk-alexaforbusiness - aws-cpp-sdk-amp - aws-cpp-sdk-amplify - aws-cpp-sdk-amplifybackend - aws-cpp-sdk-amplifyuibuilder - aws-cpp-sdk-apigateway - aws-cpp-sdk-apigatewaymanagementapi - aws-cpp-sdk-apigatewayv2 - aws-cpp-sdk-appconfig - aws-cpp-sdk-appconfigdata - aws-cpp-sdk-appflow - aws-cpp-sdk-appintegrations - aws-cpp-sdk-application-autoscaling - aws-cpp-sdk-application-insights - aws-cpp-sdk-applicationcostprofiler - aws-cpp-sdk-appmesh - aws-cpp-sdk-apprunner - aws-cpp-sdk-appstream - aws-cpp-sdk-appsync - aws-cpp-sdk-arc-zonal-shift - aws-cpp-sdk-athena - aws-cpp-sdk-auditmanager - aws-cpp-sdk-autoscaling - aws-cpp-sdk-autoscaling-plans - aws-cpp-sdk-awstransfer - aws-cpp-sdk-backup - aws-cpp-sdk-backup-gateway - aws-cpp-sdk-backupstorage - aws-cpp-sdk-batch - aws-cpp-sdk-billingconductor - aws-cpp-sdk-braket - aws-cpp-sdk-budgets - aws-cpp-sdk-ce - aws-cpp-sdk-chime - aws-cpp-sdk-chime-sdk-identity - aws-cpp-sdk-chime-sdk-media-pipelines - aws-cpp-sdk-chime-sdk-meetings - aws-cpp-sdk-chime-sdk-messaging - aws-cpp-sdk-chime-sdk-voice - aws-cpp-sdk-cleanrooms - aws-cpp-sdk-cloud9 - aws-cpp-sdk-cloudcontrol - aws-cpp-sdk-clouddirectory - aws-cpp-sdk-cloudformation - aws-cpp-sdk-cloudfront - aws-cpp-sdk-cloudfront-integration-tests - aws-cpp-sdk-cloudhsm - aws-cpp-sdk-cloudhsmv2 - aws-cpp-sdk-cloudsearch - aws-cpp-sdk-cloudsearchdomain - aws-cpp-sdk-cloudtrail - aws-cpp-sdk-codeartifact - aws-cpp-sdk-codebuild - aws-cpp-sdk-codecatalyst - aws-cpp-sdk-codecommit - aws-cpp-sdk-codedeploy - aws-cpp-sdk-codeguru-reviewer - aws-cpp-sdk-codeguruprofiler - aws-cpp-sdk-codepipeline - aws-cpp-sdk-codestar - aws-cpp-sdk-codestar-connections - aws-cpp-sdk-codestar-notifications - aws-cpp-sdk-cognito-idp - aws-cpp-sdk-cognito-sync - aws-cpp-sdk-cognitoidentity-integration-tests - aws-cpp-sdk-comprehend - aws-cpp-sdk-comprehendmedical - aws-cpp-sdk-compute-optimizer - aws-cpp-sdk-connect - aws-cpp-sdk-connect-contact-lens - aws-cpp-sdk-connectcampaigns - aws-cpp-sdk-connectcases - aws-cpp-sdk-connectparticipant - aws-cpp-sdk-controltower - aws-cpp-sdk-cur - aws-cpp-sdk-custom-service-integration-tests - aws-cpp-sdk-customer-profiles - aws-cpp-sdk-databrew - aws-cpp-sdk-dataexchange - aws-cpp-sdk-datapipeline - aws-cpp-sdk-datasync - aws-cpp-sdk-dax - aws-cpp-sdk-detective - aws-cpp-sdk-devicefarm - aws-cpp-sdk-devops-guru - aws-cpp-sdk-directconnect - aws-cpp-sdk-discovery - aws-cpp-sdk-dlm - aws-cpp-sdk-dms - aws-cpp-sdk-docdb - aws-cpp-sdk-docdb-elastic - aws-cpp-sdk-drs - aws-cpp-sdk-ds - aws-cpp-sdk-dynamodb - aws-cpp-sdk-dynamodb-integration-tests - aws-cpp-sdk-dynamodbstreams - aws-cpp-sdk-ebs - aws-cpp-sdk-ec2 - aws-cpp-sdk-ec2-instance-connect - aws-cpp-sdk-ec2-integration-tests - aws-cpp-sdk-ecr - aws-cpp-sdk-ecr-public - aws-cpp-sdk-ecs - aws-cpp-sdk-eks - aws-cpp-sdk-elastic-inference - aws-cpp-sdk-elasticache - aws-cpp-sdk-elasticbeanstalk - aws-cpp-sdk-elasticfilesystem - aws-cpp-sdk-elasticfilesystem-integration-tests - aws-cpp-sdk-elasticloadbalancing - aws-cpp-sdk-elasticloadbalancingv2 - aws-cpp-sdk-elasticmapreduce - aws-cpp-sdk-elastictranscoder - aws-cpp-sdk-email - aws-cpp-sdk-emr-containers - aws-cpp-sdk-emr-serverless - aws-cpp-sdk-es - aws-cpp-sdk-eventbridge - aws-cpp-sdk-eventbridge-tests - aws-cpp-sdk-events - aws-cpp-sdk-evidently - aws-cpp-sdk-finspace - aws-cpp-sdk-finspace-data - aws-cpp-sdk-firehose - aws-cpp-sdk-fis - aws-cpp-sdk-fms - aws-cpp-sdk-forecast - aws-cpp-sdk-forecastquery - aws-cpp-sdk-frauddetector - aws-cpp-sdk-fsx - aws-cpp-sdk-gamelift - aws-cpp-sdk-gamesparks - aws-cpp-sdk-glacier - aws-cpp-sdk-globalaccelerator - aws-cpp-sdk-glue - aws-cpp-sdk-grafana - aws-cpp-sdk-greengrass - aws-cpp-sdk-greengrassv2 - aws-cpp-sdk-groundstation - aws-cpp-sdk-guardduty - aws-cpp-sdk-health - aws-cpp-sdk-healthlake - aws-cpp-sdk-honeycode - aws-cpp-sdk-iam - aws-cpp-sdk-identitystore - aws-cpp-sdk-imagebuilder - aws-cpp-sdk-importexport - aws-cpp-sdk-inspector - aws-cpp-sdk-inspector2 - aws-cpp-sdk-iot - aws-cpp-sdk-iot-data - aws-cpp-sdk-iot-jobs-data - aws-cpp-sdk-iot-roborunner - aws-cpp-sdk-iot1click-devices - aws-cpp-sdk-iot1click-projects - aws-cpp-sdk-iotanalytics - aws-cpp-sdk-iotdeviceadvisor - aws-cpp-sdk-iotevents - aws-cpp-sdk-iotevents-data - aws-cpp-sdk-iotfleethub - aws-cpp-sdk-iotfleetwise - aws-cpp-sdk-iotsecuretunneling - aws-cpp-sdk-iotsitewise - aws-cpp-sdk-iotthingsgraph - aws-cpp-sdk-iottwinmaker - aws-cpp-sdk-iotwireless - aws-cpp-sdk-ivs - aws-cpp-sdk-ivschat - aws-cpp-sdk-kafka - aws-cpp-sdk-kafkaconnect - aws-cpp-sdk-kendra - aws-cpp-sdk-kendra-ranking - aws-cpp-sdk-keyspaces - aws-cpp-sdk-kinesis - aws-cpp-sdk-kinesis-integration-tests - aws-cpp-sdk-kinesis-video-archived-media - aws-cpp-sdk-kinesis-video-media - aws-cpp-sdk-kinesis-video-signaling - aws-cpp-sdk-kinesis-video-webrtc-storage - aws-cpp-sdk-kinesisanalytics - aws-cpp-sdk-kinesisanalyticsv2 - aws-cpp-sdk-kinesisvideo - aws-cpp-sdk-kms - aws-cpp-sdk-lakeformation - aws-cpp-sdk-lambda - aws-cpp-sdk-lambda-integration-tests - aws-cpp-sdk-lex - aws-cpp-sdk-lex-models - aws-cpp-sdk-lexv2-models - aws-cpp-sdk-lexv2-runtime - aws-cpp-sdk-license-manager - aws-cpp-sdk-license-manager-linux-subscriptions - aws-cpp-sdk-license-manager-user-subscriptions - aws-cpp-sdk-lightsail - aws-cpp-sdk-location - aws-cpp-sdk-logs - aws-cpp-sdk-logs-integration-tests - aws-cpp-sdk-lookoutequipment - aws-cpp-sdk-lookoutmetrics - aws-cpp-sdk-lookoutvision - aws-cpp-sdk-m2 - aws-cpp-sdk-machinelearning - aws-cpp-sdk-macie - aws-cpp-sdk-macie2 - aws-cpp-sdk-managedblockchain - aws-cpp-sdk-marketplace-catalog - aws-cpp-sdk-marketplace-entitlement - aws-cpp-sdk-marketplacecommerceanalytics - aws-cpp-sdk-mediaconnect - aws-cpp-sdk-mediaconvert - aws-cpp-sdk-medialive - aws-cpp-sdk-mediapackage - aws-cpp-sdk-mediapackage-vod - aws-cpp-sdk-mediastore - aws-cpp-sdk-mediastore-data - aws-cpp-sdk-mediastore-data-integration-tests - aws-cpp-sdk-mediatailor - aws-cpp-sdk-memorydb - aws-cpp-sdk-meteringmarketplace - aws-cpp-sdk-mgn - aws-cpp-sdk-migration-hub-refactor-spaces - aws-cpp-sdk-migrationhub-config - aws-cpp-sdk-migrationhuborchestrator - aws-cpp-sdk-migrationhubstrategy - aws-cpp-sdk-mobile - aws-cpp-sdk-monitoring - aws-cpp-sdk-mq - aws-cpp-sdk-mturk-requester - aws-cpp-sdk-mwaa - aws-cpp-sdk-neptune - aws-cpp-sdk-network-firewall - aws-cpp-sdk-networkmanager - aws-cpp-sdk-nimble - aws-cpp-sdk-oam - aws-cpp-sdk-omics - aws-cpp-sdk-opensearch - aws-cpp-sdk-opensearchserverless - aws-cpp-sdk-opsworks - aws-cpp-sdk-opsworkscm - aws-cpp-sdk-organizations - aws-cpp-sdk-outposts - aws-cpp-sdk-panorama - aws-cpp-sdk-personalize - aws-cpp-sdk-personalize-events - aws-cpp-sdk-personalize-runtime - aws-cpp-sdk-pi - aws-cpp-sdk-pinpoint - aws-cpp-sdk-pinpoint-email - aws-cpp-sdk-pinpoint-sms-voice-v2 - aws-cpp-sdk-pipes - aws-cpp-sdk-polly - aws-cpp-sdk-polly-sample - aws-cpp-sdk-pricing - aws-cpp-sdk-privatenetworks - aws-cpp-sdk-proton - aws-cpp-sdk-qldb - aws-cpp-sdk-qldb-session - aws-cpp-sdk-queues - aws-cpp-sdk-quicksight - aws-cpp-sdk-ram - aws-cpp-sdk-rbin - aws-cpp-sdk-rds - aws-cpp-sdk-rds-data - aws-cpp-sdk-rds-integration-tests - aws-cpp-sdk-redshift - aws-cpp-sdk-redshift-data - aws-cpp-sdk-redshift-integration-tests - aws-cpp-sdk-redshift-serverless - aws-cpp-sdk-rekognition - aws-cpp-sdk-resiliencehub - aws-cpp-sdk-resource-explorer-2 - aws-cpp-sdk-resource-groups - aws-cpp-sdk-resourcegroupstaggingapi - aws-cpp-sdk-robomaker - aws-cpp-sdk-rolesanywhere - aws-cpp-sdk-route53 - aws-cpp-sdk-route53-recovery-cluster - aws-cpp-sdk-route53-recovery-control-config - aws-cpp-sdk-route53-recovery-readiness - aws-cpp-sdk-route53domains - aws-cpp-sdk-route53resolver - aws-cpp-sdk-rum - aws-cpp-sdk-sagemaker - aws-cpp-sdk-sagemaker-a2i-runtime - aws-cpp-sdk-sagemaker-edge - aws-cpp-sdk-sagemaker-featurestore-runtime - aws-cpp-sdk-sagemaker-geospatial - aws-cpp-sdk-sagemaker-metrics - aws-cpp-sdk-sagemaker-runtime - aws-cpp-sdk-savingsplans - aws-cpp-sdk-scheduler - aws-cpp-sdk-schemas - aws-cpp-sdk-sdb - aws-cpp-sdk-secretsmanager - aws-cpp-sdk-securityhub - aws-cpp-sdk-securitylake - aws-cpp-sdk-serverlessrepo - aws-cpp-sdk-service-quotas - aws-cpp-sdk-servicecatalog - aws-cpp-sdk-servicecatalog-appregistry - aws-cpp-sdk-servicediscovery - aws-cpp-sdk-sesv2 - aws-cpp-sdk-shield - aws-cpp-sdk-signer - aws-cpp-sdk-simspaceweaver - aws-cpp-sdk-sms - aws-cpp-sdk-sms-voice - aws-cpp-sdk-snow-device-management - aws-cpp-sdk-snowball - aws-cpp-sdk-sns - aws-cpp-sdk-sqs - aws-cpp-sdk-sqs-integration-tests - aws-cpp-sdk-ssm - aws-cpp-sdk-ssm-contacts - aws-cpp-sdk-ssm-incidents - aws-cpp-sdk-ssm-sap - aws-cpp-sdk-sso - aws-cpp-sdk-sso-admin - aws-cpp-sdk-sso-oidc - aws-cpp-sdk-states - aws-cpp-sdk-storagegateway - aws-cpp-sdk-support - aws-cpp-sdk-support-app - aws-cpp-sdk-swf - aws-cpp-sdk-synthetics - aws-cpp-sdk-text-to-speech - aws-cpp-sdk-text-to-speech-tests - aws-cpp-sdk-textract - aws-cpp-sdk-timestream-query - aws-cpp-sdk-timestream-write - aws-cpp-sdk-transcribe - aws-cpp-sdk-transcribestreaming - aws-cpp-sdk-transcribestreaming-integration-tests - aws-cpp-sdk-translate - aws-cpp-sdk-voice-id - aws-cpp-sdk-waf - aws-cpp-sdk-waf-regional - aws-cpp-sdk-wafv2 - aws-cpp-sdk-wellarchitected - aws-cpp-sdk-wisdom - aws-cpp-sdk-workdocs - aws-cpp-sdk-worklink - aws-cpp-sdk-workmail - aws-cpp-sdk-workmailmessageflow - aws-cpp-sdk-workspaces - aws-cpp-sdk-workspaces-web - aws-cpp-sdk-xray - code-generation - crt - doc_crosslinks - doc_crosslinks_new - doxygen - generated - scripts - testing-resources) diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index 32d962b0cf2..134f47b12ef 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -552,7 +552,7 @@ function(ADD_BENCHMARK REL_BENCHMARK_NAME) if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME}.cc) # This benchmark has a corresponding .cc file, set it up as an executable. - set(BENCHMARK_PATH "${EXECUTABLE_OUTPUT_PATH}/${BENCHMARK_NAME}") + set(BENCHMARK_PATH "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${BENCHMARK_NAME}") add_executable(${BENCHMARK_NAME} ${SOURCES}) if(ARG_STATIC_LINK_LIBS) @@ -581,7 +581,8 @@ function(ADD_BENCHMARK REL_BENCHMARK_NAME) PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH TRUE INSTALL_RPATH - "$ENV{CONDA_PREFIX}/lib;${EXECUTABLE_OUTPUT_PATH}") + "$ENV{CONDA_PREFIX}/lib;${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" + ) endif() # Add test as dependency of relevant label targets @@ -682,7 +683,7 @@ function(ADD_TEST_CASE REL_TEST_NAME) # Make sure the executable name contains only hyphens, not underscores string(REPLACE "_" "-" TEST_NAME ${TEST_NAME}) - set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}") + set(TEST_PATH "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_NAME}") add_executable(${TEST_NAME} ${SOURCES}) # With OSX and conda, we need to set the correct RPATH so that dependencies @@ -695,7 +696,8 @@ function(ADD_TEST_CASE REL_TEST_NAME) PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH TRUE INSTALL_RPATH - "${EXECUTABLE_OUTPUT_PATH};$ENV{CONDA_PREFIX}/lib") + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY};$ENV{CONDA_PREFIX}/lib" + ) endif() # Ensure using bundled GoogleTest when we use bundled GoogleTest. @@ -826,7 +828,7 @@ function(ADD_ARROW_EXAMPLE REL_EXAMPLE_NAME) if(EXISTS ${CMAKE_SOURCE_DIR}/examples/arrow/${REL_EXAMPLE_NAME}.cc) # This example has a corresponding .cc file, set it up as an executable. - set(EXAMPLE_PATH "${EXECUTABLE_OUTPUT_PATH}/${EXAMPLE_NAME}") + set(EXAMPLE_PATH "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${EXAMPLE_NAME}") add_executable(${EXAMPLE_NAME} "${REL_EXAMPLE_NAME}.cc" ${ARG_EXTRA_SOURCES}) target_link_libraries(${EXAMPLE_NAME} ${ARROW_EXAMPLE_LINK_LIBS}) add_dependencies(runexample ${EXAMPLE_NAME}) diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index fdb28b540e2..9ecca91ecd5 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -152,8 +152,6 @@ set(CMAKE_CXX_EXTENSIONS OFF) # shared libraries set(CMAKE_POSITION_INDEPENDENT_CODE ${ARROW_POSITION_INDEPENDENT_CODE}) -string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE) - set(UNKNOWN_COMPILER_MESSAGE "Unknown compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}") @@ -265,7 +263,7 @@ endif() # `RELEASE`, then it will default to `PRODUCTION`. The goal of defaulting to # `CHECKIN` is to avoid friction with long response time from CI. if(NOT BUILD_WARNING_LEVEL) - if("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE") + if("${UPPERCASE_BUILD_TYPE}" STREQUAL "RELEASE") set(BUILD_WARNING_LEVEL PRODUCTION) else() set(BUILD_WARNING_LEVEL CHECKIN) @@ -602,7 +600,7 @@ if(NOT WIN32 AND NOT APPLE) if(MUST_USE_GOLD) message(STATUS "Using hard-wired gold linker (version ${GOLD_VERSION})") if(ARROW_BUGGY_GOLD) - if("${ARROW_LINK}" STREQUAL "d" AND "${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE") + if("${ARROW_LINK}" STREQUAL "d" AND "${UPPERCASE_BUILD_TYPE}" STREQUAL "RELEASE") message(SEND_ERROR "Configured to use buggy gold with dynamic linking " "in a RELEASE build") endif() @@ -808,7 +806,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") set(CMAKE_SHARED_LINKER_FLAGS "-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS}") if(ARROW_TESTING) # flags for building test executables for use in node - if("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE") + if("${UPPERCASE_BUILD_TYPE}" STREQUAL "RELEASE") set(CMAKE_EXE_LINKER_FLAGS "${ARROW_EMSCRIPTEN_LINKER_FLAGS} -sALLOW_MEMORY_GROWTH -lnodefs.js -lnoderawfs.js --pre-js ${BUILD_SUPPORT_DIR}/emscripten-test-init.js" ) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 193711a6fbd..5264beed783 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1012,7 +1012,7 @@ endif() set(MAKE_BUILD_ARGS "-j${NPROC}") include(FetchContent) -set(FC_DECLARE_COMMON_OPTIONS) +set(FC_DECLARE_COMMON_OPTIONS SYSTEM) if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.28) list(APPEND FC_DECLARE_COMMON_OPTIONS EXCLUDE_FROM_ALL TRUE) endif() @@ -1020,13 +1020,17 @@ endif() macro(prepare_fetchcontent) set(BUILD_SHARED_LIBS OFF) set(BUILD_STATIC_LIBS ON) - set(CMAKE_COMPILE_WARNING_AS_ERROR FALSE) - set(CMAKE_EXPORT_NO_PACKAGE_REGISTRY TRUE) + set(BUILD_TESTING OFF) + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "") + set(CMAKE_COMPILE_WARNING_AS_ERROR OFF) + set(CMAKE_EXPORT_NO_PACKAGE_REGISTRY OFF) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "") set(CMAKE_MACOSX_RPATH ${ARROW_INSTALL_NAME_RPATH}) # We set CMAKE_POLICY_VERSION_MINIMUM temporarily due to failures with CMake 4 # We should remove it once we have updated the dependencies: # https://github.com/apache/arrow/issues/45985 set(CMAKE_POLICY_VERSION_MINIMUM 3.5) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "") if(MSVC) string(REPLACE "/WX" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") @@ -1058,7 +1062,7 @@ macro(build_boost) if(ARROW_BOOST_REQUIRE_LIBRARY) set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib") set(BOOST_BUILD_LINK "static") - if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + if("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG") set(BOOST_BUILD_VARIANT "debug") else() set(BOOST_BUILD_VARIANT "release") @@ -2657,11 +2661,13 @@ if(ARROW_WITH_ZLIB) resolve_dependency(ZLIB PC_PACKAGE_NAMES zlib) endif() -macro(build_lz4) +function(build_lz4) message(STATUS "Building LZ4 from source using FetchContent") # Set LZ4 as vendored - set(LZ4_VENDORED TRUE) + set(LZ4_VENDORED + TRUE + PARENT_SCOPE) # Declare the content fetchcontent_declare(lz4 @@ -2692,7 +2698,10 @@ macro(build_lz4) # Add to bundled static libs. # We must use lz4_static (not imported target) not LZ4::lz4 (imported target). list(APPEND ARROW_BUNDLED_STATIC_LIBS lz4_static) -endmacro() + set(ARROW_BUNDLED_STATIC_LIBS + ${ARROW_BUNDLED_STATIC_LIBS} + PARENT_SCOPE) +endfunction() if(ARROW_WITH_LZ4) resolve_dependency(lz4 @@ -4778,7 +4787,7 @@ function(build_orc) ${ARROW_PROTOBUF_PROTOC} ${ARROW_ZSTD_LIBZSTD} ${Snappy_TARGET} - LZ4::lz4 + ${ORC_LZ4_TARGET} ZLIB::ZLIB) add_library(orc::orc STATIC IMPORTED) set_target_properties(orc::orc PROPERTIES IMPORTED_LOCATION "${ORC_STATIC_LIB}") @@ -5055,439 +5064,217 @@ endif() # ---------------------------------------------------------------------- # AWS SDK for C++ -include(AWSSDKVariables) +function(build_awssdk) + message(STATUS "Building AWS SDK for C++ from source") -macro(build_awssdk) - message(STATUS "Building AWS C++ SDK from source") - set(AWSSDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-install") - set(AWSSDK_INCLUDE_DIR "${AWSSDK_PREFIX}/include") - - # The AWS SDK has a few warnings around shortening lengths - set(AWS_C_FLAGS "${EP_C_FLAGS}") - set(AWS_CXX_FLAGS "${EP_CXX_FLAGS}") - if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL - "Clang") - # Negate warnings that AWS SDK cannot build under - string(APPEND AWS_C_FLAGS " -Wno-error=shorten-64-to-32") - string(APPEND AWS_CXX_FLAGS " -Wno-error=shorten-64-to-32") - endif() - if(NOT MSVC) - string(APPEND AWS_C_FLAGS " -Wno-deprecated") - string(APPEND AWS_CXX_FLAGS " -Wno-deprecated") + # aws-c-common must be the first product because others depend on + # this. + set(AWSSDK_PRODUCTS aws-c-common) + if(LINUX) + list(APPEND AWSSDK_PRODUCTS aws-lc s2n-tls) endif() - # GH-44950: This is required to build under Rtools40 and we may be able to - # remove it if/when we no longer need to build under Rtools40 - if(WIN32 AND NOT MSVC) - string(APPEND - AWS_C_FLAGS - " -D_WIN32_WINNT=0x0601 -D__USE_MINGW_ANSI_STDIO=1 -Wno-error -Wno-error=format= -Wno-error=format-extra-args -Wno-unused-local-typedefs -Wno-unused-variable" - ) - string(APPEND - AWS_CXX_FLAGS - " -D_WIN32_WINNT=0x0601 -D__USE_MINGW_ANSI_STDIO=1 -Wno-error -Wno-error=format= -Wno-error=format-extra-args -Wno-unused-local-typedefs -Wno-unused-variable" + list(APPEND + AWSSDK_PRODUCTS + # We can't sort this in alphabetical order because some + # products depend on other products. + aws-checksums + aws-c-cal + aws-c-io + aws-c-event-stream + aws-c-sdkutils + aws-c-compression + aws-c-http + aws-c-mqtt + aws-c-auth + aws-c-s3 + aws-crt-cpp + aws-sdk-cpp) + set(AWS_SDK_CPP_SOURCE_URL "${AWSSDK_SOURCE_URL}") + set(ARROW_AWS_SDK_CPP_BUILD_SHA256_CHECKSUM "${ARROW_AWSSDK_BUILD_SHA256_CHECKSUM}") + foreach(AWSSDK_PRODUCT ${AWSSDK_PRODUCTS}) + # aws-c-cal -> + # AWS-C-CAL + string(TOUPPER "${AWSSDK_PRODUCT}" BASE_VARIABLE_NAME) + # AWS-C-CAL -> + # AWS_C_CAL + string(REGEX REPLACE "-" "_" BASE_VARIABLE_NAME "${BASE_VARIABLE_NAME}") + if(MINGW AND AWSSDK_PRODUCT STREQUAL "aws-c-common") + find_program(PATCH patch REQUIRED) + set(${BASE_VARIABLE_NAME}_PATCH_COMMAND + ${PATCH} -p1 -i ${CMAKE_CURRENT_LIST_DIR}/aws-c-common-1208.patch) + endif() + fetchcontent_declare(${AWSSDK_PRODUCT} + ${FC_DECLARE_COMMON_OPTIONS} OVERRIDE_FIND_PACKAGE + PATCH_COMMAND ${${BASE_VARIABLE_NAME}_PATCH_COMMAND} + URL ${${BASE_VARIABLE_NAME}_SOURCE_URL} + URL_HASH "SHA256=${ARROW_${BASE_VARIABLE_NAME}_BUILD_SHA256_CHECKSUM}" ) - endif() + endforeach() - set(AWSSDK_COMMON_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_C_FLAGS=${AWS_C_FLAGS} - -DCMAKE_CXX_FLAGS=${AWS_CXX_FLAGS} - -DCPP_STANDARD=${CMAKE_CXX_STANDARD} - -DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX} - -DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX} - -DENABLE_TESTING=OFF - -DENABLE_UNITY_BUILD=ON - -DOPENSSL_CRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY} - -DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR} - -DOPENSSL_SSL_LIBRARY=${OPENSSL_SSL_LIBRARY} - -Dcrypto_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR} - -Dcrypto_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}) - if(ARROW_OPENSSL_USE_SHARED) - list(APPEND AWSSDK_COMMON_CMAKE_ARGS - -Dcrypto_SHARED_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}) - else() - list(APPEND AWSSDK_COMMON_CMAKE_ARGS - -Dcrypto_STATIC_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}) - endif() - set(AWSSDK_CMAKE_ARGS - ${AWSSDK_COMMON_CMAKE_ARGS} - -DBUILD_DEPS=OFF - -DBUILD_ONLY=config\\$s3\\$transfer\\$identity-management\\$sts - -DMINIMIZE_SIZE=ON) - # Remove unused directories to save build directory storage. - # 807MB -> 31MB - set(AWSSDK_PATCH_COMMAND ${CMAKE_COMMAND} -E) - if(CMAKE_VERSION VERSION_LESS 3.17) - list(APPEND AWSSDK_PATCH_COMMAND remove_directory) - else() - list(APPEND AWSSDK_PATCH_COMMAND rm -rf) + prepare_fetchcontent() + set(BUILD_DEPS OFF) + set(BUILD_TOOL OFF) + set(ENABLE_TESTING OFF) + set(IN_SOURCE_BUILD ON) + set(MINIMIZE_SIZE ON) + set(USE_OPENSSL ON) + + # For aws-c-common + if(MINGW) + # PPROCESSOR_NUMBER requires Windows 7 or later. + # + # 0x0601 == _WIN32_WINNT_WIN7 + string(APPEND CMAKE_C_FLAGS " -D_WIN32_WINNT=0x0601") + string(APPEND CMAKE_CXX_FLAGS " -D_WIN32_WINNT=0x0601") endif() - list(APPEND AWSSDK_PATCH_COMMAND ${AWSSDK_UNUSED_DIRECTORIES}) - # Patch parts of the AWSSDK EP so it builds cleanly under Rtools40 - if(WIN32 AND NOT MSVC) - find_program(PATCH patch REQUIRED) - # Patch aws_c_common to build under Rtools40 - set(AWS_C_COMMON_PATCH_COMMAND ${PATCH} -p1 -i - ${CMAKE_SOURCE_DIR}/../ci/rtools/aws_c_common_ep.patch) - message(STATUS "Hello ${AWS_C_COMMON_PATCH_COMMAND}") - # aws_c_io_ep to build under Rtools40 - set(AWS_C_IO_PATCH_COMMAND ${PATCH} -p1 -i - ${CMAKE_SOURCE_DIR}/../ci/rtools/aws_c_io_ep.patch) - message(STATUS "Hello ${AWS_C_IO_PATCH_COMMAND}") - # awssdk_ep to build under Rtools40 - list(APPEND - AWSSDK_PATCH_COMMAND - && - ${PATCH} - -p1 - -i - ${CMAKE_SOURCE_DIR}/../ci/rtools/awssdk_ep.patch) - message(STATUS "Hello ${AWSSDK_PATCH_COMMAND}") - endif() + # For aws-lc + set(DISABLE_GO ON) + set(DISABLE_PERL ON) - if(UNIX) - # on Linux and macOS curl seems to be required - find_curl() - get_filename_component(CURL_ROOT_HINT "${CURL_INCLUDE_DIRS}" DIRECTORY) - get_filename_component(ZLIB_ROOT_HINT "${ZLIB_INCLUDE_DIRS}" DIRECTORY) + # For s2n-tls + set(crypto_INCLUDE_DIR "$") + set(crypto_STATIC_LIBRARY "$") + set(S2N_INTERN_LIBCRYPTO ON) - # provide hint for AWS SDK to link with the already located libcurl and zlib - list(APPEND - AWSSDK_CMAKE_ARGS - -DCURL_INCLUDE_DIR=${CURL_ROOT_HINT}/include - -DCURL_LIBRARY=${CURL_ROOT_HINT}/lib - -DZLIB_INCLUDE_DIR=${ZLIB_ROOT_HINT}/include - -DZLIB_LIBRARY=${ZLIB_ROOT_HINT}/lib) - endif() - - file(MAKE_DIRECTORY ${AWSSDK_INCLUDE_DIR}) - - # AWS C++ SDK related libraries to link statically - set(_AWSSDK_LIBS - aws-cpp-sdk-identity-management - aws-cpp-sdk-sts - aws-cpp-sdk-cognito-identity - aws-cpp-sdk-s3 - aws-cpp-sdk-core - aws-crt-cpp - aws-c-s3 - aws-c-auth - aws-c-mqtt - aws-c-http - aws-c-compression - aws-c-sdkutils - aws-c-event-stream - aws-c-io - aws-c-cal - aws-checksums - aws-c-common) - - # aws-lc needs to be installed on a separate folder to hide from unintended use - set(AWS_LC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws_lc_ep-install") - set(AWS_LC_INCLUDE_DIR "${AWS_LC_PREFIX}/include") - - if(UNIX AND NOT APPLE) # aws-lc and s2n-tls only needed on linux - file(MAKE_DIRECTORY ${AWS_LC_INCLUDE_DIR}) - list(APPEND _AWSSDK_LIBS s2n-tls aws-lc) - endif() - - set(AWSSDK_LIBRARIES) - foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) - # aws-c-common -> AWS-C-COMMON - string(TOUPPER ${_AWSSDK_LIB} _AWSSDK_LIB_UPPER) - # AWS-C-COMMON -> AWS_C_COMMON - string(REPLACE "-" "_" _AWSSDK_LIB_NAME_PREFIX ${_AWSSDK_LIB_UPPER}) - set(_AWSSDK_STATIC_LIBRARY - "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWSSDK_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - if(${_AWSSDK_LIB} STREQUAL "s2n-tls") # Build output of s2n-tls is libs2n.a - set(_AWSSDK_STATIC_LIBRARY - "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}s2n${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - elseif(${_AWSSDK_LIB} STREQUAL "aws-lc") # We only need libcrypto from aws-lc - set(_AWSSDK_STATIC_LIBRARY - "${AWS_LC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}crypto${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - endif() - if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-") - set(_AWSSDK_TARGET_NAME ${_AWSSDK_LIB}) - elseif(${_AWSSDK_LIB} STREQUAL "aws-lc") - set(_AWSSDK_TARGET_NAME AWS::crypto) - else() - set(_AWSSDK_TARGET_NAME AWS::${_AWSSDK_LIB}) - endif() - add_library(${_AWSSDK_TARGET_NAME} STATIC IMPORTED) - set_target_properties(${_AWSSDK_TARGET_NAME} PROPERTIES IMPORTED_LOCATION - ${_AWSSDK_STATIC_LIBRARY}) - target_include_directories(${_AWSSDK_TARGET_NAME} BEFORE - INTERFACE "${AWSSDK_INCLUDE_DIR}") - if(${_AWSSDK_LIB} STREQUAL "aws-lc") - set_target_properties(${_AWSSDK_TARGET_NAME} PROPERTIES IMPORTED_LOCATION - ${_AWSSDK_STATIC_LIBRARY}) - target_include_directories(${_AWSSDK_TARGET_NAME} BEFORE - INTERFACE "${AWS_LC_INCLUDE_DIR}") - endif() - set("${_AWSSDK_LIB_NAME_PREFIX}_STATIC_LIBRARY" ${_AWSSDK_STATIC_LIBRARY}) + # For aws-lc and s2n-tls + # + # Link time optimization is causing trouble like GH-34349 + string(REPLACE "-flto=auto" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + string(REPLACE "-ffat-lto-objects" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") - if(NOT ${_AWSSDK_LIB} STREQUAL "aws-lc") - # aws-lc only linked against s2n but not arrow - list(APPEND AWSSDK_LIBRARIES ${_AWSSDK_TARGET_NAME}) - endif() - endforeach() + # For aws-c-io + if(MINGW AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9") + # This is for RTools 40. We can remove this after we dropped + # support for R < 4.2. schannel.h in RTools 40 is old. - externalproject_add(aws_c_common_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_COMMON_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM}" - PATCH_COMMAND ${AWS_C_COMMON_PATCH_COMMAND} - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_COMMON_STATIC_LIBRARY}) - add_dependencies(AWS::aws-c-common aws_c_common_ep) - - set(AWS_CHECKSUMS_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) - if(NOT WIN32) - # On non-Windows, always build in release mode. - # Especially with gcc, debug builds can fail with "asm constraint" errors: - # https://github.com/TileDB-Inc/TileDB/issues/1351 - list(APPEND AWS_CHECKSUMS_CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release) + # For schannel.h + # + # See also: + # https://learn.microsoft.com/en-us/windows/win32/api/schannel/ns-schannel-schannel_cred + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_0_SERVER=0x00000040") + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_0_CLIENT=0x00000080") + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_1_SERVER=0x00000100") + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_1_CLIENT=0x00000200") + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_2_SERVER=0x00000400") + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_2_CLIENT=0x00000800") + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_3_SERVER=0x00001000") + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_3_CLIENT=0x00002000") + string(APPEND CMAKE_C_FLAGS " -DSCH_USE_STRONG_CRYPTO=0x00400000") + + # For sspi.h + # + # See also: + # https://learn.microsoft.com/en-us/windows/win32/api/sspi/ne-sspi-sec_application_protocol_negotiation_ext + string(APPEND CMAKE_C_FLAGS " -DSecApplicationProtocolNegotiationExt_ALPN=2") + # See also: + # https://learn.microsoft.com/en-us/windows/win32/api/sspi/ns-sspi-secbuffer + string(APPEND CMAKE_C_FLAGS " -DSECBUFFER_ALERT=17") endif() - externalproject_add(aws_checksums_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_CHECKSUMS_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWS_CHECKSUMS_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_CHECKSUMS_STATIC_LIBRARY} - DEPENDS aws_c_common_ep) - add_dependencies(AWS::aws-checksums aws_checksums_ep) - - if("s2n-tls" IN_LIST _AWSSDK_LIBS) - # Remove unused directories to save build directory storage. - # 169MB -> 105MB - set(AWS_LC_PATCH_COMMAND ${CMAKE_COMMAND} -E) - if(CMAKE_VERSION VERSION_LESS 3.17) - list(APPEND AWS_LC_PATCH_COMMAND remove_directory) - else() - list(APPEND AWS_LC_PATCH_COMMAND rm -rf) - endif() - list(APPEND AWS_LC_PATCH_COMMAND fuzz) - set(AWS_LC_C_FLAGS ${EP_C_FLAGS}) - string(APPEND AWS_LC_C_FLAGS " -Wno-error=overlength-strings -Wno-error=pedantic") - # Link time optimization is causing trouble like #34349 - string(REPLACE "-flto=auto" "" AWS_LC_C_FLAGS "${AWS_LC_C_FLAGS}") - string(REPLACE "-ffat-lto-objects" "" AWS_LC_C_FLAGS "${AWS_LC_C_FLAGS}") - - set(AWS_LC_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) - list(APPEND AWS_LC_CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${AWS_LC_PREFIX} - -DCMAKE_C_FLAGS=${AWS_LC_C_FLAGS}) - - externalproject_add(aws_lc_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_LC_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_LC_BUILD_SHA256_CHECKSUM}" - PATCH_COMMAND ${AWS_LC_PATCH_COMMAND} - CMAKE_ARGS ${AWS_LC_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_LC_STATIC_LIBRARY}) - add_dependencies(AWS::crypto aws_lc_ep) - - set(S2N_TLS_C_FLAGS ${EP_C_FLAGS}) - # Link time optimization is causing trouble like #34349 - string(REPLACE "-flto=auto" "" S2N_TLS_C_FLAGS "${S2N_TLS_C_FLAGS}") - string(REPLACE "-ffat-lto-objects" "" S2N_TLS_C_FLAGS "${S2N_TLS_C_FLAGS}") - - set(S2N_TLS_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) - list(APPEND - S2N_TLS_CMAKE_ARGS - # internalize libcrypto to avoid name conflict with OpenSSL - -DS2N_INTERN_LIBCRYPTO=ON - # path to find crypto provided by aws-lc - -DCMAKE_PREFIX_PATH=${AWS_LC_PREFIX} - -DCMAKE_C_FLAGS=${S2N_TLS_C_FLAGS} - # paths to find crypto provided by aws-lc - -Dcrypto_INCLUDE_DIR=${AWS_LC_PREFIX}/include - -Dcrypto_LIBRARY=${AWS_LC_STATIC_LIBRARY} - -Dcrypto_STATIC_LIBRARY=${AWS_LC_STATIC_LIBRARY}) - - externalproject_add(s2n_tls_ep - ${EP_COMMON_OPTIONS} - URL ${S2N_TLS_SOURCE_URL} - URL_HASH "SHA256=${ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${S2N_TLS_CMAKE_ARGS} - BUILD_BYPRODUCTS ${S2N_TLS_STATIC_LIBRARY} - DEPENDS aws_lc_ep) - add_dependencies(AWS::s2n-tls s2n_tls_ep) + # For aws-sdk-cpp + # + # We need to use CACHE variables because aws-sdk-cpp < 12.0.0 uses + # CMP0077 OLD policy. We can use normal variables when we use + # aws-sdk-cpp >= 12.0.0. + set(AWS_SDK_WARNINGS_ARE_ERRORS + OFF + CACHE BOOL "" FORCE) + set(BUILD_DEPS + OFF + CACHE BOOL "" FORCE) + set(BUILD_ONLY + "" + CACHE STRING "" FORCE) + list(APPEND + BUILD_ONLY + config + core + identity-management + s3 + sts + transfer) + set(BUILD_SHARED_LIBS + OFF + CACHE BOOL "" FORCE) + set(ENABLE_TESTING + OFF + CACHE BOOL "" FORCE) + if(NOT WIN32) + set(ZLIB_INCLUDE_DIR + "$" + CACHE STRING "" FORCE) + set(ZLIB_LIBRARY + "$" + CACHE STRING "" FORCE) endif() + if(MINGW AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9") + # This is for RTools 40. We can remove this after we dropped + # support for R < 4.2. schannel.h in RTools 40 is old. - externalproject_add(aws_c_cal_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_CAL_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_CAL_STATIC_LIBRARY} - DEPENDS aws_c_common_ep) - add_dependencies(AWS::aws-c-cal aws_c_cal_ep) - - set(AWS_C_IO_DEPENDS aws_c_common_ep aws_c_cal_ep) - if(TARGET s2n_tls_ep) - list(APPEND AWS_C_IO_DEPENDS s2n_tls_ep) - endif() - externalproject_add(aws_c_io_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_IO_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM}" - PATCH_COMMAND ${AWS_C_IO_PATCH_COMMAND} - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_IO_STATIC_LIBRARY} - DEPENDS ${AWS_C_IO_DEPENDS}) - add_dependencies(AWS::aws-c-io aws_c_io_ep) - - externalproject_add(aws_c_event_stream_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_EVENT_STREAM_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_EVENT_STREAM_STATIC_LIBRARY} - DEPENDS aws_checksums_ep aws_c_io_ep) - add_dependencies(AWS::aws-c-event-stream aws_c_event_stream_ep) - - externalproject_add(aws_c_sdkutils_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_SDKUTILS_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_SDKUTILS_STATIC_LIBRARY} - DEPENDS aws_c_common_ep) - add_dependencies(AWS::aws-c-sdkutils aws_c_sdkutils_ep) - - externalproject_add(aws_c_compression_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_COMPRESSION_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_COMPRESSION_STATIC_LIBRARY} - DEPENDS aws_c_common_ep) - add_dependencies(AWS::aws-c-compression aws_c_compression_ep) - - externalproject_add(aws_c_http_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_HTTP_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_HTTP_STATIC_LIBRARY} - DEPENDS aws_c_io_ep aws_c_compression_ep) - add_dependencies(AWS::aws-c-http aws_c_http_ep) - - externalproject_add(aws_c_mqtt_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_MQTT_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_MQTT_STATIC_LIBRARY} - DEPENDS aws_c_http_ep) - add_dependencies(AWS::aws-c-mqtt aws_c_mqtt_ep) - - externalproject_add(aws_c_auth_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_AUTH_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_AUTH_STATIC_LIBRARY} - DEPENDS aws_c_sdkutils_ep aws_c_cal_ep aws_c_http_ep) - add_dependencies(AWS::aws-c-auth aws_c_auth_ep) - - externalproject_add(aws_c_s3_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_S3_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_S3_STATIC_LIBRARY} - DEPENDS aws_checksums_ep aws_c_auth_ep) - add_dependencies(AWS::aws-c-s3 aws_c_s3_ep) - - externalproject_add(aws_crt_cpp_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_CRT_CPP_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_CRT_CPP_STATIC_LIBRARY} - DEPENDS aws_c_auth_ep - aws_c_cal_ep - aws_c_common_ep - aws_c_event_stream_ep - aws_c_http_ep - aws_c_io_ep - aws_c_mqtt_ep - aws_c_s3_ep - aws_checksums_ep) - add_dependencies(AWS::aws-crt-cpp aws_crt_cpp_ep) - - externalproject_add(awssdk_ep - ${EP_COMMON_OPTIONS} - URL ${AWSSDK_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWSSDK_BUILD_SHA256_CHECKSUM}" - PATCH_COMMAND ${AWSSDK_PATCH_COMMAND} - CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_CPP_SDK_COGNITO_IDENTITY_STATIC_LIBRARY} - ${AWS_CPP_SDK_CORE_STATIC_LIBRARY} - ${AWS_CPP_SDK_IDENTITY_MANAGEMENT_STATIC_LIBRARY} - ${AWS_CPP_SDK_S3_STATIC_LIBRARY} - ${AWS_CPP_SDK_STS_STATIC_LIBRARY} - DEPENDS aws_crt_cpp_ep) - foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) - if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-") - add_dependencies(${_AWSSDK_LIB} awssdk_ep) + # For winhttp.h + # + # See also: + # https://learn.microsoft.com/en-us/windows/win32/winhttp/error-messages + string(APPEND CMAKE_CXX_FLAGS " -DERROR_WINHTTP_UNHANDLED_SCRIPT_TYPE=12176") + string(APPEND CMAKE_CXX_FLAGS " -DERROR_WINHTTP_SCRIPT_EXECUTION_ERROR=12177") + # See also: + # https://learn.microsoft.com/en-us/windows/win32/api/winhttp/ns-winhttp-winhttp_async_result + string(APPEND CMAKE_CXX_FLAGS " -DAPI_GET_PROXY_FOR_URL=6") + # See also: + # https://learn.microsoft.com/en-us/windows/win32/api/winhttp/nc-winhttp-winhttp_status_callback + string(APPEND CMAKE_CXX_FLAGS " -DWINHTTP_CALLBACK_STATUS_CLOSE_COMPLETE=0x02000000") + string(APPEND CMAKE_CXX_FLAGS + " -DWINHTTP_CALLBACK_STATUS_SHUTDOWN_COMPLETE=0x04000000") + # See also: + # https://learn.microsoft.com/en-us/windows/win32/winhttp/option-flags + string(APPEND CMAKE_CXX_FLAGS " -DWINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2=0x00000800") + string(APPEND CMAKE_CXX_FLAGS " -DWINHTTP_NO_CLIENT_CERT_CONTEXT=0") + endif() + + set(AWSSDK_LINK_LIBRARIES) + foreach(AWSSDK_PRODUCT ${AWSSDK_PRODUCTS}) + if("${AWSSDK_PRODUCT}" STREQUAL "s2n-tls") + # Use aws-lc's openssl/*.h not openssl/*.h in system. + set(ADDITIONAL_FLAGS "-DCOMPILE_DEFINITIONS=-I${aws-lc_SOURCE_DIR}/include") endif() - endforeach() - - set(AWSSDK_VENDORED TRUE) - list(APPEND ARROW_BUNDLED_STATIC_LIBS ${AWSSDK_LIBRARIES}) - set(AWSSDK_LINK_LIBRARIES ${AWSSDK_LIBRARIES}) - if(UNIX) - # on Linux and macOS curl seems to be required - set_property(TARGET aws-cpp-sdk-core - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES CURL::libcurl) - set_property(TARGET AWS::aws-c-cal - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES OpenSSL::Crypto OpenSSL::SSL) - if(APPLE) - set_property(TARGET AWS::aws-c-cal - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES "-framework Security") + fetchcontent_makeavailable(${AWSSDK_PRODUCT}) + if(CMAKE_VERSION VERSION_LESS 3.28) + set_property(DIRECTORY ${${AWSSDK_PRODUCT}_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL + TRUE) endif() - if(ZLIB_VENDORED) - set_property(TARGET aws-cpp-sdk-core - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES ZLIB::ZLIB) - add_dependencies(awssdk_ep zlib_ep) + list(PREPEND CMAKE_MODULE_PATH "${${AWSSDK_PRODUCT}_SOURCE_DIR}/cmake") + if(NOT "${AWSSDK_PRODUCT}" STREQUAL "aws-sdk-cpp") + if("${AWSSDK_PRODUCT}" STREQUAL "aws-lc") + # We don't need to link aws-lc. It's used only by s2n-tls. + elseif("${AWSSDK_PRODUCT}" STREQUAL "s2n-tls") + list(PREPEND AWSSDK_LINK_LIBRARIES s2n) + else() + list(PREPEND AWSSDK_LINK_LIBRARIES ${AWSSDK_PRODUCT}) + # This is for find_package(aws-*) in aws-crt-cpp and aws-sdk-cpp. + add_library(AWS::${AWSSDK_PRODUCT} ALIAS ${AWSSDK_PRODUCT}) + endif() endif() - set_property(TARGET AWS::aws-c-io - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES ${CMAKE_DL_LIBS}) - elseif(WIN32) - set_property(TARGET aws-cpp-sdk-core - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES - "winhttp.lib" - "bcrypt.lib" - "wininet.lib" - "userenv.lib" - "version.lib") - set_property(TARGET AWS::aws-c-cal - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES - "bcrypt.lib" - "ncrypt.lib" - "Secur32.lib" - "Shlwapi.lib") - set_property(TARGET AWS::aws-c-io - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES "crypt32.lib") - endif() - - # AWSSDK is static-only build -endmacro() + endforeach() + list(PREPEND + AWSSDK_LINK_LIBRARIES + aws-cpp-sdk-identity-management + aws-cpp-sdk-sts + aws-cpp-sdk-cognito-identity + aws-cpp-sdk-s3 + aws-cpp-sdk-core) + + set(AWSSDK_VENDORED + TRUE + PARENT_SCOPE) + list(APPEND ARROW_BUNDLED_STATIC_LIBS ${AWSSDK_LINK_LIBRARIES}) + set(ARROW_BUNDLED_STATIC_LIBS + ${ARROW_BUNDLED_STATIC_LIBS} + PARENT_SCOPE) + set(AWSSDK_LINK_LIBRARIES + ${AWSSDK_LINK_LIBRARIES} + PARENT_SCOPE) +endfunction() if(ARROW_S3) resolve_dependency(AWSSDK HAVE_ALT TRUE) @@ -5510,16 +5297,6 @@ if(ARROW_S3) endif() endif() endif() - - if(APPLE) - # CoreFoundation's path is hardcoded in the CMake files provided by - # aws-sdk-cpp to use the macOS SDK provided by XCode which makes - # XCode a hard dependency. Command Line Tools is often used instead - # of the full XCode suite, so let the linker to find it. - set_target_properties(AWS::aws-c-common - PROPERTIES INTERFACE_LINK_LIBRARIES - "-pthread;pthread;-framework CoreFoundation") - endif() endif() # ---------------------------------------------------------------------- diff --git a/cpp/cmake_modules/aws-c-common-1208.patch b/cpp/cmake_modules/aws-c-common-1208.patch new file mode 100644 index 00000000000..72a5f70776b --- /dev/null +++ b/cpp/cmake_modules/aws-c-common-1208.patch @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +From 06c435c6765833734c62ac45909b5ab1fc5276a7 Mon Sep 17 00:00:00 2001 +From: Sutou Kouhei +Date: Sat, 7 Jun 2025 20:40:36 +0900 +Subject: [PATCH] Use _WIN32 not _MSC_VER for MinGW + +This was missed in https://github.com/awslabs/aws-c-common/pull/801 +and https://github.com/awslabs/aws-c-common/pull/822 . + +MSYS2 also includes this change: +https://github.com/msys2/MINGW-packages/blob/master/mingw-w64-aws-c-common/001-fix-build-on-mingw-aarch64.patch + +Error message without this: + +https://github.com/ursacomputing/crossbow/actions/runs/15502494926/job/43652596580#step:7:5884 + + cd /D/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-build && /D/a/crossbow/crossbow/sccache/sccache.exe /C/rtools40/mingw32/bin/cc.exe -DARROW_HAVE_RUNTIME_SSE4_2 -DARROW_HAVE_SSE4_2 -DARROW_WITH_TIMING_TESTS -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE -DCJSON_HIDE_SYMBOLS -DINTEL_NO_ITTNOTIFY_API -DPSAPI_VERSION=1 -DWINDOWS_KERNEL_LIB=kernel32 -D_CRT_SECURE_NO_WARNINGS @CMakeFiles/aws-c-common.dir/includes_C.rsp -O3 -DNDEBUG -O2 -ftree-vectorize -std=gnu99 -fvisibility=hidden -Wall -Wstrict-prototypes -pedantic -Wno-long-long -fPIC -D_FILE_OFFSET_BITS=64 -MD -MT _deps/aws-c-common-build/CMakeFiles/aws-c-common.dir/source/allocator.c.obj -MF CMakeFiles/aws-c-common.dir/source/allocator.c.obj.d -o CMakeFiles/aws-c-common.dir/source/allocator.c.obj -c /D/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/source/allocator.c + In file included from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_order.h:71, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_buf.h:9, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/thread.h:8, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/logging.h:11, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/source/allocator.c:8: + D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_order.inl: In function 'aws_hton64': + D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_order.inl:47:23: warning: implicit declaration of function 'htonl'; did you mean 'wtoll'? [-Wimplicit-function-declaration] + return ((uint64_t)htonl(low)) << 32 | htonl(high); + ^~~~~ + wtoll + In file included from C:/rtools40/mingw32/i686-w64-mingw32/include/windows.h:92, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/source/allocator.c:15: + C:/rtools40/mingw32/i686-w64-mingw32/include/winsock.h: At top level: + C:/rtools40/mingw32/i686-w64-mingw32/include/winsock.h:286:37: error: conflicting types for 'htonl' + WINSOCK_API_LINKAGE u_long WSAAPI htonl(u_long hostlong); + ^~~~~ + In file included from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_order.h:71, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_buf.h:9, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/thread.h:8, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/logging.h:11, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/source/allocator.c:8: + D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_order.inl:47:23: note: previous implicit declaration of 'htonl' was here + return ((uint64_t)htonl(low)) << 32 | htonl(high); + ^~~~~ +--- + include/aws/common/byte_order.inl | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/include/aws/common/byte_order.inl b/include/aws/common/byte_order.inl +index 1204be06a1..0abd9cb8cd 100644 +--- a/include/aws/common/byte_order.inl ++++ b/include/aws/common/byte_order.inl +@@ -13,7 +13,7 @@ + # include + #else + # include +-#endif /* _MSC_VER */ ++#endif /* _WIN32 */ + + AWS_EXTERN_C_BEGIN + +@@ -39,7 +39,7 @@ AWS_STATIC_IMPL uint64_t aws_hton64(uint64_t x) { + uint64_t v; + __asm__("bswap %q0" : "=r"(v) : "0"(x)); + return v; +-#elif defined(_MSC_VER) ++#elif defined(_WIN32) + return _byteswap_uint64(x); + #else + uint32_t low = x & UINT32_MAX; diff --git a/cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh b/cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh deleted file mode 100755 index 79b560a4a14..00000000000 --- a/cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eu - -version=$1 - -base_dir="$(dirname "$0")" -output="${base_dir}/AWSSDKVariables.cmake" - -cat <
${output} -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Generated by: -# $ cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh ${version} - -HEADER - -rm -f ${version}.tar.gz -wget https://github.com/aws/aws-sdk-cpp/archive/${version}.tar.gz -base_name=aws-sdk-cpp-${version} -rm -rf ${base_name} -tar xf ${version}.tar.gz - -echo "set(AWSSDK_UNUSED_DIRECTORIES" >> ${output} -find ${base_name} -mindepth 1 -maxdepth 1 -type d | \ - sort | \ - grep -v cmake | \ - grep -v toolchains | \ - grep -v aws-cpp-sdk-cognito-identity | \ - grep -v aws-cpp-sdk-core | \ - grep -v aws-cpp-sdk-config | \ - grep -v aws-cpp-sdk-s3 | \ - grep -v aws-cpp-sdk-transfer | \ - grep -v aws-cpp-sdk-identity-management | \ - grep -v aws-cpp-sdk-sts | \ - sed -E -e "s,^${base_name}/, ,g" >> ${output} -echo ")" >> ${output} - -rm -rf ${base_name} -rm -f ${version}.tar.gz diff --git a/cpp/thirdparty/README.md b/cpp/thirdparty/README.md index 81806810530..32b216861f1 100644 --- a/cpp/thirdparty/README.md +++ b/cpp/thirdparty/README.md @@ -23,3 +23,19 @@ See the "Build Dependency Management" section in the [C++ Developer Documentation][1]. [1]: https://github.com/apache/arrow/blob/main/docs/source/developers/cpp/building.rst + +## Update versions automatically + +There is a convenient script that update versions in `versions.txt` to +the latest version automatically. You can use it like the following: + +```console +cpp/thirdparty/update.rb PRODUCT_PATTERN1 PRODUCT_PATTERN2 ... +``` + +For example, you can update AWS SDK for C++ related products' versions +by the following command line: + +```console +cpp/thirdparty/update.rb "AWS*" "S2N*" +``` diff --git a/cpp/thirdparty/update.rb b/cpp/thirdparty/update.rb new file mode 100755 index 00000000000..77f22de04fb --- /dev/null +++ b/cpp/thirdparty/update.rb @@ -0,0 +1,189 @@ +#!/usr/bin/env ruby +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require "digest/sha2" +require "json" +require "open-uri" +require "optparse" + +option_parser = OptionParser.new +option_parser.banner = + "Usage: #{$0} [options] PRODUCT_PATTERN1 PRODUCT_PATTERN2 ..." +patterns = option_parser.parse!(ARGV) +if patterns.empty? + puts(option_parser) + exit(false) +end + +# Extract product information from the `cpp/thirdparty/versions.txt` +# content. +# +# Output: +# +# { +# "ABSL" => { +# version: "20211102.0", +# checksum: "dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4", +# url_template: "https://github.com/abseil/abseil-cpp/archive/%{version}.tar.gz" +# }, +# "AWS_C_AUTH" => { +# version: "v0.9.0", +# checksum: "aa6e98864fefb95c249c100da4ae7aed36ba13a8a91415791ec6fad20bec0427", +# url_template: "https://github.com/awslabs/aws-c-auth/archive/%{version}.tar.gz", +# }, +# ... +# } +def parse_versions_txt_content(content) + products = {} + content.each_line(chomp: true) do |line| + case line + when /\AARROW_([A-Za-z0-9_-]+)_BUILD_VERSION=(.+?)\z/ + product = Regexp.last_match[1] + version = Regexp.last_match[2] + products[product] = {version: version} + when /\AARROW_([A-Za-z0-9_-]+)_BUILD_SHA256_CHECKSUM=(.+?)\z/ + product = Regexp.last_match[1] + checksum = Regexp.last_match[2] + products[product][:checksum] = checksum + when /\A "ARROW_([A-Za-z0-9_-]+)_URL (?:\S+) (\S+)"\z/ + product = Regexp.last_match[1] + url_template = Regexp.last_match[2] + url_template.gsub!(/\${.+?}/) do |matched| + if matched.end_with?("//./_}") + "%{version_underscore}" + else + "%{version}" + end + end + products[product][:url_template] = url_template + end + end + products +end + +# Update `metadata[:version]` and `metadata[:checksum]` for +# `latest_version`. +# +# This is used by product specific `#update_product_*` such as +# `#update_product_github` and `#update_product_apache`. +def update_product_generic(product, metadata, latest_version) + version = metadata[:version] + url_template = metadata[:url_template] + url = url_template % { + version: latest_version, + version_underscore: latest_version.gsub(".", "_"), + } + $stderr.puts("Updating #{product}: #{version} -> #{latest_version}") + metadata[:version] = latest_version + URI.open(url, "rb") do |response| + metadata[:checksum] = Digest::SHA256.hexdigest(response.read) + end + $stderr.puts(" Checksum: #{metadata[:checksum]}") +end + +# Update metadata to the latest version. This is for products hosted +# on GitHub. +def update_product_github(product, metadata, repository) + version = metadata[:version] + tags_url = "https://api.github.com/repos/#{repository}/tags" + tags = URI.open(tags_url) do |response| + JSON.parse(response.read) + end + latest_tag_name = tags[0]["name"] + if latest_tag_name.start_with?("v") + if metadata[:version].start_with?("v") + latest_version = latest_tag_name + else + latest_version = latest_tag_name[1..-1] + end + else + latest_version = latest_tag_name + end + return if version == latest_version + + update_product_generic(product, metadata, latest_version) +end + +# Update metadata to the latest version. This is for products +# developed by Apache Software Foundation. +def update_product_apache(product, metadata, project) + version = metadata[:version] + version_directory_pattern = metadata[:version_directory_template] % { + version: "(\\d+(?:\\.\\d+)+)", + } + versions = URI.open("https://downloads.apache.org/#{project}/") do |response| + response.read.scan(//).flatten + end + latest_version = versions.last + return if version == latest_version + + update_product_generic(product, metadata, latest_version) +end + +# Update one product to the latest version. +def update_product(product, metadata) + url_template = metadata[:url_template] + if url_template.nil? + $stderr.puts("#{product} isn't supported " + + "because there is no associated URL") + return + end + + case url_template + when /\Ahttps:\/\/github\.com\/((?:[^\/]+)\/(?:[^\/]+))\// + github_repository = Regexp.last_match[1] + update_product_github(product, metadata, github_repository) + when /\Ahttps:\/\/www\.apache\.org\/dyn\/closer\.lua\/ + ((?:[^\/]+))\/((?:[^\/]+))\//x + apache_project = Regexp.last_match[1] + metadata[:version_directory_template] = Regexp.last_match[2] + update_product_apache(product, metadata, apache_project) + else + $stderr.puts("TODO: #{product} isn't supported yet: #{url_template}") + end +end + +# Update `versions.txt` content with `products`. `products` must be +# the same structure as `Hash` returned by +# `#parse_versions_txt_content`. +def update_versions_txt_content!(content, products) + products.each do |product, metadata| + prefix = "ARROW_#{Regexp.escape(product)}" + content.gsub!(/^#{prefix}_BUILD_VERSION=.*$/) do + "ARROW_#{product}_BUILD_VERSION=#{metadata[:version]}" + end + content.gsub!(/^#{prefix}_BUILD_SHA256_CHECKSUM=.*?$/) do + "ARROW_#{product}_BUILD_SHA256_CHECKSUM=#{metadata[:checksum]}" + end + end +end + +versions_txt = File.join(__dir__, "versions.txt") +versions_txt_content = File.read(versions_txt) +products = parse_versions_txt_content(versions_txt_content) +patterns.each do |pattern| + target_products = products.filter do |product, _| + File.fnmatch?(pattern, product) + end + target_products.each do |product, metadata| + update_product(product, metadata) + end +end +update_versions_txt_content!(versions_txt_content, products) +File.write(versions_txt, versions_txt_content) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 1fc53c4d5e6..96ae7c6f035 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -25,34 +25,34 @@ ARROW_ABSL_BUILD_VERSION=20211102.0 ARROW_ABSL_BUILD_SHA256_CHECKSUM=dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4 -ARROW_AWS_C_AUTH_BUILD_VERSION=v0.6.22 -ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=691a6b4418afcd3dc141351b6ad33fccd8e3ff84df0e9e045b42295d284ee14c -ARROW_AWS_C_CAL_BUILD_VERSION=v0.5.20 -ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=acc352359bd06f8597415c366cf4ec4f00d0b0da92d637039a73323dd55b6cd0 -ARROW_AWS_C_COMMON_BUILD_VERSION=v0.8.9 -ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=2f3fbaf7c38eae5a00e2a816d09b81177f93529ae8ba1b82dc8f31407565327a -ARROW_AWS_C_COMPRESSION_BUILD_VERSION=v0.2.16 -ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=044b1dbbca431a07bde8255ef9ec443c300fc60d4c9408d4b862f65e496687f4 -ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.2.18 -ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=310ca617f713bf664e4c7485a3d42c1fb57813abd0107e49790d107def7cde4f -ARROW_AWS_C_HTTP_BUILD_VERSION=v0.7.3 -ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=07e16c6bf5eba6f0dea96b6f55eae312a7c95b736f4d2e4a210000f45d8265ae -ARROW_AWS_C_IO_BUILD_VERSION=v0.13.14 -ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=12b66510c3d9a4f7e9b714e9cfab2a5bf835f8b9ce2f909d20ae2a2128608c71 -ARROW_AWS_C_MQTT_BUILD_VERSION=v0.8.4 -ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=232eeac63e72883d460c686a09b98cdd811d24579affac47c5c3f696f956773f -ARROW_AWS_C_S3_BUILD_VERSION=v0.2.3 -ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=a00b3c9f319cd1c9aa2c3fa15098864df94b066dcba0deaccbb3caa952d902fe -ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.1.6 -ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=8a2951344b2fb541eab1e9ca17c18a7fcbfd2aaff4cdd31d362d1fad96111b91 -ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.13 -ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=0f897686f1963253c5069a0e495b85c31635ba146cd3ac38cc2ea31eaf54694d -ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.18.16 -ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=9e69bc1dc4b50871d1038aa9ff6ddeb4c9b28f7d6b5e5b1b69041ccf50a13483 -ARROW_AWS_LC_BUILD_VERSION=v1.3.0 -ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=ae96a3567161552744fc0cae8b4d68ed88b1ec0f3d3c98700070115356da5a37 -ARROW_AWSSDK_BUILD_VERSION=1.10.55 -ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=2d552fb1a84bef4a9b65e34aa7031851ed2aef5319e02cc6e4cb735c48aa30de +ARROW_AWS_C_AUTH_BUILD_VERSION=v0.9.0 +ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=aa6e98864fefb95c249c100da4ae7aed36ba13a8a91415791ec6fad20bec0427 +ARROW_AWS_C_CAL_BUILD_VERSION=v0.9.2 +ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=f9f3bc6a069e2efe25fcdf73e4d2b16b5608c327d2eb57c8f7a8524e9e1fcad0 +ARROW_AWS_C_COMMON_BUILD_VERSION=v0.12.3 +ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=a4e7ac6c6f840cb6ab56b8ee0bcd94a61c59d68ca42570bca518432da4c94273 +ARROW_AWS_C_COMPRESSION_BUILD_VERSION=v0.3.1 +ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=d89fca17a37de762dc34f332d2da402343078da8dbd2224c46a11a88adddf754 +ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.5.4 +ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=cef8b78e362836d15514110fb43a0a0c7a86b0a210d5fe25fd248a82027a7272 +ARROW_AWS_C_HTTP_BUILD_VERSION=v0.10.2 +ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=048d9d683459ade363fd7cc448c2b6329c78f67a2a0c0cb61c16de4634a2fc6b +ARROW_AWS_C_IO_BUILD_VERSION=v0.19.1 +ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=f2fea0c066924f7fe3c2b1c7b2fa9be640f5b16a6514854226330e63a1faacd0 +ARROW_AWS_C_MQTT_BUILD_VERSION=v0.13.1 +ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=c54d02c1e46f55bae8d5e6f9c4b0d78d84c1c9d9ac16ba8d78c3361edcd8b5bb +ARROW_AWS_C_S3_BUILD_VERSION=v0.8.1 +ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=c8b09780691d2b94e50d101c68f01fa2d1c3debb0ff3aed313d93f0d3c9af663 +ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.2.4 +ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=493cbed4fa57e0d4622fcff044e11305eb4fc12445f32c8861025597939175fc +ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.2.7 +ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=178e8398d98111f29150f7813a70c20ad97ab30be0de02525440355fe84ccb1d +ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.32.8 +ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=db44260452a0296341fb8e7b987e4c328f08f7829b9f1c740fed9c963e081e93 +ARROW_AWS_LC_BUILD_VERSION=v1.52.1 +ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=fe552e3c3522f73afc3c30011745c431c633f7b4e25dcd7b38325f194a7b3b75 +ARROW_AWSSDK_BUILD_VERSION=1.11.587 +ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=b9944ba9905a68d6e53abb4f36ab2b3bd18ac88d8571647bb9f2b8026b76f8cd # Despite the confusing version name this is still the whole Azure SDK for C++ including core, keyvault, storage-common, etc. ARROW_AZURE_SDK_BUILD_VERSION=azure-identity_1.9.0 ARROW_AZURE_SDK_BUILD_SHA256_CHECKSUM=97065bfc971ac8df450853ce805f820f52b59457bd7556510186a1569502e4a1 @@ -105,8 +105,8 @@ ARROW_SNAPPY_BUILD_VERSION=1.2.2 ARROW_SNAPPY_BUILD_SHA256_CHECKSUM=90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc ARROW_SUBSTRAIT_BUILD_VERSION=v0.44.0 ARROW_SUBSTRAIT_BUILD_SHA256_CHECKSUM=f989a862f694e7dbb695925ddb7c4ce06aa6c51aca945105c075139aed7e55a2 -ARROW_S2N_TLS_BUILD_VERSION=v1.3.35 -ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM=9d32b26e6bfcc058d98248bf8fc231537e347395dd89cf62bb432b55c5da990d +ARROW_S2N_TLS_BUILD_VERSION=v1.5.21 +ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM=203d69d6f557f6ab303438ad186fca13fd2c60581b2cca6348a9fbee10d79995 ARROW_THRIFT_BUILD_VERSION=0.20.0 ARROW_THRIFT_BUILD_SHA256_CHECKSUM=b5d8311a779470e1502c027f428a1db542f5c051c8e1280ccd2163fa935ff2d6 ARROW_UTF8PROC_BUILD_VERSION=v2.10.0 diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index 2838372bd17..e90613a6626 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -2043,6 +2043,8 @@ def test_concurrent_s3fs_init(): @pytest.mark.s3 +@pytest.mark.skip(reason="atexit(ensure_s3_finalized) will be called too late " + "with bundled aws-sdk-cpp 1.11.587") @pytest.mark.skipif(running_on_musllinux(), reason="Leaking S3ClientFinalizer causes " "segfault on musl based systems") def test_uwsgi_integration():