ARROW-4479: [Plasma] Add S3 as external store for Plasma

cpp/CMakeLists.txt

@@ -249,6 +249,10 @@ static|shared (default shared)")
     "Build the plasma object store java client"
     OFF)
 
+  option(ARROW_PLASMA_S3
+    "Build S3 external store bindings for the plasma object store"
+    OFF)
+
   #----------------------------------------------------------------------
   # Thirdparty toolchain options
 

cpp/cmake_modules/BuildUtils.cmake

@@ -629,3 +629,12 @@ function(ARROW_ADD_PKG_CONFIG MODULE)
     FILES "${CMAKE_CURRENT_BINARY_DIR}/${MODULE}.pc"
     DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
 endfunction()
+
+function(AWSSDK_TARGETS MODULE_LIST TARGETS)
+  # We will always link against these targets
+  set(awssdk_targets "curl_static" "awssdk_core_static")
+  foreach(MODULE IN LISTS MODULE_LIST)
+    list(APPEND awssdk_targets "awssdk_${MODULE}_static")
+  endforeach()
+  set(${TARGETS} ${awssdk_targets} PARENT_SCOPE)
+endfunction(AWSSDK_TARGETS)

cpp/cmake_modules/FindAWSSDK.cmake

@@ -0,0 +1,95 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tries to find AWSSDK headers and libraries.
+#
+# Usage of this module as follows:
+#
+#  find_package(AWSSDK COMPONENTS service1 service2 ...)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  AWSSDK_HOME - When set, this path is inspected instead of standard library
+#                locations as the root of the AWSSDK installation.
+#                The environment variable AWSSDK_HOME overrides this veriable.
+#
+# This module defines
+#  AWSSDK_INCLUDE_DIR, directory containing headers
+#  AWSSDK_LIBRARY_DIR, directory containing libraries
+#  AWSSDK_STATIC_LIBS, static AWS SDK libraries
+#  AWSSDK_FOUND, whether AWS SDK has been found
+
+if( NOT "${AWSSDK_HOME}" STREQUAL "")
+  file( TO_CMAKE_PATH "${AWSSDK_HOME}" _native_path )
+  list( APPEND _awssdk_roots ${_native_path} )
+elseif ( AWSSDK_HOME )
+  list( APPEND _awssdk_roots ${AWSSDK_HOME} )
+endif()
+
+set(AWSSDK_MODULES ${AWSSDK_FIND_COMPONENTS})
+set(AWSSDK_STATIC_LIBS)
+
+find_path(AWSSDK_INCLUDE_DIR NAMES aws/core/Aws.h
+        PATHS ${_awssdk_roots}
+        NO_DEFAULT_PATH
+        PATH_SUFFIXES "include")
+
+foreach(MODULE ${AWSSDK_MODULES})
+  string(TOUPPER ${MODULE} MODULE_UPPERCASE)
+  find_library( AWSSDK_STATIC_${MODULE_UPPERCASE}_LIB NAMES ${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-${MODULE}${CMAKE_STATIC_LIBRARY_SUFFIX}
+          PATHS ${_awssdk_roots}
+          NO_DEFAULT_PATH
+          PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" )
+  # Fail if any module library is missing
+  if (NOT AWSSDK_STATIC_${MODULE_UPPERCASE}_LIB)
+    set(LIBRARY_NOT_FOUND "YES")
+  else()
+    list(APPEND AWSSDK_STATIC_LIBS ${AWSSDK_STATIC_${MODULE_UPPERCASE}_LIB})
+  endif()
+endforeach()
+
+if (AWSSDK_INCLUDE_DIR AND NOT LIBRARY_NOT_FOUND)
+  set(AWSSDK_FOUND TRUE)
+  get_filename_component(AWSSDK_LIBRARY_DIR ${AWSSDK_STATIC_CORE_LIB} PATH)
+else ()
+  set(AWSSDK_FOUND FALSE)
+endif ()
+
+if (AWSSDK_FOUND)
+  if (NOT AWSSDK_FIND_QUIETLY)
+    message(STATUS "Found the AWSSDK libraries: ${AWSSDK_STATIC_LIBS}")
+  endif ()
+else ()
+  set(AWSSDK_STATIC_LIBS)
+  if (NOT AWSSDK_FIND_QUIETLY)
+    set(AWSSDK_ERR_MSG "Could not find the AWSSDK library. Looked")
+    if ( _awssdk_roots )
+      set(AWSSDK_ERR_MSG "${AWSSDK_ERR_MSG} in ${_awssdk_roots}.")
+    else ()
+      set(AWSSDK_ERR_MSG "${AWSSDK_ERR_MSG} in system search paths.")
+    endif ()
+    if (AWSSDK_FIND_REQUIRED)
+      message(FATAL_ERROR "${AWSSDK_ERR_MSG}")
+    else (AWSSDK_FIND_REQUIRED)
+      message(STATUS "${AWSSDK_ERR_MSG}")
+    endif (AWSSDK_FIND_REQUIRED)
+  endif ()
+endif ()
+
+mark_as_advanced(
+        AWSSDK_INCLUDE_DIR
+        AWSSDK_LIBRARY_DIR
+        AWSSDK_STATIC_LIBS
+        AWSSDK_FOUND
+)

cpp/cmake_modules/FindCURL.cmake

@@ -0,0 +1,102 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tries to find CURL headers and libraries.
+#
+# Usage of this module as follows:
+#
+#  find_package(CURL)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  CURL_HOME - When set, this path is inspected instead of standard library
+#             locations as the root of the CURL installation.
+#             The environment variable CURL_HOME overrides this variable.
+#
+# - Find CURL (curl/curl.h, libcurl.a)
+# This module defines
+#  CURL_INCLUDE_DIR, directory containing headers
+#  CURL_STATIC_LIB, path to curl's static library
+#  CURL_FOUND, whether curl has been found
+
+if( NOT "${CURL_HOME}" STREQUAL "")
+  file( TO_CMAKE_PATH "${CURL_HOME}" _native_path )
+  list( APPEND _curl_roots ${_native_path} )
+elseif ( CURL_HOME )
+  list( APPEND _curl_roots ${CURL_HOME} )
+endif()
+
+if (MSVC)
+  # curl uses curl.lib for Windows.
+  set(CURL_LIB_NAME curllib)
+else ()
+  # curl uses libcurl.a for non Windows.
+  set(CURL_LIB_NAME
+          ${CMAKE_SHARED_LIBRARY_PREFIX}curl${CMAKE_STATIC_LIBRARY_SUFFIX})
+endif ()
+
+# Try the parameterized roots, if they exist
+if (_curl_roots)
+  find_path(CURL_INCLUDE_DIR NAMES curl/curl.h
+          PATHS ${_curl_roots} NO_DEFAULT_PATH
+          PATH_SUFFIXES "include")
+  find_library(CURL_STATIC_LIB
+          NAMES ${CURL_LIB_NAME}
+          PATHS ${_curl_roots} NO_DEFAULT_PATH
+          PATH_SUFFIXES "lib")
+else ()
+  pkg_check_modules(PKG_CURL libcurl)
+  if (PKG_CURL_FOUND)
+    set(CURL_INCLUDE_DIR ${PKG_CURL_INCLUDEDIR})
+    find_library(CURL_STATIC_LIB
+            NAMES ${CURL_LIB_NAME}
+            PATHS ${PKG_CURL_LIBDIR} NO_DEFAULT_PATH)
+  else ()
+    find_path(CURL_INCLUDE_DIR NAMES curl/curl.h)
+    find_library(CURL_STATIC_LIB NAMES ${CURL_LIB_NAME})
+  endif ()
+endif ()
+
+if (CURL_INCLUDE_DIR AND CURL_STATIC_LIB)
+  set(CURL_FOUND TRUE)
+else ()
+  set(CURL_FOUND FALSE)
+endif ()
+
+if (CURL_FOUND)
+  if (NOT CURL_FIND_QUIETLY)
+    if (CURL_STATIC_LIB)
+      message(STATUS "Found the CURL static library: ${CURL_STATIC_LIB}")
+    endif ()
+  endif ()
+else ()
+  if (NOT CURL_FIND_QUIETLY)
+    set(CURL_ERR_MSG "Could not find the CURL library. Looked in")
+    if ( _curl_roots )
+      set(CURL_ERR_MSG "${CURL_ERR_MSG} ${_curl_roots}.")
+    else ()
+      set(CURL_ERR_MSG "${CURL_ERR_MSG} system search paths.")
+    endif ()
+    if (CURL_FIND_REQUIRED)
+      message(FATAL_ERROR "${CURL_ERR_MSG}")
+    else (CURL_FIND_REQUIRED)
+      message(STATUS "${CURL_ERR_MSG}")
+    endif (CURL_FIND_REQUIRED)
+  endif ()
+endif ()
+
+mark_as_advanced(
+        CURL_INCLUDE_DIR
+        CURL_STATIC_LIB
+)