ARROW-14421: [C++] Implement Flight SQL

ci/docker/conda-cpp.dockerfile

@@ -41,6 +41,7 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_DATASET=ON \
     ARROW_DEPENDENCY_SOURCE=CONDA \
     ARROW_FLIGHT=ON \
+    ARROW_FLIGHT_SQL=ON \
     ARROW_GANDIVA=ON \
     ARROW_HOME=$CONDA_PREFIX \
     ARROW_ORC=ON \

ci/scripts/cpp_build.sh

@@ -70,6 +70,7 @@ cmake \
   -DARROW_EXTRA_ERROR_CONTEXT=${ARROW_EXTRA_ERROR_CONTEXT:-OFF} \
   -DARROW_FILESYSTEM=${ARROW_FILESYSTEM:-ON} \
   -DARROW_FLIGHT=${ARROW_FLIGHT:-OFF} \
+  -DARROW_FLIGHT_SQL=${ARROW_FLIGHT_SQL:-OFF} \
   -DARROW_FUZZING=${ARROW_FUZZING:-OFF} \
   -DARROW_GANDIVA_JAVA=${ARROW_GANDIVA_JAVA:-OFF} \
   -DARROW_GANDIVA_PC_CXX_FLAGS=${ARROW_GANDIVA_PC_CXX_FLAGS:-} \

cpp/CMakeLists.txt

@@ -334,6 +334,14 @@ if(ARROW_GANDIVA)
   set(ARROW_WITH_RE2 ON)
 endif()
 
+if(ARROW_BUILD_INTEGRATION AND ARROW_FLIGHT)
+  set(ARROW_FLIGHT_SQL ON)
+endif()
+
+if(ARROW_FLIGHT_SQL)
+  set(ARROW_FLIGHT ON)
+endif()
+
 if(ARROW_CUDA
    OR ARROW_FLIGHT
    OR ARROW_PARQUET

cpp/cmake_modules/DefineOptions.cmake

@@ -226,6 +226,8 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   define_option(ARROW_FLIGHT
                 "Build the Arrow Flight RPC System (requires GRPC, Protocol Buffers)" OFF)
 
+  define_option(ARROW_FLIGHT_SQL "Build the Arrow Flight SQL extension" OFF)
+
   define_option(ARROW_GANDIVA "Build the Gandiva libraries" OFF)
 
   define_option(ARROW_GCS

cpp/cmake_modules/FindArrowFlightSql.cmake

@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# - Find Arrow Flight SQL
+#
+# This module requires Arrow from which it uses
+#  arrow_find_package()
+#
+# This module defines
+#  ARROW_FLIGHT_SQL_FOUND, whether Flight has been found
+#  ARROW_FLIGHT_SQL_IMPORT_LIB,
+#    path to libarrow_flight's import library (Windows only)
+#  ARROW_FLIGHT_SQL_INCLUDE_DIR, directory containing headers
+#  ARROW_FLIGHT_SQL_LIBS, deprecated. Use ARROW_FLIGHT_SQL_LIB_DIR instead
+#  ARROW_FLIGHT_SQL_LIB_DIR, directory containing Flight libraries
+#  ARROW_FLIGHT_SQL_SHARED_IMP_LIB, deprecated. Use ARROW_FLIGHT_SQL_IMPORT_LIB instead
+#  ARROW_FLIGHT_SQL_SHARED_LIB, path to libarrow_flight's shared library
+#  ARROW_FLIGHT_SQL_STATIC_LIB, path to libarrow_flight.a
+
+if(DEFINED ARROW_FLIGHT_SQL_FOUND)
+  return()
+endif()
+
+set(find_package_arguments)
+if(${CMAKE_FIND_PACKAGE_NAME}_FIND_VERSION)
+  list(APPEND find_package_arguments "${${CMAKE_FIND_PACKAGE_NAME}_FIND_VERSION}")
+endif()
+if(${CMAKE_FIND_PACKAGE_NAME}_FIND_REQUIRED)
+  list(APPEND find_package_arguments REQUIRED)
+endif()
+if(${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY)
+  list(APPEND find_package_arguments QUIET)
+endif()
+find_package(Arrow ${find_package_arguments})
+
+if(ARROW_FOUND)
+  arrow_find_package(ARROW_FLIGHT_SQL
+                     "${ARROW_HOME}"
+                     arrow_flight_sql
+                     arrow/flight/sql/api.h
+                     ArrowFlightSql
+                     arrow-flight-sql)
+  if(NOT ARROW_FLIGHT_SQL_VERSION)
+    set(ARROW_FLIGHT_SQL_VERSION "${ARROW_VERSION}")
+  endif()
+endif()
+
+if("${ARROW_FLIGHT_SQL_VERSION}" VERSION_EQUAL "${ARROW_VERSION}")
+  set(ARROW_FLIGHT_SQL_VERSION_MATCH TRUE)
+else()
+  set(ARROW_FLIGHT_SQL_VERSION_MATCH FALSE)
+endif()
+
+mark_as_advanced(ARROW_FLIGHT_SQL_IMPORT_LIB
+                 ARROW_FLIGHT_SQL_INCLUDE_DIR
+                 ARROW_FLIGHT_SQL_LIBS
+                 ARROW_FLIGHT_SQL_LIB_DIR
+                 ARROW_FLIGHT_SQL_SHARED_IMP_LIB
+                 ARROW_FLIGHT_SQL_SHARED_LIB
+                 ARROW_FLIGHT_SQL_STATIC_LIB
+                 ARROW_FLIGHT_SQL_VERSION
+                 ARROW_FLIGHT_SQL_VERSION_MATCH)
+
+find_package_handle_standard_args(
+  ArrowFlightSql
+  REQUIRED_VARS ARROW_FLIGHT_SQL_INCLUDE_DIR ARROW_FLIGHT_SQL_LIB_DIR
+                ARROW_FLIGHT_SQL_VERSION_MATCH
+  VERSION_VAR ARROW_FLIGHT_SQL_VERSION)
+set(ARROW_FLIGHT_SQL_FOUND ${ArrowFlightSql_FOUND})
+
+if(ArrowFlightSql_FOUND AND NOT ArrowFlightSql_FIND_QUIETLY)
+  message(STATUS "Found the Arrow Flight SQL by ${ARROW_FLIGHT_SQL_FIND_APPROACH}")
+  message(STATUS "Found the Arrow Flight SQL shared library: ${ARROW_FLIGHT_SQL_SHARED_LIB}"
+  )
+  message(STATUS "Found the Arrow Flight SQL import library: ${ARROW_FLIGHT_SQL_IMPORT_LIB}"
+  )
+  message(STATUS "Found the Arrow Flight SQL static library: ${ARROW_FLIGHT_SQL_STATIC_LIB}"
+  )
+endif()

cpp/cmake_modules/FindSQLite3Alt.cmake

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Once done this will define
+# - FindSQLite3Alt
+#
+# This module will set the following variables if found:
+#  SQLite3_INCLUDE_DIRS  - SQLite3 include dir.
+#  SQLite3_LIBRARIES     - List of libraries when using SQLite3.
+#  SQLite3_FOUND         - True if SQLite3 found.
+#
+# Usage of this module as follows:
+# find_package(SQLite3Alt)
+
+find_path(SQLite3_INCLUDE_DIR sqlite3.h)
+find_library(SQLite3_LIBRARY NAMES sqlite3)
+
+# handle the QUIETLY and REQUIRED arguments and set SQLite3_FOUND to TRUE if
+# all listed variables are TRUE
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(SQLite3Alt REQUIRED_VARS SQLite3_LIBRARY
+                                                           SQLite3_INCLUDE_DIR)
+
+mark_as_advanced(SQLite3_LIBRARY SQLite3_INCLUDE_DIR)
+
+if(SQLite3Alt_FOUND)
+  set(SQLite3_INCLUDE_DIRS ${SQLite3_INCLUDE_DIR})
+  set(SQLite3_LIBRARIES ${SQLite3_LIBRARY})
+endif()

cpp/src/arrow/CMakeLists.txt

@@ -732,6 +732,16 @@ if(ARROW_FLIGHT)
   add_subdirectory(flight)
 endif()
 
+if(ARROW_FLIGHT_SQL)
+  add_subdirectory(flight/sql)
+endif()
+
+if(ARROW_FLIGHT
+   AND ARROW_FLIGHT_SQL
+   AND ARROW_BUILD_INTEGRATION)
+  add_subdirectory(flight/integration_tests)
+endif()
+
 if(ARROW_HIVESERVER2)
   add_subdirectory(dbi/hiveserver2)
 endif()

cpp/src/arrow/flight/CMakeLists.txt

@@ -25,7 +25,23 @@ if(WIN32)
   list(APPEND ARROW_FLIGHT_LINK_LIBS ws2_32.lib)
 endif()
 
-if(ARROW_TEST_LINKAGE STREQUAL "static")
+set(ARROW_FLIGHT_TEST_LINKAGE
+    "${ARROW_TEST_LINKAGE}"
+    PARENT_SCOPE)
+if(Protobuf_USE_STATIC_LIBS)
+  message(STATUS "Linking Arrow Flight tests statically due to static Protobuf")
+  set(ARROW_FLIGHT_TEST_LINKAGE
+      "static"
+      PARENT_SCOPE)
+endif()
+if(NOT ARROW_GRPC_USE_SHARED)
+  message(STATUS "Linking Arrow Flight tests statically due to static gRPC")
+  set(ARROW_FLIGHT_TEST_LINKAGE
+      "static"
+      PARENT_SCOPE)
+endif()
+
+if(ARROW_FLIGHT_TEST_LINKAGE STREQUAL "static")
   set(ARROW_FLIGHT_TEST_LINK_LIBS
       arrow_flight_static arrow_flight_testing_static ${ARROW_FLIGHT_STATIC_LINK_LIBS}
       ${ARROW_TEST_LINK_LIBS})
@@ -186,7 +202,6 @@ if(ARROW_TESTING)
                 OUTPUTS
                 ARROW_FLIGHT_TESTING_LIBRARIES
                 SOURCES
-                test_integration.cc
                 test_util.cc
                 DEPENDENCIES
                 GTest::gtest
@@ -230,21 +245,6 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
   add_dependencies(arrow_flight flight-test-server)
 endif()
 
-if(ARROW_BUILD_INTEGRATION)
-  add_executable(flight-test-integration-server test_integration_server.cc)
-  target_link_libraries(flight-test-integration-server ${ARROW_FLIGHT_TEST_LINK_LIBS}
-                        ${GFLAGS_LIBRARIES} GTest::gtest)
-
-  add_executable(flight-test-integration-client test_integration_client.cc)
-  target_link_libraries(flight-test-integration-client ${ARROW_FLIGHT_TEST_LINK_LIBS}
-                        ${GFLAGS_LIBRARIES} GTest::gtest)
-
-  add_dependencies(arrow_flight flight-test-integration-client
-                   flight-test-integration-server)
-  add_dependencies(arrow-integration flight-test-integration-client
-                   flight-test-integration-server)
-endif()
-
 if(ARROW_BUILD_BENCHMARKS)
   # Perf server for benchmarks
   set(PERF_PROTO_GENERATED_FILES "${CMAKE_CURRENT_BINARY_DIR}/perf.pb.cc"

cpp/src/arrow/flight/integration_tests/CMakeLists.txt

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+add_custom_target(arrow_flight_integration_tests)
+
+if(ARROW_FLIGHT_TEST_LINKAGE STREQUAL "static")
+  set(ARROW_FLIGHT_TEST_LINK_LIBS
+      arrow_flight_static
+      arrow_flight_testing_static
+      arrow_flight_sql_static
+      ${ARROW_FLIGHT_STATIC_LINK_LIBS}
+      ${ARROW_TEST_LINK_LIBS})
+else()
+  set(ARROW_FLIGHT_TEST_LINK_LIBS
+      arrow_flight_shared
+      arrow_flight_testing_shared
+      arrow_flight_sql_shared
+      ${ARROW_TEST_LINK_LIBS}
+      ${ARROW_FLIGHT_SQL_TEST_LINK_LIBS})
+endif()
+
+add_executable(flight-test-integration-server test_integration_server.cc
+                                              test_integration.cc)
+target_link_libraries(flight-test-integration-server ${ARROW_FLIGHT_TEST_LINK_LIBS}
+                      ${GFLAGS_LIBRARIES} GTest::gtest)
+
+add_executable(flight-test-integration-client test_integration_client.cc
+                                              test_integration.cc)
+target_link_libraries(flight-test-integration-client ${ARROW_FLIGHT_TEST_LINK_LIBS}
+                      ${GFLAGS_LIBRARIES} GTest::gtest)
+
+add_dependencies(arrow-integration flight-test-integration-client
+                 flight-test-integration-server)