GH-33854: [MATLAB] Add basic libmexclass integration code to MATLAB interface

ci/scripts/matlab_build.sh

@@ -25,6 +25,13 @@ source_dir=${base_dir}/matlab
 build_dir=${base_dir}/matlab/build
 install_dir=${base_dir}/matlab/install
 
-cmake -S ${source_dir} -B ${build_dir} -G Ninja -D MATLAB_BUILD_TESTS=ON -D CMAKE_INSTALL_PREFIX=${install_dir} -D MATLAB_ADD_INSTALL_DIR_TO_SEARCH_PATH=OFF
+cmake \
+  -S ${source_dir} \
+  -B ${build_dir} \
+  -G Ninja \
+  -D MATLAB_ARROW_INTERFACE=ON \
+  -D MATLAB_BUILD_TESTS=ON \
+  -D CMAKE_INSTALL_PREFIX=${install_dir} \
+  -D MATLAB_ADD_INSTALL_DIR_TO_SEARCH_PATH=OFF
 cmake --build ${build_dir} --config Release --target install
 ctest --test-dir ${build_dir}

matlab/CMakeLists.txt

@@ -32,39 +32,66 @@ function(build_arrow)
     message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
   endif()
 
-  # If Arrow needs to be built, the default location will be within the build tree.
   set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix")
-
-  if(WIN32)
-    # The shared library is located in the "bin" directory.
-    set(ARROW_SHARED_LIBRARY_DIR "${ARROW_PREFIX}/bin")
-
-    # Imported libraries are used
-    set(ARROW_IMPORT_LIB_FILENAME
-        "${CMAKE_IMPORT_LIBRARY_PREFIX}arrow${CMAKE_IMPORT_LIBRARY_SUFFIX}")
-    set(ARROW_IMPORT_LIB "${ARROW_PREFIX}/lib/${ARROW_IMPORT_LIB_FILENAME}")
-  else()
-    # The shared library is located in the "lib" directory.
-    set(ARROW_SHARED_LIBRARY_DIR "${ARROW_PREFIX}/lib")
-  endif()
-
-  set(ARROW_SHARED_LIB_FILENAME
-      "${CMAKE_SHARED_LIBRARY_PREFIX}arrow${CMAKE_SHARED_LIBRARY_SUFFIX}")
-  set(ARROW_SHARED_LIB "${ARROW_SHARED_LIBRARY_DIR}/${ARROW_SHARED_LIB_FILENAME}")
-
   set(ARROW_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-build")
   set(ARROW_CMAKE_ARGS "-DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX}"
                        "-DCMAKE_INSTALL_LIBDIR=lib" "-DARROW_BUILD_STATIC=OFF")
-  set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")
 
-  # The output libraries need to be guaranteed to be available for linking the test
-  # executables.
-  if(WIN32)
-    # On Windows, add the Arrow link library as a BUILD_BYPRODUCTS for arrow_ep.
-    set(ARROW_BUILD_BYPRODUCTS "${ARROW_IMPORT_LIB}")
+  if(Arrow_FOUND
+     AND NOT GTest_FOUND
+     AND ARG_BUILD_GTEST)
+    # If find_package has already found a valid Arrow installation, then
+    # we don't want to link against the Arrow libraries that will be built
+    # from source.
+    #
+    # However, we still need to create a library target to trigger building
+    # of the arrow_ep target, which will ultimately build the bundled
+    # GoogleTest binaries.
+    add_library(arrow_shared_for_gtest SHARED IMPORTED)
+    set(ARROW_LIBRARY_TARGET arrow_shared_for_gtest)
   else()
-    # On Linux and macOS, add the Arrow shared library as a BUILD_BYPRODUCTS for arrow_ep.
-    set(ARROW_BUILD_BYPRODUCTS "${ARROW_SHARED_LIB}")
+    add_library(arrow_shared SHARED IMPORTED)
+    set(ARROW_LIBRARY_TARGET arrow_shared)
+
+    # Set the runtime shared library (.dll, .so, or .dylib)
+    if(WIN32)
+      # The shared library (i.e. .dll) is located in the "bin" directory.
+      set(ARROW_SHARED_LIBRARY_DIR "${ARROW_PREFIX}/bin")
+    else()
+      # The shared library (i.e. .so or .dylib) is located in the "lib" directory.
+      set(ARROW_SHARED_LIBRARY_DIR "${ARROW_PREFIX}/lib")
+    endif()
+
+    set(ARROW_SHARED_LIB_FILENAME
+        "${CMAKE_SHARED_LIBRARY_PREFIX}arrow${CMAKE_SHARED_LIBRARY_SUFFIX}")
+    set(ARROW_SHARED_LIB "${ARROW_SHARED_LIBRARY_DIR}/${ARROW_SHARED_LIB_FILENAME}")
+
+    set_target_properties(arrow_shared PROPERTIES IMPORTED_LOCATION ${ARROW_SHARED_LIB})
+
+    # Set the link-time import library (.lib)
+    if(WIN32)
+      # The import library (i.e. .lib) is located in the "lib" directory.
+      set(ARROW_IMPORT_LIB_FILENAME
+          "${CMAKE_IMPORT_LIBRARY_PREFIX}arrow${CMAKE_IMPORT_LIBRARY_SUFFIX}")
+      set(ARROW_IMPORT_LIB "${ARROW_PREFIX}/lib/${ARROW_IMPORT_LIB_FILENAME}")
+
+      set_target_properties(arrow_shared PROPERTIES IMPORTED_IMPLIB ${ARROW_IMPORT_LIB})
+    endif()
+
+    # Set the include directories
+    set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")
+    file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}")
+    set_target_properties(arrow_shared PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+                                                  ${ARROW_INCLUDE_DIR})
+
+    # Set the build byproducts for the ExternalProject build
+    # The appropriate libraries need to be guaranteed to be available when linking the test
+    # executables.
+    if(WIN32)
+      set(ARROW_BUILD_BYPRODUCTS "${ARROW_IMPORT_LIB}")
+    else()
+      set(ARROW_BUILD_BYPRODUCTS "${ARROW_SHARED_LIB}")
+    endif()
   endif()
 
   # Building the Arrow C++ libraries and bundled GoogleTest binaries requires ExternalProject.
@@ -80,29 +107,6 @@ function(build_arrow)
                       CMAKE_ARGS "${ARROW_CMAKE_ARGS}"
                       BUILD_BYPRODUCTS "${ARROW_BUILD_BYPRODUCTS}")
 
-  set(ARROW_LIBRARY_TARGET arrow_shared)
-
-  # If find_package has already found a valid Arrow installation, then
-  # we don't want to link against the newly built arrow_shared library.
-  # However, we still need to create a library target to trigger building
-  # of the arrow_ep target, which will ultimately build the bundled
-  # GoogleTest binaries.
-  if(Arrow_FOUND)
-    set(ARROW_LIBRARY_TARGET arrow_shared_for_gtest)
-  endif()
-
-  file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}")
-  add_library(${ARROW_LIBRARY_TARGET} SHARED IMPORTED)
-  set_target_properties(${ARROW_LIBRARY_TARGET}
-                        PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${ARROW_INCLUDE_DIR}
-                                   IMPORTED_LOCATION ${ARROW_SHARED_LIB})
-  if(WIN32)
-    # On Windows, IMPORTED_IMPLIB is set to the location of arrow.lib, which is
-    # for linking arrow_matlab against the Arrow C++ library.
-    set_target_properties(${ARROW_LIBRARY_TARGET} PROPERTIES IMPORTED_IMPLIB
-                                                             ${ARROW_IMPORT_LIB})
-  endif()
-
   add_dependencies(${ARROW_LIBRARY_TARGET} arrow_ep)
 
   if(ARG_BUILD_GTEST)
@@ -202,13 +206,16 @@ endif()
 
 option(MATLAB_BUILD_TESTS "Build the C++ tests for the MATLAB interface" OFF)
 
-# Grab CMAKE Modules from the CPP interface.
-set(CPP_CMAKE_MODULES "${CMAKE_SOURCE_DIR}/../cpp/cmake_modules")
-if(EXISTS "${CPP_CMAKE_MODULES}")
-  set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CPP_CMAKE_MODULES})
-endif()
+# Add tools/cmake directory to the CMAKE_MODULE_PATH.
+list(PREPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/tools/cmake)
 
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/cmake_modules)
+# Configure find_package to look for arrow-config.cmake in Config mode
+# underneath the Arrow installation directory hierarchy specified
+# by ARROW_HOME or Arrow_ROOT.
+# NOTE: ARROW_HOME is supported for backwards compatibility.
+if(ARROW_HOME AND NOT Arrow_ROOT)
+  set(Arrow_ROOT "${ARROW_HOME}")
+endif()
 
 # Multi-Configuration generators (e.g. Visual Studio or XCode) place their build artifacts
 # in a subdirectory named ${CMAKE_BUILD_TYPE} by default, where ${CMAKE_BUILD_TYPE} varies
@@ -228,7 +235,7 @@ if(MATLAB_BUILD_TESTS)
   if(NOT GTest_FOUND)
     # find_package(Arrow) supports custom ARROW_HOME as well as package
     # managers.
-    find_package(Arrow)
+    find_package(Arrow QUIET)
     # Trigger an automatic build of the Arrow C++ libraries and bundled
     # GoogleTest binaries. If a valid Arrow installation was not already
     # found by find_package, then build_arrow will use the Arrow
@@ -258,15 +265,15 @@ if(MATLAB_BUILD_TESTS)
                                        "${GTEST_MAIN_SHARED_LIBRARY_LIB}")
     endif()
 
-    find_package(Arrow)
+    find_package(Arrow QUIET)
     if(NOT Arrow_FOUND)
       # Trigger an automatic build of the Arrow C++ libraries.
       build_arrow()
     endif()
   endif()
 
 else()
-  find_package(Arrow)
+  find_package(Arrow QUIET)
   if(NOT Arrow_FOUND)
     build_arrow()
   endif()
@@ -317,6 +324,51 @@ matlab_add_mex(R2018a
 
 target_include_directories(mexcall PRIVATE ${CPP_SOURCE_DIR})
 
+# ARROW_SHARED_LIB
+# On Windows, this will be ARROW_HOME/bin/arrow.dll and on Linux and macOS, it is the arrow.so/dylib in the newly built arrow_shared library.
+if(NOT Arrow_FOUND)
+  message(STATUS "ARROW_SHARED_LIB will be set using IMPORTED_LOCATION value when building."
+  )
+  get_target_property(ARROW_SHARED_LIB arrow_shared IMPORTED_LOCATION)
+else()
+  # If not building Arrow, ARROW_SHARED_LIB derived from ARROW_PREFIX set to the ARROW_HOME specified with cmake would be non-empty.
+  message(STATUS "ARROW_SHARED_LIB: ${ARROW_SHARED_LIB}")
+endif()
+
+# ARROW_LINK_LIB
+# On Windows, we use the arrow.lib for linking arrow_matlab against the Arrow C++ library.
+# The location of arrow.lib is previously saved in IMPORTED_IMPLIB.
+if(WIN32)
+  # If not building Arrow, IMPORTED_IMPLIB will be empty.
+  # Then set ARROW_LINK_LIB to ARROW_IMPORT_LIB which would have been derived from ARROW_PREFIX set to the ARROW_HOME specified with cmake. This will avoid the ARROW_LINK_LIB set to NOTFOUND error.
+  # The ARROW_IMPORT_LIB should be ARROW_HOME/lib/arrow.lib on Windows.
+  if(NOT Arrow_FOUND)
+    message(STATUS "ARROW_LINK_LIB will be set using IMPORTED_IMPLIB value when building."
+    )
+    get_target_property(ARROW_LINK_LIB arrow_shared IMPORTED_IMPLIB)
+  else()
+    set(ARROW_LINK_LIB "${ARROW_IMPORT_LIB}")
+    message(STATUS "Setting ARROW_LINK_LIB to ARROW_IMPORT_LIB: ${ARROW_IMPORT_LIB}, which is derived from the ARROW_HOME provided."
+    )
+  endif()
+else()
+  # On Linux and macOS, it is the arrow.so/dylib in the newly built arrow_shared library used for linking.
+  # On Unix, this is the same as ARROW_SHARED_LIB.
+  message(STATUS "Setting ARROW_LINK_LIB to ARROW_SHARED_LIB as they are same on Unix.")
+  set(ARROW_LINK_LIB "${ARROW_SHARED_LIB}")
+endif()
+
+# ARROW_INCLUDE_DIR should be set so that header files in the include directory can be found correctly.
+# The value of ARROW_INCLUDE_DIR should be ARROW_HOME/include on all platforms.
+if(NOT Arrow_FOUND)
+  message(STATUS "ARROW_INCLUDE_DIR will be set using INTERFACE_INCLUDE_DIRECTORIES value when building."
+  )
+  get_target_property(ARROW_INCLUDE_DIR arrow_shared INTERFACE_INCLUDE_DIRECTORIES)
+else()
+  # If not building Arrow, ARROW_INCLUDE_DIR derived from ARROW_PREFIX set to the ARROW_HOME specified with cmake would be non-empty.
+  message(STATUS "ARROW_INCLUDE_DIR: ${ARROW_INCLUDE_DIR}")
+endif()
+
 # ##############################################################################
 # C++ Tests
 # ##############################################################################
@@ -396,9 +448,13 @@ endif()
 # Create a subdirectory at CMAKE_INSTALL_PREFIX to install the interface.
 set(CMAKE_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/arrow_matlab")
 
+if(MATLAB_ARROW_INTERFACE)
+  include(BuildMatlabArrowInterface)
+endif()
+
 # Create a package hierarchy at CMAKE_INSTALL_PREFIX to install the mex function
 # and dependencies.
-set(CMAKE_PACKAGED_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/arrow_matlab/+arrow/+cpp")
+set(CMAKE_PACKAGED_INSTALL_DIR "${CMAKE_INSTALL_DIR}/+arrow/+cpp")
 
 # Install MATLAB source files.
 # On macOS, exclude '.DS_Store' files in the source tree from installation.
@@ -413,16 +469,9 @@ install(TARGETS arrow_matlab mexcall
         RUNTIME DESTINATION ${CMAKE_PACKAGED_INSTALL_DIR}
         LIBRARY DESTINATION ${CMAKE_PACKAGED_INSTALL_DIR})
 
-get_target_property(ARROW_SHARED_LIB arrow_shared IMPORTED_LOCATION)
 get_filename_component(ARROW_SHARED_LIB_DIR ${ARROW_SHARED_LIB} DIRECTORY)
 get_filename_component(ARROW_SHARED_LIB_FILENAME ${ARROW_SHARED_LIB} NAME_WE)
 
-if(WIN32)
-  # On Windows, arrow.dll must be installed to to CMAKE_PACKAGED_INSTALL_DIR regardless of whether
-  # Arrow_FOUND is true or false.
-  install(FILES ${ARROW_SHARED_LIB} DESTINATION "${CMAKE_PACKAGED_INSTALL_DIR}")
-endif()
-
 # On macOS, use the RPATH values below for runtime dependency resolution. This enables
 # relocation of the installation directory.
 if(APPLE)

matlab/src/cpp/arrow/matlab/array/proxy/float64_array.cc

@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "float64_array.h"
+
+namespace arrow::matlab::array::proxy {
+void Float64Array::Print(libmexclass::proxy::method::Context& context) {
+    // TODO: Return an MDA string representation of the Arrow array. 
+    std::cout << array->ToString() << std::endl;
+}
+} // namespace arrow::matlab::array::proxy

matlab/src/cpp/arrow/matlab/array/proxy/float64_array.h

@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "arrow/array.h"
+#include "arrow/builder.h"
+
+#include "libmexclass/proxy/Proxy.h"
+
+namespace arrow::matlab::array::proxy {
+class Float64Array : public libmexclass::proxy::Proxy {
+    public:
+        Float64Array(const libmexclass::proxy::FunctionArguments& constructor_arguments) {
+            // Get the mxArray from constructor arguments
+            const ::matlab::data::TypedArray<double> double_mda = constructor_arguments[0];
+
+            // Get raw pointer of mxArray
+            auto it(double_mda.cbegin());
+            auto dt = it.operator->();
+
+            // Pass pointer to Arrow array constructor that takes a buffer
+            // Do not make a copy when creating arrow::Buffer
+            std::shared_ptr<arrow::Buffer> buffer(
+                  new arrow::Buffer(reinterpret_cast<const uint8_t*>(dt), 
+                                    sizeof(double) * double_mda.getNumberOfElements()));
+
+            // Construct arrow::NumericArray specialization using arrow::Buffer.
+            // pass in nulls information...we could compute and provide the number of nulls here too
+            std::shared_ptr<arrow::Array> array_wrapper(
+                new arrow::NumericArray<arrow::DoubleType>(double_mda.getNumberOfElements(), buffer,
+                                                       nullptr, // TODO: fill validity bitmap with data
+                                                       -1));
+
+            array = array_wrapper;
+
+            // Register Proxy methods.
+            REGISTER_METHOD(Float64Array, Print);
+        }
+    private:
+        void Print(libmexclass::proxy::method::Context& context);
+
+        // "Raw" arrow::Array
+        std::shared_ptr<arrow::Array> array;
+};
+} // namespace arrow::matlab::array::proxy

matlab/src/cpp/arrow/matlab/mex/gateway.cc

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "mex.hpp"
+#include "mexAdapter.hpp"
+
+#include "libmexclass/mex/gateway.h"    
+
+#include "arrow/matlab/proxy/factory.h"
+
+class MexFunction : public matlab::mex::Function {
+    public:
+        void operator()(matlab::mex::ArgumentList outputs, matlab::mex::ArgumentList inputs) {
+            libmexclass::mex::gateway<arrow::matlab::proxy::Factory>(inputs, outputs, getEngine());
+        }
+};