ARROW-12965: [Java] C Data Interface implementation

ci/docker/linux-apt-jni.dockerfile

@@ -73,6 +73,7 @@ ENV ARROW_BUILD_TESTS=OFF \
     ARROW_GANDIVA_JAVA=ON \
     ARROW_GANDIVA=ON \
     ARROW_HOME=/usr/local \
+    ARROW_JAVA_CDATA=ON \
     ARROW_JNI=ON \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \

ci/scripts/java_build.sh

@@ -21,6 +21,7 @@ set -ex
 arrow_dir=${1}
 source_dir=${1}/java
 cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug}
+cdata_dist_dir=${2}/java/c
 with_docs=${3:-false}
 
 if [[ "$(uname -s)" == "Linux" ]] && [[ "$(uname -m)" == "s390x" ]]; then
@@ -84,6 +85,10 @@ if [ "${ARROW_JAVA_SHADE_FLATBUFFERS}" == "ON" ]; then
   ${mvn} -Pshade-flatbuffers install
 fi
 
+if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
+  ${mvn} -Darrow.c.jni.dist.dir=${cdata_dist_dir} -Parrow-c-data install
+fi
+
 if [ "${ARROW_GANDIVA_JAVA}" = "ON" ]; then
   ${mvn} -Darrow.cpp.build.dir=${cpp_build_dir} -Parrow-jni install
 fi

ci/scripts/java_cdata_build.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+arrow_dir=${1}
+build_dir=${2}
+# The directory where the final binaries will be stored when scripts finish
+dist_dir=${3}
+
+echo "=== Clear output directories and leftovers ==="
+# Clear output directories and leftovers
+rm -rf ${build_dir}
+
+echo "=== Building Arrow Java C Data Interface native library ==="
+mkdir -p "${build_dir}"
+pushd "${build_dir}"
+
+cmake \
+  -DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-release} \
+  -DCMAKE_INSTALL_LIBDIR=lib \
+  -DCMAKE_INSTALL_PREFIX=${build_dir} \
+  ${arrow_dir}/java/c
+cmake --build . --target install --config ${ARROW_BUILD_TYPE:-release}
+popd
+
+echo "=== Copying libraries to the distribution folder ==="
+mkdir -p "${dist_dir}"
+cp -L ${build_dir}/lib/*arrow_cdata_jni.* ${dist_dir}

ci/scripts/java_jni_build.sh → ci/scripts/java_full_build.sh

@@ -20,18 +20,17 @@
 set -e
 
 arrow_dir=${1}
-cpp_lib_dir=${2}
-java_dist_dir=${3}
+dist_dir=${2}
 
 export ARROW_TEST_DATA=${arrow_dir}/testing/data
 
 pushd ${arrow_dir}/java
 
 # build the entire project
-mvn clean install -P arrow-jni -Darrow.cpp.build.dir=$cpp_lib_dir
+mvn clean install -Parrow-c-data -Parrow-jni -Darrow.cpp.build.dir=$dist_dir -Darrow.c.jni.dist.dir=$dist_dir
 
 # copy all jars and pom files to the distribution folder
-find . -name "*.jar" -exec echo {} \; -exec cp {} $java_dist_dir \;
-find . -name "*.pom" -exec echo {} \; -exec cp {} $java_dist_dir \;
+find . -name "*.jar" -exec echo {} \; -exec cp {} $dist_dir \;
+find . -name "*.pom" -exec echo {} \; -exec cp {} $dist_dir \;
 
 popd

ci/scripts/java_test.sh

@@ -21,6 +21,7 @@ set -ex
 arrow_dir=${1}
 source_dir=${1}/java
 cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug}
+cdata_dist_dir=${2}/java/c
 
 # For JNI and Plasma tests
 export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
@@ -38,6 +39,10 @@ if [ "${ARROW_JNI}" = "ON" ]; then
   ${mvn} test -Parrow-jni -pl adapter/orc,gandiva,dataset -Darrow.cpp.build.dir=${cpp_build_dir}
 fi
 
+if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
+  ${mvn} test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir=${cdata_dist_dir}
+fi
+
 if [ "${ARROW_PLASMA}" = "ON" ]; then
   pushd ${source_dir}/plasma
   java -cp target/test-classes:target/classes \

dev/release/01-prepare-test.rb

@@ -207,6 +207,9 @@ def test_version_pre_tag
                    { hunks: [["-    <version>#{@snapshot_version}</version>",
                               "+    <version>#{@release_version}</version>"]],
                      path: "java/algorithm/pom.xml" },
+                   { hunks: [["-        <version>#{@snapshot_version}</version>",
+                              "+        <version>#{@release_version}</version>"]],
+                     path: "java/c/pom.xml" },
                    { hunks: [["-    <version>#{@snapshot_version}</version>",
                               "+    <version>#{@release_version}</version>"]],
                      path: "java/compression/pom.xml" },
@@ -405,6 +408,9 @@ def test_version_post_tag
                    { hunks: [["-    <version>#{@snapshot_version}</version>",
                               "+    <version>#{@next_snapshot_version}</version>"]],
                      path: "java/algorithm/pom.xml" },
+                   { hunks: [["-        <version>#{@snapshot_version}</version>",
+                              "+        <version>#{@next_snapshot_version}</version>"]],
+                     path: "java/c/pom.xml" },
                    { hunks: [["-    <version>#{@snapshot_version}</version>",
                               "+    <version>#{@next_snapshot_version}</version>"]],
                      path: "java/compression/pom.xml" },

dev/tasks/java-jars/github.yml

@@ -51,6 +51,13 @@ jobs:
     steps:
       {{ macros.github_checkout_arrow()|indent }}
       {{ macros.github_install_archery()|indent }}
+      - name: Build C Data Interface lib
+        run: |
+          set -e
+          arrow/ci/scripts/java_cdata_build.sh \
+            $GITHUB_WORKSPACE/arrow \
+            $GITHUB_WORKSPACE/arrow/java-native-build \
+            $GITHUB_WORKSPACE/arrow/java/dist
       - name: Build C++ Libs
         run: |
           set -e
@@ -86,17 +93,18 @@ jobs:
           tar -xvzf arrow-shared-libs-linux.tar.gz
       - name: Test that Shared Libraries Exist
         run: |
+          test -f arrow/java/dist/libarrow_cdata_jni.dylib  
           test -f arrow/java/dist/libarrow_dataset_jni.dylib
           test -f arrow/java/dist/libgandiva_jni.dylib
           test -f arrow/java/dist/libarrow_orc_jni.dylib
+          test -f arrow/java/dist/libarrow_cdata_jni.so  
           test -f arrow/java/dist/libarrow_dataset_jni.so
           test -f arrow/java/dist/libarrow_orc_jni.so
           test -f arrow/java/dist/libgandiva_jni.so
       - name: Build Bundled Jar
         run: |
           set -e
-          arrow/ci/scripts/java_jni_build.sh \
+          arrow/ci/scripts/java_full_build.sh \
             $GITHUB_WORKSPACE/arrow \
-            $GITHUB_WORKSPACE/arrow/java/dist \
             $GITHUB_WORKSPACE/arrow/java/dist
       {{ macros.github_upload_releases(["arrow/java/dist/*.jar", "arrow/java/dist/*.pom"])|indent }}

docker-compose.yml

@@ -875,6 +875,7 @@ services:
       - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated
     command:
       ["pip install -e /arrow/dev/archery &&
+        /arrow/ci/scripts/java_cdata_build.sh /arrow /java-native-build /arrow/java/dist &&
         /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java/dist"]
 
   ##############################  Integration #################################
@@ -1308,7 +1309,7 @@ services:
         /arrow/ci/scripts/java_test.sh /arrow /build"
 
   debian-java-jni:
-    # Includes plasma test and jni for gandiva and orc.
+    # Includes plasma test, jni for gandiva and orc, and C data interface.
     # Usage:
     #   docker-compose build debian-java
     #   docker-compose build debian-java-jni
@@ -1332,6 +1333,7 @@ services:
     command:
       /bin/bash -c "
         /arrow/ci/scripts/cpp_build.sh /arrow /build &&
+        /arrow/ci/scripts/java_cdata_build.sh /arrow /build/java/c/build /build/java/c &&
         /arrow/ci/scripts/java_build.sh /arrow /build &&
         /arrow/ci/scripts/java_test.sh /arrow /build"
 

java/.gitignore

@@ -22,3 +22,4 @@ Makefile
 cmake_install.cmake
 install_manifest.txt
 ?/
+!c/

java/c/.gitignore

@@ -0,0 +1 @@
+build/**

java/c/CMakeLists.txt

@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#
+# arrow_cdata_java
+#
+
+cmake_minimum_required(VERSION 3.11)
+message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
+project(arrow_cdata_java)
+
+# Find java/jni
+include(UseJava)
+
+find_package(Java REQUIRED)
+find_package(JNI REQUIRED)
+
+set(JNI_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated")
+
+include_directories(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}
+                    ${JNI_INCLUDE_DIRS} ${JNI_HEADERS_DIR})
+
+add_jar(${PROJECT_NAME}
+        src/main/java/org/apache/arrow/c/jni/JniLoader.java
+        src/main/java/org/apache/arrow/c/jni/JniWrapper.java
+        src/main/java/org/apache/arrow/c/jni/PrivateData.java
+        GENERATE_NATIVE_HEADERS
+        arrow_cdata_java-native
+        DESTINATION
+        ${JNI_HEADERS_DIR})
+
+set(SOURCES src/main/cpp/jni_wrapper.cc)
+add_library(arrow_cdata_jni SHARED ${SOURCES})
+target_link_libraries(arrow_cdata_jni ${JAVA_JVM_LIBRARY})
+add_dependencies(arrow_cdata_jni ${PROJECT_NAME})
+
+install(TARGETS arrow_cdata_jni DESTINATION ${CMAKE_INSTALL_LIBDIR})

java/c/README.md

@@ -0,0 +1,54 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# C Interfaces for Arrow Java
+
+## Setup Build Environment
+
+install:
+ - Java 8 or later
+ - Maven 3.3 or later
+ - A C++11-enabled compiler
+ - CMake 3.11 or later
+ - Make or ninja build utilities
+
+## Building JNI wrapper shared library
+
+```
+mkdir -p build
+pushd build
+cmake ..
+cmake --build .
+popd
+```
+
+## Building and running tests
+
+Run tests with
+
+```
+mvn test
+```
+
+To install Apache Arrow (Java) with this module enabled run the following from the project root directory:
+
+```
+cd java
+mvn -Parrow-c-data install
+```

java/c/pom.xml

@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>arrow-java-root</artifactId>
+        <groupId>org.apache.arrow</groupId>
+        <version>6.0.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>arrow-c-data</artifactId>
+    <name>Arrow Java C Data Interface</name>
+    <description>Java implementation of C Data Interface</description>
+    <packaging>jar</packaging>
+    <properties>
+        <arrow.c.jni.dist.dir>./build</arrow.c.jni.dist.dir>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-vector</artifactId>
+            <version>${project.version}</version>
+            <scope>compile</scope>
+            <classifier>${arrow.vector.classifier}</classifier>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-vector</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-memory-core</artifactId>
+            <version>${project.version}</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-memory-unsafe</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${dep.guava.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+    <build>
+        <resources>
+            <resource>
+                <directory>${arrow.c.jni.dist.dir}</directory>
+                <includes>
+                    <include>**/*arrow_cdata_jni.*</include>
+                </includes>
+            </resource>
+        </resources>
+    </build>
+
+</project>