From cc21b7cc4301b9533bfdfcb801bb0864426f86cb Mon Sep 17 00:00:00 2001 From: david dali susanibar arce Date: Mon, 15 Aug 2022 12:37:26 -0500 Subject: [PATCH 1/5] Dataset/ORC/Gandiva library (.so / .dylib) able to compile with mvn command --- java/pom.xml | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/java/pom.xml b/java/pom.xml index 08bc54c140e..b56eb41367d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -887,6 +887,99 @@ + + generate-jnicpp-dylib_so + + OFF + OFF + OFF + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + jnicppdir + generate-resources + + exec + + + mkdir + -p java-dist java-native-cpp + ../ + + + + jnicppdefine + generate-resources + + exec + + + cmake + + -DARROW_BOOST_USE_SHARED=OFF + -DARROW_BROTLI_USE_SHARED=OFF + -DARROW_BZ2_USE_SHARED=OFF + -DARROW_GFLAGS_USE_SHARED=OFF + -DARROW_GRPC_USE_SHARED=OFF + -DARROW_LZ4_USE_SHARED=OFF + -DARROW_OPENSSL_USE_SHARED=OFF + -DARROW_PROTOBUF_USE_SHARED=OFF + -DARROW_SNAPPY_USE_SHARED=OFF + -DARROW_THRIFT_USE_SHARED=OFF + -DARROW_UTF8PROC_USE_SHARED=OFF + -DARROW_ZSTD_USE_SHARED=OFF + -DARROW_JNI=ON + -DARROW_PARQUET=ON + -DARROW_FILESYSTEM=ON + -DARROW_DATASET=${ARROW_DATASET} + -DARROW_GANDIVA_JAVA=ON + -DARROW_GANDIVA_STATIC_LIBSTDCPP=ON + -DARROW_GANDIVA=${ARROW_GANDIVA} + -DARROW_ORC=${ARROW_ORC} + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_INSTALL_LIBDIR=lib + -DCMAKE_INSTALL_PREFIX=../java-dist + -DCMAKE_UNITY_BUILD=ON + -Dre2_SOURCE=BUNDLED + -DBoost_SOURCE=BUNDLED + -Dutf8proc_SOURCE=BUNDLED + -DSnappy_SOURCE=BUNDLED + -DORC_SOURCE=BUNDLED + -DZLIB_SOURCE=BUNDLED + -Dxsimd_SOURCE=BUNDLED + -Dzstd_SOURCE=BUNDLED + -Dlz4_SOURCE=BUNDLED + ../cpp + + ../java-native-cpp + + + + jnicppbuild + generate-resources + + exec + + + cmake + + --build . --target install + + ../java-native-cpp + + + + + + + + From b4974c6c0b67f5284470dafcccd832b85184dc93 Mon Sep 17 00:00:00 2001 From: david dali susanibar arce Date: Mon, 15 Aug 2022 19:16:21 -0500 Subject: [PATCH 2/5] Simplifying Arrow building parameters --- java/pom.xml | 40 +++++----------------------------------- 1 file changed, 5 insertions(+), 35 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index b56eb41367d..6bab5470322 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -893,6 +893,7 @@ OFF OFF OFF + ../java-dist @@ -901,18 +902,6 @@ exec-maven-plugin 3.1.0 - - jnicppdir - generate-resources - - exec - - - mkdir - -p java-dist java-native-cpp - ../ - - jnicppdefine generate-resources @@ -922,18 +911,7 @@ cmake - -DARROW_BOOST_USE_SHARED=OFF - -DARROW_BROTLI_USE_SHARED=OFF - -DARROW_BZ2_USE_SHARED=OFF - -DARROW_GFLAGS_USE_SHARED=OFF - -DARROW_GRPC_USE_SHARED=OFF - -DARROW_LZ4_USE_SHARED=OFF - -DARROW_OPENSSL_USE_SHARED=OFF - -DARROW_PROTOBUF_USE_SHARED=OFF - -DARROW_SNAPPY_USE_SHARED=OFF - -DARROW_THRIFT_USE_SHARED=OFF - -DARROW_UTF8PROC_USE_SHARED=OFF - -DARROW_ZSTD_USE_SHARED=OFF + -DARROW_*_USE_SHARED=OFF -DARROW_JNI=ON -DARROW_PARQUET=ON -DARROW_FILESYSTEM=ON @@ -944,17 +922,9 @@ -DARROW_ORC=${ARROW_ORC} -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib - -DCMAKE_INSTALL_PREFIX=../java-dist + -DCMAKE_INSTALL_PREFIX=${arrow.cpp.build.dir} -DCMAKE_UNITY_BUILD=ON - -Dre2_SOURCE=BUNDLED - -DBoost_SOURCE=BUNDLED - -Dutf8proc_SOURCE=BUNDLED - -DSnappy_SOURCE=BUNDLED - -DORC_SOURCE=BUNDLED - -DZLIB_SOURCE=BUNDLED - -Dxsimd_SOURCE=BUNDLED - -Dzstd_SOURCE=BUNDLED - -Dlz4_SOURCE=BUNDLED + -DARROW_DEPENDENCY_SOURCE=BUNDLED ../cpp ../java-native-cpp @@ -969,7 +939,7 @@ cmake - --build . --target install + --build . --target install --config release ../java-native-cpp From e881031353bc6161fd0fa9cc6db6404ee4a6f492 Mon Sep 17 00:00:00 2001 From: david dali susanibar arce Date: Mon, 12 Sep 2022 16:29:55 -0500 Subject: [PATCH 3/5] Apply suggestions from code review Co-authored-by: Sutou Kouhei --- java/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 053c859f1ef..4f563ea8aff 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -929,7 +929,7 @@ cmake - -DARROW_*_USE_SHARED=OFF + -DARROW_DEPENDENCY_USE_SHARED=OFF -DARROW_JNI=ON -DARROW_PARQUET=ON -DARROW_FILESYSTEM=ON @@ -957,7 +957,7 @@ cmake - --build . --target install --config release + --build . --target install --config Release ../java-native-cpp From 679e9dd2d1294f4b68c00537273b68765e248760 Mon Sep 17 00:00:00 2001 From: david dali susanibar arce Date: Mon, 12 Sep 2022 21:23:52 -0500 Subject: [PATCH 4/5] Update Dataset build base on lats CMake changes --- java/pom.xml | 158 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 143 insertions(+), 15 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 4f563ea8aff..518cee72327 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -906,12 +906,10 @@ - generate-jnicpp-dylib_so + generate-dataset-dylib_so - OFF - OFF - OFF - ../java-dist + java-dist/lib + false @@ -921,7 +919,19 @@ 3.1.0 - jnicppdefine + datasetdir + generate-resources + + exec + + + mkdir + -p cpp-jni java-dist java-jni + ../ + + + + datasetarrowdependency generate-resources exec @@ -929,27 +939,145 @@ cmake + -S cpp + -B cpp-jni -DARROW_DEPENDENCY_USE_SHARED=OFF + -DARROW_CSV=ON + -DARROW_DATASET=ON + -DARROW_FILESYSTEM=ON + -DARROW_GANDIVA=ON + -DARROW_GANDIVA_JAVA=ON + -DARROW_GANDIVA_STATIC_LIBSTDCPP=ON -DARROW_JNI=ON + -DARROW_ORC=ON -DARROW_PARQUET=ON + -DARROW_PLASMA=ON + -DARROW_PLASMA_JAVA_CLIENT=ON + -DARROW_S3=ON + -DARROW_USE_CCACHE=ON + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_INSTALL_LIBDIR=lib + -DCMAKE_INSTALL_PREFIX=java-dist + -DCMAKE_UNITY_BUILD=ON + -DARROW_DEPENDENCY_SOURCE=BUNDLED + + ../ + ${cpp.dependencies.builded} + + + + datasetarrowdependencybuild + generate-resources + + exec + + + cmake + + --build cpp-jni --target install --config Release + + ../ + ${cpp.dependencies.builded} + + + + datasetdefine + generate-resources + + exec + + + cmake + + -S java + -B java-jni + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_INSTALL_PREFIX=${arrow.c.jni.dist.dir} + -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist + -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF + -DARROW_JAVA_JNI_ENABLE_DATASET=ON + -DBUILD_TESTING=OFF + + ../ + + + + datasetbuild + generate-resources + + exec + + + cmake + + --build java-jni --target install --config Release + + ../ + + + + + + + + + + generate-jnicpp-dylib_so + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + jnidir + generate-resources + + exec + + + mkdir + -p java-dist cpp-jni + ../ + + + + jnidefine + generate-resources + + exec + + + cmake + + -S cpp + -B cpp-jni + -DARROW_DEPENDENCY_USE_SHARED=OFF + -DARROW_CSV=ON + -DARROW_DATASET=ON -DARROW_FILESYSTEM=ON - -DARROW_DATASET=${ARROW_DATASET} + -DARROW_GANDIVA=ON -DARROW_GANDIVA_JAVA=ON -DARROW_GANDIVA_STATIC_LIBSTDCPP=ON - -DARROW_GANDIVA=${ARROW_GANDIVA} - -DARROW_ORC=${ARROW_ORC} + -DARROW_JNI=ON + -DARROW_ORC=ON + -DARROW_PARQUET=ON + -DARROW_PLASMA=ON + -DARROW_PLASMA_JAVA_CLIENT=ON + -DARROW_S3=ON + -DARROW_USE_CCACHE=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib - -DCMAKE_INSTALL_PREFIX=${arrow.cpp.build.dir} + -DCMAKE_INSTALL_PREFIX=java-dist -DCMAKE_UNITY_BUILD=ON -DARROW_DEPENDENCY_SOURCE=BUNDLED - ../cpp - ../java-native-cpp + ../ - jnicppbuild + jnidefinebuild generate-resources exec @@ -957,9 +1085,9 @@ cmake - --build . --target install --config Release + --build cpp-jni --target install --config Release - ../java-native-cpp + ../ From 0716f963cac7947384da8b880bd7312cf22b807a Mon Sep 17 00:00:00 2001 From: david dali susanibar arce Date: Tue, 13 Sep 2022 09:29:56 -0500 Subject: [PATCH 5/5] Sort CMake lists --- java/pom.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 518cee72327..90642300fca 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -941,9 +941,10 @@ -S cpp -B cpp-jni - -DARROW_DEPENDENCY_USE_SHARED=OFF -DARROW_CSV=ON -DARROW_DATASET=ON + -DARROW_DEPENDENCY_SOURCE=BUNDLED + -DARROW_DEPENDENCY_USE_SHARED=OFF -DARROW_FILESYSTEM=ON -DARROW_GANDIVA=ON -DARROW_GANDIVA_JAVA=ON @@ -959,7 +960,6 @@ -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_PREFIX=java-dist -DCMAKE_UNITY_BUILD=ON - -DARROW_DEPENDENCY_SOURCE=BUNDLED ../ ${cpp.dependencies.builded} @@ -991,12 +991,12 @@ -S java -B java-jni + -DARROW_JAVA_JNI_ENABLE_DATASET=ON + -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF + -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${arrow.c.jni.dist.dir} -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist - -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF - -DARROW_JAVA_JNI_ENABLE_DATASET=ON - -DBUILD_TESTING=OFF ../ @@ -1053,9 +1053,10 @@ -S cpp -B cpp-jni - -DARROW_DEPENDENCY_USE_SHARED=OFF -DARROW_CSV=ON -DARROW_DATASET=ON + -DARROW_DEPENDENCY_SOURCE=BUNDLED + -DARROW_DEPENDENCY_USE_SHARED=OFF -DARROW_FILESYSTEM=ON -DARROW_GANDIVA=ON -DARROW_GANDIVA_JAVA=ON @@ -1071,7 +1072,6 @@ -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_PREFIX=java-dist -DCMAKE_UNITY_BUILD=ON - -DARROW_DEPENDENCY_SOURCE=BUNDLED ../