From 65be6c236189c357c8a1bd4394cbffffa9963ab7 Mon Sep 17 00:00:00 2001 From: James Duong Date: Mon, 4 Dec 2023 14:10:14 -0800 Subject: [PATCH 01/54] compression and tools --- .../src/main/java/module-info.java | 24 +++++++++++++++++ java/tools/src/main/java/module-info.java | 27 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 java/compression/src/main/java/module-info.java create mode 100644 java/tools/src/main/java/module-info.java diff --git a/java/compression/src/main/java/module-info.java b/java/compression/src/main/java/module-info.java new file mode 100644 index 00000000000..da09f0c772d --- /dev/null +++ b/java/compression/src/main/java/module-info.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.compression { + exports org.apache.arrow.compression; + + requires com.github.luben.zstd_jni; + requires org.apache.arrow.vector; + requires org.apache.commons.compress; +} diff --git a/java/tools/src/main/java/module-info.java b/java/tools/src/main/java/module-info.java new file mode 100644 index 00000000000..8e5b7c64da2 --- /dev/null +++ b/java/tools/src/main/java/module-info.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.tools { + exports org.apache.arrow.tools; + + requires com.fasterxml.jackson.databind; + requires com.google.common; + requires commons.cli; + requires org.apache.arrow.compression; + requires org.apache.arrow.vector; + requires slf4j.api; +} From e7800634ca3a42ed8cd394441ea4808823b33187 Mon Sep 17 00:00:00 2001 From: James Duong Date: Mon, 11 Dec 2023 11:27:43 -0800 Subject: [PATCH 02/54] Fix dependencies in compression and tools --- java/compression/src/main/java/module-info.java | 1 + java/tools/pom.xml | 4 ++++ java/tools/src/main/java/module-info.java | 1 + 3 files changed, 6 insertions(+) diff --git a/java/compression/src/main/java/module-info.java b/java/compression/src/main/java/module-info.java index da09f0c772d..6bf989e4c14 100644 --- a/java/compression/src/main/java/module-info.java +++ b/java/compression/src/main/java/module-info.java @@ -19,6 +19,7 @@ exports org.apache.arrow.compression; requires com.github.luben.zstd_jni; + requires org.apache.arrow.memory.core; requires org.apache.arrow.vector; requires org.apache.commons.compress; } diff --git a/java/tools/pom.xml b/java/tools/pom.xml index 8df436bac9a..0d7eacfe2dd 100644 --- a/java/tools/pom.xml +++ b/java/tools/pom.xml @@ -34,6 +34,10 @@ org.apache.arrow arrow-compression + + org.immutables + value + com.google.guava guava diff --git a/java/tools/src/main/java/module-info.java b/java/tools/src/main/java/module-info.java index 8e5b7c64da2..c8f58c94927 100644 --- a/java/tools/src/main/java/module-info.java +++ b/java/tools/src/main/java/module-info.java @@ -22,6 +22,7 @@ requires com.google.common; requires commons.cli; requires org.apache.arrow.compression; + requires org.apache.arrow.memory.core; requires org.apache.arrow.vector; requires slf4j.api; } From c962a4bee5c25d1f6d189c5bf2d75c3a3ba8fd97 Mon Sep 17 00:00:00 2001 From: James Duong Date: Mon, 11 Dec 2023 12:26:20 -0800 Subject: [PATCH 03/54] Correctly reference the SLF4J module in arrow-tools --- java/tools/src/main/java/module-info.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/tools/src/main/java/module-info.java b/java/tools/src/main/java/module-info.java index c8f58c94927..6406ff189e3 100644 --- a/java/tools/src/main/java/module-info.java +++ b/java/tools/src/main/java/module-info.java @@ -24,5 +24,5 @@ requires org.apache.arrow.compression; requires org.apache.arrow.memory.core; requires org.apache.arrow.vector; - requires slf4j.api; + requires org.slf4j; } From 1d5e9acb24697ed458a971eacd302a0781c4438e Mon Sep 17 00:00:00 2001 From: James Duong Date: Mon, 11 Dec 2023 13:32:11 -0800 Subject: [PATCH 04/54] Add dependency on immutables to compression Transitive dependency from arrow-memory-core must be made explicit due to JPMS. --- java/compression/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/java/compression/pom.xml b/java/compression/pom.xml index 9a9f029fee1..e8008c97543 100644 --- a/java/compression/pom.xml +++ b/java/compression/pom.xml @@ -35,6 +35,10 @@ arrow-memory-unsafe test + + org.immutables + value + org.apache.commons commons-compress From 6afcb4398fcbfa763116db6493a1c1bce600a559 Mon Sep 17 00:00:00 2001 From: James Duong Date: Mon, 11 Dec 2023 13:33:10 -0800 Subject: [PATCH 05/54] Remove faulty JPMS name for commons-cli commons-cli does not have an automatic module name and should be loaded in the unnamed module instead --- java/tools/src/main/java/module-info.java | 1 - 1 file changed, 1 deletion(-) diff --git a/java/tools/src/main/java/module-info.java b/java/tools/src/main/java/module-info.java index 6406ff189e3..6b4329eb84f 100644 --- a/java/tools/src/main/java/module-info.java +++ b/java/tools/src/main/java/module-info.java @@ -20,7 +20,6 @@ requires com.fasterxml.jackson.databind; requires com.google.common; - requires commons.cli; requires org.apache.arrow.compression; requires org.apache.arrow.memory.core; requires org.apache.arrow.vector; From ca44382ca4215a00e5dd16579ffececd6bb9636e Mon Sep 17 00:00:00 2001 From: James Duong Date: Tue, 12 Dec 2023 00:21:33 -0800 Subject: [PATCH 06/54] Change avro adapter package structure Move contents from org.apache.arrow to org.apache.arrow.adapter.avro This is consistent with the packages used in the Orc and Jdbc adapters and avoids polluting the more generic org.apache.arrow package. --- .../avro/src/main/java/module-info.java | 0 .../arrow/{ => adapter/avro}/AvroToArrow.java | 2 +- .../{ => adapter/avro}/AvroToArrowConfig.java | 2 +- .../avro}/AvroToArrowConfigBuilder.java | 2 +- .../{ => adapter/avro}/AvroToArrowUtils.java | 50 +++++++++---------- .../avro}/AvroToArrowVectorIterator.java | 4 +- .../avro}/consumers/AvroArraysConsumer.java | 2 +- .../avro}/consumers/AvroBooleanConsumer.java | 2 +- .../avro}/consumers/AvroBytesConsumer.java | 2 +- .../avro}/consumers/AvroDoubleConsumer.java | 2 +- .../avro}/consumers/AvroEnumConsumer.java | 2 +- .../avro}/consumers/AvroFixedConsumer.java | 2 +- .../avro}/consumers/AvroFloatConsumer.java | 2 +- .../avro}/consumers/AvroIntConsumer.java | 2 +- .../avro}/consumers/AvroLongConsumer.java | 2 +- .../avro}/consumers/AvroMapConsumer.java | 2 +- .../avro}/consumers/AvroNullConsumer.java | 2 +- .../avro}/consumers/AvroStringConsumer.java | 2 +- .../avro}/consumers/AvroStructConsumer.java | 2 +- .../avro}/consumers/AvroUnionsConsumer.java | 2 +- .../avro}/consumers/BaseAvroConsumer.java | 2 +- .../consumers/CompositeAvroConsumer.java | 2 +- .../avro}/consumers/Consumer.java | 2 +- .../avro}/consumers/SkipConsumer.java | 2 +- .../avro}/consumers/SkipFunction.java | 2 +- .../consumers/logical/AvroDateConsumer.java | 4 +- .../logical/AvroDecimalConsumer.java | 4 +- .../logical/AvroTimeMicroConsumer.java | 4 +- .../logical/AvroTimeMillisConsumer.java | 4 +- .../logical/AvroTimestampMicrosConsumer.java | 4 +- .../logical/AvroTimestampMillisConsumer.java | 4 +- .../avro}/AvroLogicalTypesTest.java | 2 +- .../{ => adapter/avro}/AvroSkipFieldTest.java | 2 +- .../{ => adapter/avro}/AvroTestBase.java | 2 +- .../avro}/AvroToArrowIteratorTest.java | 2 +- .../{ => adapter/avro}/AvroToArrowTest.java | 2 +- .../avro}/TestWriteReadAvroRecord.java | 2 +- .../arrow/adapter/AvroAdapterBenchmarks.java | 8 +-- 38 files changed, 71 insertions(+), 71 deletions(-) create mode 100644 java/adapter/avro/src/main/java/module-info.java rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/AvroToArrow.java (98%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/AvroToArrowConfig.java (98%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/AvroToArrowConfigBuilder.java (98%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/AvroToArrowUtils.java (95%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/AvroToArrowVectorIterator.java (98%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroArraysConsumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroBooleanConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroBytesConsumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroDoubleConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroEnumConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroFixedConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroFloatConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroIntConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroLongConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroMapConsumer.java (98%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroNullConsumer.java (96%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroStringConsumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroStructConsumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/AvroUnionsConsumer.java (98%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/BaseAvroConsumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/CompositeAvroConsumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/Consumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/SkipConsumer.java (97%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/SkipFunction.java (95%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/logical/AvroDateConsumer.java (91%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/logical/AvroDecimalConsumer.java (95%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/logical/AvroTimeMicroConsumer.java (91%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/logical/AvroTimeMillisConsumer.java (91%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/logical/AvroTimestampMicrosConsumer.java (92%) rename java/adapter/avro/src/main/java/org/apache/arrow/{ => adapter/avro}/consumers/logical/AvroTimestampMillisConsumer.java (92%) rename java/adapter/avro/src/test/java/org/apache/arrow/{ => adapter/avro}/AvroLogicalTypesTest.java (99%) rename java/adapter/avro/src/test/java/org/apache/arrow/{ => adapter/avro}/AvroSkipFieldTest.java (99%) rename java/adapter/avro/src/test/java/org/apache/arrow/{ => adapter/avro}/AvroTestBase.java (99%) rename java/adapter/avro/src/test/java/org/apache/arrow/{ => adapter/avro}/AvroToArrowIteratorTest.java (99%) rename java/adapter/avro/src/test/java/org/apache/arrow/{ => adapter/avro}/AvroToArrowTest.java (99%) rename java/adapter/avro/src/test/java/org/apache/arrow/{ => adapter/avro}/TestWriteReadAvroRecord.java (98%) diff --git a/java/adapter/avro/src/main/java/module-info.java b/java/adapter/avro/src/main/java/module-info.java new file mode 100644 index 00000000000..e69de29bb2d diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrow.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrow.java similarity index 98% rename from java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrow.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrow.java index 33f18039378..8baa60a72dd 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrow.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrow.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowConfig.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java similarity index 98% rename from java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowConfig.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java index 4f59ef3843d..f9210fb0125 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowConfig.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import java.util.Set; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowConfigBuilder.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfigBuilder.java similarity index 98% rename from java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowConfigBuilder.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfigBuilder.java index 474c1eb5ca7..41e486d0a1c 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowConfigBuilder.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowConfigBuilder.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import java.util.HashSet; import java.util.Set; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java similarity index 95% rename from java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java index 80293c8b85c..1f5ad9e7689 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; @@ -31,30 +31,30 @@ import java.util.Set; import java.util.stream.Collectors; -import org.apache.arrow.consumers.AvroArraysConsumer; -import org.apache.arrow.consumers.AvroBooleanConsumer; -import org.apache.arrow.consumers.AvroBytesConsumer; -import org.apache.arrow.consumers.AvroDoubleConsumer; -import org.apache.arrow.consumers.AvroEnumConsumer; -import org.apache.arrow.consumers.AvroFixedConsumer; -import org.apache.arrow.consumers.AvroFloatConsumer; -import org.apache.arrow.consumers.AvroIntConsumer; -import org.apache.arrow.consumers.AvroLongConsumer; -import org.apache.arrow.consumers.AvroMapConsumer; -import org.apache.arrow.consumers.AvroNullConsumer; -import org.apache.arrow.consumers.AvroStringConsumer; -import org.apache.arrow.consumers.AvroStructConsumer; -import org.apache.arrow.consumers.AvroUnionsConsumer; -import org.apache.arrow.consumers.CompositeAvroConsumer; -import org.apache.arrow.consumers.Consumer; -import org.apache.arrow.consumers.SkipConsumer; -import org.apache.arrow.consumers.SkipFunction; -import org.apache.arrow.consumers.logical.AvroDateConsumer; -import org.apache.arrow.consumers.logical.AvroDecimalConsumer; -import org.apache.arrow.consumers.logical.AvroTimeMicroConsumer; -import org.apache.arrow.consumers.logical.AvroTimeMillisConsumer; -import org.apache.arrow.consumers.logical.AvroTimestampMicrosConsumer; -import org.apache.arrow.consumers.logical.AvroTimestampMillisConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroArraysConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroBooleanConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroBytesConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroDoubleConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroEnumConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroFixedConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroFloatConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroIntConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroLongConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroMapConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroNullConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroStringConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroStructConsumer; +import org.apache.arrow.adapter.avro.consumers.AvroUnionsConsumer; +import org.apache.arrow.adapter.avro.consumers.CompositeAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.Consumer; +import org.apache.arrow.adapter.avro.consumers.SkipConsumer; +import org.apache.arrow.adapter.avro.consumers.SkipFunction; +import org.apache.arrow.adapter.avro.consumers.logical.AvroDateConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroDecimalConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimeMicroConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimeMillisConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampMicrosConsumer; +import org.apache.arrow.adapter.avro.consumers.logical.AvroTimestampMillisConsumer; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BaseIntVector; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowVectorIterator.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java similarity index 98% rename from java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowVectorIterator.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java index 1faa7595c68..4a439ade811 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowVectorIterator.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import java.io.EOFException; import java.util.ArrayList; @@ -23,7 +23,7 @@ import java.util.List; import java.util.stream.Collectors; -import org.apache.arrow.consumers.CompositeAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.CompositeAvroConsumer; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroArraysConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroArraysConsumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroArraysConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroArraysConsumer.java index b9d0f84cfde..fd25986c32b 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroArraysConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroArraysConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroBooleanConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBooleanConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroBooleanConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBooleanConsumer.java index 4ca5f24451f..bf41828d19f 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroBooleanConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBooleanConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroBytesConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBytesConsumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroBytesConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBytesConsumer.java index eede68ebd39..c8370e48060 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroBytesConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBytesConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroDoubleConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroDoubleConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroDoubleConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroDoubleConsumer.java index 356707a140b..7cc7dd33b15 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroDoubleConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroDoubleConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroEnumConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroEnumConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroEnumConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroEnumConsumer.java index 2f4443b7417..32a2c85f6fc 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroEnumConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroEnumConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroFixedConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFixedConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroFixedConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFixedConsumer.java index a065466e395..16b70898fd3 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroFixedConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFixedConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroFloatConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFloatConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroFloatConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFloatConsumer.java index c8de4a21af4..b09d2881875 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroFloatConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFloatConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroIntConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroIntConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroIntConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroIntConsumer.java index bc8d4de78ab..ae5a2719c56 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroIntConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroIntConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroLongConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroLongConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroLongConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroLongConsumer.java index b9016c58f05..4db836acc45 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroLongConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroLongConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroMapConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroMapConsumer.java similarity index 98% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroMapConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroMapConsumer.java index b8e8bd585ee..1ea97e63b61 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroMapConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroMapConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroNullConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullConsumer.java similarity index 96% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroNullConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullConsumer.java index 64768008a95..4c7bb8c03ba 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroNullConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroStringConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStringConsumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroStringConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStringConsumer.java index 10fe234ac6c..072270aa6c0 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroStringConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStringConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroStructConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStructConsumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroStructConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStructConsumer.java index 792d01ee502..a02b1577f9f 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroStructConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStructConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroUnionsConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroUnionsConsumer.java similarity index 98% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroUnionsConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroUnionsConsumer.java index c0bb0200fcd..76287543b06 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/AvroUnionsConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroUnionsConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/BaseAvroConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/BaseAvroConsumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/BaseAvroConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/BaseAvroConsumer.java index 303be8e504f..66a6cda6840 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/BaseAvroConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/BaseAvroConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import org.apache.arrow.vector.FieldVector; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/CompositeAvroConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/CompositeAvroConsumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/CompositeAvroConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/CompositeAvroConsumer.java index af476d27cb8..97812226180 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/CompositeAvroConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/CompositeAvroConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; import java.util.List; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/Consumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/Consumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/Consumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/Consumer.java index 8c4ee9a96e0..c2ae1ce77b2 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/Consumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/Consumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/SkipConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipConsumer.java similarity index 97% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/SkipConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipConsumer.java index 94c5b339d87..1ac0a6d7155 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/SkipConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipConsumer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/SkipFunction.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipFunction.java similarity index 95% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/SkipFunction.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipFunction.java index 61938916a5e..93fc4a7fede 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/SkipFunction.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipFunction.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.consumers; +package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroDateConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDateConsumer.java similarity index 91% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroDateConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDateConsumer.java index 3aa8970d9a9..a5c36d88fb7 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroDateConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDateConsumer.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.arrow.consumers.logical; +package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; -import org.apache.arrow.consumers.BaseAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.vector.DateDayVector; import org.apache.avro.io.Decoder; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroDecimalConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimalConsumer.java similarity index 95% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroDecimalConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimalConsumer.java index 24d73cf829f..ebe5ca3884e 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroDecimalConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimalConsumer.java @@ -15,12 +15,12 @@ * limitations under the License. */ -package org.apache.arrow.consumers.logical; +package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.arrow.consumers.BaseAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.DecimalVector; import org.apache.avro.io.Decoder; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimeMicroConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMicroConsumer.java similarity index 91% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimeMicroConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMicroConsumer.java index e68ba158ffa..89216d4ad14 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimeMicroConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMicroConsumer.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.arrow.consumers.logical; +package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; -import org.apache.arrow.consumers.BaseAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.vector.TimeMicroVector; import org.apache.avro.io.Decoder; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimeMillisConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMillisConsumer.java similarity index 91% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimeMillisConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMillisConsumer.java index f76186fc378..ab5df8d4bc8 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimeMillisConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMillisConsumer.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.arrow.consumers.logical; +package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; -import org.apache.arrow.consumers.BaseAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.vector.TimeMilliVector; import org.apache.avro.io.Decoder; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimestampMicrosConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java similarity index 92% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimestampMicrosConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java index 82da0e8054b..93b39d479ff 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimestampMicrosConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.arrow.consumers.logical; +package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; -import org.apache.arrow.consumers.BaseAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.vector.TimeStampMicroVector; import org.apache.avro.io.Decoder; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimestampMillisConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java similarity index 92% rename from java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimestampMillisConsumer.java rename to java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java index 159f49e1480..9e651c3959f 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/consumers/logical/AvroTimestampMillisConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.arrow.consumers.logical; +package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; -import org.apache.arrow.consumers.BaseAvroConsumer; +import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.vector.TimeStampMilliVector; import org.apache.avro.io.Decoder; diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/AvroLogicalTypesTest.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java similarity index 99% rename from java/adapter/avro/src/test/java/org/apache/arrow/AvroLogicalTypesTest.java rename to java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java index 050a50ddaea..6ee04e33a5c 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/AvroLogicalTypesTest.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import static junit.framework.TestCase.assertNull; import static junit.framework.TestCase.assertTrue; diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/AvroSkipFieldTest.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroSkipFieldTest.java similarity index 99% rename from java/adapter/avro/src/test/java/org/apache/arrow/AvroSkipFieldTest.java rename to java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroSkipFieldTest.java index b946dbd8653..a37eca6514e 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/AvroSkipFieldTest.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroSkipFieldTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import static org.junit.Assert.assertEquals; diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/AvroTestBase.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java similarity index 99% rename from java/adapter/avro/src/test/java/org/apache/arrow/AvroTestBase.java rename to java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java index 16d8e52722c..47915fbe80e 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/AvroTestBase.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowIteratorTest.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowIteratorTest.java similarity index 99% rename from java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowIteratorTest.java rename to java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowIteratorTest.java index 2b05a19f380..02f7a373373 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowIteratorTest.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowIteratorTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import static org.junit.Assert.assertEquals; diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowTest.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowTest.java similarity index 99% rename from java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowTest.java rename to java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowTest.java index c007e1ac7eb..1c642041917 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/AvroToArrowTest.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import static org.junit.Assert.assertEquals; diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/TestWriteReadAvroRecord.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java similarity index 98% rename from java/adapter/avro/src/test/java/org/apache/arrow/TestWriteReadAvroRecord.java rename to java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java index 0a153a28cbc..13d9444c88a 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/TestWriteReadAvroRecord.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow; +package org.apache.arrow.adapter.avro; import static org.junit.Assert.assertEquals; diff --git a/java/performance/src/test/java/org/apache/arrow/adapter/AvroAdapterBenchmarks.java b/java/performance/src/test/java/org/apache/arrow/adapter/AvroAdapterBenchmarks.java index 884647b5af3..c07aeffafb1 100644 --- a/java/performance/src/test/java/org/apache/arrow/adapter/AvroAdapterBenchmarks.java +++ b/java/performance/src/test/java/org/apache/arrow/adapter/AvroAdapterBenchmarks.java @@ -21,10 +21,10 @@ import java.io.ByteArrayOutputStream; import java.util.concurrent.TimeUnit; -import org.apache.arrow.AvroToArrow; -import org.apache.arrow.AvroToArrowConfig; -import org.apache.arrow.AvroToArrowConfigBuilder; -import org.apache.arrow.AvroToArrowVectorIterator; +import org.apache.arrow.adapter.avro.AvroToArrow; +import org.apache.arrow.adapter.avro.AvroToArrowConfig; +import org.apache.arrow.adapter.avro.AvroToArrowConfigBuilder; +import org.apache.arrow.adapter.avro.AvroToArrowVectorIterator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.IntVector; From 94c8852ce83fce04417ef5a75393e877d99c4886 Mon Sep 17 00:00:00 2001 From: James Duong Date: Tue, 12 Dec 2023 04:12:20 -0800 Subject: [PATCH 07/54] Modularize arrow-avro - Add module-info file - Rework resource lookups in tests to use reflection to call getModule().getResourceAsStream() in JDK 9+ otherwise Class.getRsourceAsStream() in 8. --- java/adapter/avro/pom.xml | 5 ++++ .../avro/src/main/java/module-info.java | 26 +++++++++++++++++++ .../arrow/adapter/avro/AvroTestBase.java | 26 ++++++++++++++----- .../adapter/avro/TestWriteReadAvroRecord.java | 7 +---- 4 files changed, 51 insertions(+), 13 deletions(-) diff --git a/java/adapter/avro/pom.xml b/java/adapter/avro/pom.xml index c0410ea4c23..90864eab006 100644 --- a/java/adapter/avro/pom.xml +++ b/java/adapter/avro/pom.xml @@ -46,6 +46,11 @@ arrow-vector + + org.immutables + value + + org.apache.avro avro diff --git a/java/adapter/avro/src/main/java/module-info.java b/java/adapter/avro/src/main/java/module-info.java index e69de29bb2d..5c6204be60e 100644 --- a/java/adapter/avro/src/main/java/module-info.java +++ b/java/adapter/avro/src/main/java/module-info.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.adapter.avro { + exports org.apache.arrow.adapter.avro.consumers; + exports org.apache.arrow.adapter.avro.consumers.logical; + exports org.apache.arrow.adapter.avro; + + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; + requires org.apache.avro; +} diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java index 47915fbe80e..60a3a285db3 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java @@ -24,9 +24,9 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.InputStream; +import java.lang.reflect.Method; import java.nio.ByteBuffer; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; @@ -62,11 +62,23 @@ public void init() { config = new AvroToArrowConfigBuilder(allocator).build(); } - protected Schema getSchema(String schemaName) throws Exception { - Path schemaPath = Paths.get(Paths.get(TestWriteReadAvroRecord.class.getResource("/").toURI()).toString(), - "schema", schemaName); - - return new Schema.Parser().parse(schemaPath.toFile()); + public static Schema getSchema(String schemaName) throws Exception { + try { + // Attempt to use JDK 9 behavior of getting the module then the resource stream from the module. + // Note that this code is caller-sensitive. + Method getModuleMethod = Class.class.getMethod("getModule"); + Object module = getModuleMethod.invoke(TestWriteReadAvroRecord.class); + Method getResourceAsStreamFromModule = module.getClass().getMethod("getResourceAsStream", String.class); + try (InputStream is = (InputStream) getResourceAsStreamFromModule.invoke(module, "/schema/" + schemaName)) { + return new Schema.Parser() + .parse(is); + } + } catch (NoSuchMethodException ex) { + // Use JDK8 behavior. + try (InputStream is = TestWriteReadAvroRecord.class.getResourceAsStream("/schema/" + schemaName)) { + return new Schema.Parser().parse(is); + } + } } protected VectorSchemaRoot writeAndRead(Schema schema, List data) throws Exception { diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java index 13d9444c88a..afbddaa6ed8 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java @@ -20,8 +20,6 @@ import static org.junit.Assert.assertEquals; import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; @@ -48,10 +46,7 @@ public class TestWriteReadAvroRecord { public void testWriteAndRead() throws Exception { File dataFile = TMP.newFile(); - Path schemaPath = Paths.get( - Paths.get(TestWriteReadAvroRecord.class.getResource("/").toURI()).toString(), - "schema", "test.avsc"); - Schema schema = new Schema.Parser().parse(schemaPath.toFile()); + Schema schema = AvroTestBase.getSchema("test.avsc"); //write data to disk GenericRecord user1 = new GenericData.Record(schema); From 83f22d1a49b574f2c1210a5fba39200d82a39db1 Mon Sep 17 00:00:00 2001 From: James Duong Date: Tue, 12 Dec 2023 04:24:57 -0800 Subject: [PATCH 08/54] Modularize adapter-jdbc Add dependency on Jackson's yaml module at test-time only. --- java/adapter/jdbc/pom.xml | 28 +++++++++++++++++++ .../jdbc/src/main/java/module-info.java | 28 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 java/adapter/jdbc/src/main/java/module-info.java diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index f95956d1f61..1cb1fb744fb 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -47,6 +47,11 @@ ${arrow.vector.classifier} + + org.immutables + value + + com.h2database @@ -85,4 +90,27 @@ + + + error-prone-jdk11+ + + [11,] + + !m2e.version + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-reads=org.apache.arrow.adapter.jdbc=com.fasterxml.jackson.dataformat.yaml --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + + + + + + diff --git a/java/adapter/jdbc/src/main/java/module-info.java b/java/adapter/jdbc/src/main/java/module-info.java new file mode 100644 index 00000000000..5b59ce76847 --- /dev/null +++ b/java/adapter/jdbc/src/main/java/module-info.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.adapter.jdbc { + exports org.apache.arrow.adapter.jdbc.consumer; + exports org.apache.arrow.adapter.jdbc; + exports org.apache.arrow.adapter.jdbc.binder; + + requires com.fasterxml.jackson.databind; + requires java.sql; + requires jdk.unsupported; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; +} From df90de675bc88bb4756bbd78159fe45cd6b9f00e Mon Sep 17 00:00:00 2001 From: James Duong Date: Tue, 12 Dec 2023 04:32:12 -0800 Subject: [PATCH 09/54] Modularize arrow-algorithm --- java/algorithm/pom.xml | 4 +++ java/algorithm/src/main/java/module-info.java | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 java/algorithm/src/main/java/module-info.java diff --git a/java/algorithm/pom.xml b/java/algorithm/pom.xml index 3e32d955ec4..99740f20028 100644 --- a/java/algorithm/pom.xml +++ b/java/algorithm/pom.xml @@ -42,6 +42,10 @@ arrow-memory-netty test + + org.immutables + value + diff --git a/java/algorithm/src/main/java/module-info.java b/java/algorithm/src/main/java/module-info.java new file mode 100644 index 00000000000..b347f55aa4d --- /dev/null +++ b/java/algorithm/src/main/java/module-info.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.algorithm { + exports org.apache.arrow.algorithm.search; + exports org.apache.arrow.algorithm.deduplicate; + exports org.apache.arrow.algorithm.dictionary; + exports org.apache.arrow.algorithm.rank; + exports org.apache.arrow.algorithm.misc; + exports org.apache.arrow.algorithm.sort; + + requires jdk.unsupported; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; +} From 74309feea4d96d7be57304cf3f991a4abd2f699e Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 17 Nov 2023 11:29:54 -0800 Subject: [PATCH 10/54] [Java] Move contents of flight-grpc into flight-core Delete flight-grpc --- java/bom/pom.xml | 5 - java/flight/flight-core/pom.xml | 5 + .../apache/arrow/flight/FlightGrpcUtils.java | 0 .../arrow/flight/TestFlightGrpcUtils.java | 0 .../src/test/protobuf/test.proto | 0 java/flight/flight-grpc/pom.xml | 123 ------------------ .../src/test/resources/logback.xml | 28 ---- java/flight/pom.xml | 1 - 8 files changed, 5 insertions(+), 157 deletions(-) rename java/flight/{flight-grpc => flight-core}/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java (100%) rename java/flight/{flight-grpc => flight-core}/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java (100%) rename java/flight/{flight-grpc => flight-core}/src/test/protobuf/test.proto (100%) delete mode 100644 java/flight/flight-grpc/pom.xml delete mode 100644 java/flight/flight-grpc/src/test/resources/logback.xml diff --git a/java/bom/pom.xml b/java/bom/pom.xml index 5c2ed33dadd..7ffb833e7f6 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -82,11 +82,6 @@ flight-core ${project.version} - - org.apache.arrow - flight-grpc - ${project.version} - org.apache.arrow flight-integration-tests diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index 8f41d2b65b7..f1c7e443dde 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -95,6 +95,11 @@ grpc-services test + + io.grpc + grpc-inprocess + test + com.fasterxml.jackson.core diff --git a/java/flight/flight-grpc/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java similarity index 100% rename from java/flight/flight-grpc/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java rename to java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightGrpcUtils.java diff --git a/java/flight/flight-grpc/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java similarity index 100% rename from java/flight/flight-grpc/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java rename to java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightGrpcUtils.java diff --git a/java/flight/flight-grpc/src/test/protobuf/test.proto b/java/flight/flight-core/src/test/protobuf/test.proto similarity index 100% rename from java/flight/flight-grpc/src/test/protobuf/test.proto rename to java/flight/flight-core/src/test/protobuf/test.proto diff --git a/java/flight/flight-grpc/pom.xml b/java/flight/flight-grpc/pom.xml deleted file mode 100644 index af765f8c436..00000000000 --- a/java/flight/flight-grpc/pom.xml +++ /dev/null @@ -1,123 +0,0 @@ - - - - - arrow-flight - org.apache.arrow - 15.0.0-SNAPSHOT - ../pom.xml - - 4.0.0 - - flight-grpc - Arrow Flight GRPC - (Experimental)Contains utility class to expose Flight gRPC service and client - jar - - - 1 - - - - - org.apache.arrow - flight-core - - - io.netty - netty-transport-native-unix-common - - - io.netty - netty-transport-native-kqueue - - - io.netty - netty-transport-native-epoll - - - - - io.grpc - grpc-stub - - - io.grpc - grpc-inprocess - test - - - org.apache.arrow - arrow-memory-core - compile - - - org.apache.arrow - arrow-memory-netty - runtime - - - io.grpc - grpc-protobuf - - - com.google.guava - guava - - - com.google.protobuf - protobuf-java - - - io.grpc - grpc-api - - - - - - - - kr.motd.maven - os-maven-plugin - 1.7.0 - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - 0.6.1 - - com.google.protobuf:protoc:${dep.protobuf-bom.version}:exe:${os.detected.classifier} - false - grpc-java - io.grpc:protoc-gen-grpc-java:${dep.grpc-bom.version}:exe:${os.detected.classifier} - - - - test - - ${basedir}/src/test/protobuf - ${project.build.directory}/generated-test-sources//protobuf - - - compile - compile-custom - - - - - - - - diff --git a/java/flight/flight-grpc/src/test/resources/logback.xml b/java/flight/flight-grpc/src/test/resources/logback.xml deleted file mode 100644 index 4c54d18a210..00000000000 --- a/java/flight/flight-grpc/src/test/resources/logback.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - diff --git a/java/flight/pom.xml b/java/flight/pom.xml index 7ddda94f77b..9ef01d07a73 100644 --- a/java/flight/pom.xml +++ b/java/flight/pom.xml @@ -26,7 +26,6 @@ flight-core - flight-grpc flight-sql flight-sql-jdbc-core flight-sql-jdbc-driver From 9bb974a47ba89aa2e8d34266cdd9a299d7487547 Mon Sep 17 00:00:00 2001 From: James Duong Date: Tue, 12 Dec 2023 16:32:12 -0800 Subject: [PATCH 11/54] Move ArrowTestDataUtil out of exported package Move ArrowTestDataUtil out of a module-exported package so that it can be reused properly in other tests. --- .../src/test/java/org/apache/arrow/dataset/TestAllTypes.java | 2 +- .../src/test/java/org/apache/arrow/flight/FlightTestUtil.java | 2 +- .../apache/arrow/vector/{ => test}/util/ArrowTestDataUtil.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename java/vector/src/test/java/org/apache/arrow/vector/{ => test}/util/ArrowTestDataUtil.java (97%) diff --git a/java/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java b/java/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java index 5293aca0c32..13b24745234 100644 --- a/java/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java +++ b/java/dataset/src/test/java/org/apache/arrow/dataset/TestAllTypes.java @@ -68,6 +68,7 @@ import org.apache.arrow.vector.complex.impl.UnionListWriter; import org.apache.arrow.vector.ipc.ArrowStreamReader; import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.arrow.vector.test.util.ArrowTestDataUtil; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; @@ -75,7 +76,6 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import org.apache.arrow.vector.util.ArrowTestDataUtil; import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; import org.apache.arrow.vector.util.Text; import org.junit.ClassRule; diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java index f9def74b56d..11510dbd320 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/FlightTestUtil.java @@ -25,7 +25,7 @@ import java.util.List; import java.util.Random; -import org.apache.arrow.vector.util.ArrowTestDataUtil; +import org.apache.arrow.vector.test.util.ArrowTestDataUtil; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.function.Executable; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/ArrowTestDataUtil.java b/java/vector/src/test/java/org/apache/arrow/vector/test/util/ArrowTestDataUtil.java similarity index 97% rename from java/vector/src/test/java/org/apache/arrow/vector/util/ArrowTestDataUtil.java rename to java/vector/src/test/java/org/apache/arrow/vector/test/util/ArrowTestDataUtil.java index 1c525c0c271..901a09e313f 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/ArrowTestDataUtil.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/test/util/ArrowTestDataUtil.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.vector.util; +package org.apache.arrow.vector.test.util; import java.nio.file.Path; import java.nio.file.Paths; From 1219971009769cfdd1e27710a64f9faf7fd0a2d6 Mon Sep 17 00:00:00 2001 From: James Duong Date: Tue, 12 Dec 2023 16:34:12 -0800 Subject: [PATCH 12/54] Modularize flight-core --- java/flight/flight-core/pom.xml | 32 ++++++++++++++ .../src/main/java/module-info.java | 43 +++++++++++++++++++ .../flight-sql/src/main/java/module-info.java | 0 .../arrow/flight/{ => sql}/TestFlightSql.java | 0 .../{ => sql}/TestFlightSqlStreams.java | 0 5 files changed, 75 insertions(+) create mode 100644 java/flight/flight-core/src/main/java/module-info.java create mode 100644 java/flight/flight-sql/src/main/java/module-info.java rename java/flight/flight-sql/src/test/java/org/apache/arrow/flight/{ => sql}/TestFlightSql.java (100%) rename java/flight/flight-sql/src/test/java/org/apache/arrow/flight/{ => sql}/TestFlightSqlStreams.java (100%) diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index f1c7e443dde..7caa039eb74 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -113,6 +113,10 @@ javax.annotation javax.annotation-api + + org.immutables + value + com.google.api.grpc @@ -310,4 +314,32 @@ + + + + error-prone-jdk11+ + + [11,] + + !m2e.version + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-opens=org.apache.arrow.flight.core/org.apache.arrow.flight.perf.impl=protobuf.java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + false + + ${project.basedir}/../../../testing/data + + + + + + + + diff --git a/java/flight/flight-core/src/main/java/module-info.java b/java/flight/flight-core/src/main/java/module-info.java new file mode 100644 index 00000000000..e76f22d6168 --- /dev/null +++ b/java/flight/flight-core/src/main/java/module-info.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.flight.core { + exports org.apache.arrow.flight; + exports org.apache.arrow.flight.auth; + exports org.apache.arrow.flight.auth2; + exports org.apache.arrow.flight.client; + exports org.apache.arrow.flight.impl; + exports org.apache.arrow.flight.sql.impl; + + requires com.fasterxml.jackson.databind; + requires com.google.common; + requires com.google.errorprone.annotations; + requires failureaccess; + requires io.grpc; + requires io.grpc.internal; + requires io.grpc.netty; + requires io.grpc.protobuf; + requires io.grpc.stub; + requires io.netty.common; + requires io.netty.handler; + requires io.netty.transport; + requires org.apache.arrow.format; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; + requires protobuf.java; + requires org.slf4j; +} diff --git a/java/flight/flight-sql/src/main/java/module-info.java b/java/flight/flight-sql/src/main/java/module-info.java new file mode 100644 index 00000000000..e69de29bb2d diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/TestFlightSql.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/TestFlightSql.java similarity index 100% rename from java/flight/flight-sql/src/test/java/org/apache/arrow/flight/TestFlightSql.java rename to java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/TestFlightSql.java diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/TestFlightSqlStreams.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/TestFlightSqlStreams.java similarity index 100% rename from java/flight/flight-sql/src/test/java/org/apache/arrow/flight/TestFlightSqlStreams.java rename to java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/TestFlightSqlStreams.java From 0f593dae85f2b8b014a6035378ce47751f62d39c Mon Sep 17 00:00:00 2001 From: James Duong Date: Wed, 13 Dec 2023 09:25:09 -0800 Subject: [PATCH 13/54] Modularize flight-sql --- java/flight/flight-sql/pom.xml | 27 +++++++++++++++++ .../flight-sql/src/main/java/module-info.java | 29 +++++++++++++++++++ .../flight/sql/{ => test}/TestFlightSql.java | 13 +++++++-- .../sql/{ => test}/TestFlightSqlStreams.java | 11 ++++++- 4 files changed, 77 insertions(+), 3 deletions(-) rename java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/{ => test}/TestFlightSql.java (98%) rename java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/{ => test}/TestFlightSqlStreams.java (96%) diff --git a/java/flight/flight-sql/pom.xml b/java/flight/flight-sql/pom.xml index 3c7e4b3495e..7ccab8e813c 100644 --- a/java/flight/flight-sql/pom.xml +++ b/java/flight/flight-sql/pom.xml @@ -50,6 +50,10 @@ org.apache.arrow arrow-memory-core + + org.immutables + value + org.apache.arrow arrow-jdbc @@ -110,4 +114,27 @@ + + + error-prone-jdk11+ + + [11,] + + !m2e.version + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-reads=org.apache.arrow.flight.sql=org.slf4j --add-reads=org.apache.arrow.flight.core=ALL-UNNAMED --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + + + + + + diff --git a/java/flight/flight-sql/src/main/java/module-info.java b/java/flight/flight-sql/src/main/java/module-info.java index e69de29bb2d..5514d5b870a 100644 --- a/java/flight/flight-sql/src/main/java/module-info.java +++ b/java/flight/flight-sql/src/main/java/module-info.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.flight.sql { + exports org.apache.arrow.flight.sql; + exports org.apache.arrow.flight.sql.example; + exports org.apache.arrow.flight.sql.util; + + requires com.google.common; + requires java.sql; + requires org.apache.arrow.flight.core; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; + requires protobuf.java; +} diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/TestFlightSql.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java similarity index 98% rename from java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/TestFlightSql.java rename to java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java index 948364a9200..a39736e939f 100644 --- a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/TestFlightSql.java +++ b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.flight; +package org.apache.arrow.flight.sql.test; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; @@ -38,6 +38,15 @@ import java.util.Optional; import java.util.stream.IntStream; +import org.apache.arrow.flight.CancelFlightInfoRequest; +import org.apache.arrow.flight.FlightClient; +import org.apache.arrow.flight.FlightInfo; +import org.apache.arrow.flight.FlightRuntimeException; +import org.apache.arrow.flight.FlightServer; +import org.apache.arrow.flight.FlightStatusCode; +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.RenewFlightEndpointRequest; import org.apache.arrow.flight.sql.FlightSqlClient; import org.apache.arrow.flight.sql.FlightSqlClient.PreparedStatement; import org.apache.arrow.flight.sql.FlightSqlColumnMetadata; @@ -914,7 +923,7 @@ public void testCancelFlightInfo() { FlightInfo info = sqlClient.getSqlInfo(); CancelFlightInfoRequest request = new CancelFlightInfoRequest(info); FlightRuntimeException fre = assertThrows(FlightRuntimeException.class, () -> sqlClient.cancelFlightInfo(request)); - assertEquals(FlightStatusCode.UNIMPLEMENTED, fre.status().code()); + Assertions.assertEquals(FlightStatusCode.UNIMPLEMENTED, fre.status().code()); } @Test diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/TestFlightSqlStreams.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java similarity index 96% rename from java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/TestFlightSqlStreams.java rename to java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java index 1dd925eb53a..1dd96f0fd4e 100644 --- a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/TestFlightSqlStreams.java +++ b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.flight; +package org.apache.arrow.flight.sql.test; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; @@ -28,6 +28,15 @@ import java.util.Collections; import java.util.List; +import org.apache.arrow.flight.CallStatus; +import org.apache.arrow.flight.FlightClient; +import org.apache.arrow.flight.FlightDescriptor; +import org.apache.arrow.flight.FlightEndpoint; +import org.apache.arrow.flight.FlightInfo; +import org.apache.arrow.flight.FlightServer; +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.Ticket; import org.apache.arrow.flight.sql.BasicFlightSqlProducer; import org.apache.arrow.flight.sql.FlightSqlClient; import org.apache.arrow.flight.sql.FlightSqlProducer; From a02bb424d306c720ad16799dad89a0c03b7f09ed Mon Sep 17 00:00:00 2001 From: James Duong Date: Wed, 13 Dec 2023 13:18:35 -0800 Subject: [PATCH 14/54] Restructure arrow-c-data for package conflicts --- java/c/src/main/java/org/apache/arrow/c/Data.java | 2 -- .../org/apache/arrow/{vector => c}/StructVectorLoader.java | 4 +++- .../org/apache/arrow/{vector => c}/StructVectorUnloader.java | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) rename java/c/src/main/java/org/apache/arrow/{vector => c}/StructVectorLoader.java (98%) rename java/c/src/main/java/org/apache/arrow/{vector => c}/StructVectorUnloader.java (97%) diff --git a/java/c/src/main/java/org/apache/arrow/c/Data.java b/java/c/src/main/java/org/apache/arrow/c/Data.java index a92853b3504..c90ce7604d6 100644 --- a/java/c/src/main/java/org/apache/arrow/c/Data.java +++ b/java/c/src/main/java/org/apache/arrow/c/Data.java @@ -19,8 +19,6 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.StructVectorLoader; -import org.apache.arrow.vector.StructVectorUnloader; import org.apache.arrow.vector.VectorLoader; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.VectorUnloader; diff --git a/java/c/src/main/java/org/apache/arrow/vector/StructVectorLoader.java b/java/c/src/main/java/org/apache/arrow/c/StructVectorLoader.java similarity index 98% rename from java/c/src/main/java/org/apache/arrow/vector/StructVectorLoader.java rename to java/c/src/main/java/org/apache/arrow/c/StructVectorLoader.java index 4a62be7851a..d9afd0189d8 100644 --- a/java/c/src/main/java/org/apache/arrow/vector/StructVectorLoader.java +++ b/java/c/src/main/java/org/apache/arrow/c/StructVectorLoader.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.arrow.vector; +package org.apache.arrow.c; import static org.apache.arrow.util.Preconditions.checkArgument; @@ -27,6 +27,8 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.Collections2; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.TypeLayout; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.compression.CompressionCodec; import org.apache.arrow.vector.compression.CompressionUtil; diff --git a/java/c/src/main/java/org/apache/arrow/vector/StructVectorUnloader.java b/java/c/src/main/java/org/apache/arrow/c/StructVectorUnloader.java similarity index 97% rename from java/c/src/main/java/org/apache/arrow/vector/StructVectorUnloader.java rename to java/c/src/main/java/org/apache/arrow/c/StructVectorUnloader.java index e75156cf237..aa6d9b4d0f6 100644 --- a/java/c/src/main/java/org/apache/arrow/vector/StructVectorUnloader.java +++ b/java/c/src/main/java/org/apache/arrow/c/StructVectorUnloader.java @@ -15,12 +15,14 @@ * limitations under the License. */ -package org.apache.arrow.vector; +package org.apache.arrow.c; import java.util.ArrayList; import java.util.List; import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.TypeLayout; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.compression.CompressionCodec; import org.apache.arrow.vector.compression.CompressionUtil; From 5519e725b6d30dabf7f36351d30264173eac7f89 Mon Sep 17 00:00:00 2001 From: James Duong Date: Wed, 13 Dec 2023 13:18:59 -0800 Subject: [PATCH 15/54] Modularize arrow-c-data - add module-info.java - add dependency on org.immutables.value --- java/c/pom.xml | 4 ++++ java/c/src/main/java/module-info.java | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 java/c/src/main/java/module-info.java diff --git a/java/c/pom.xml b/java/c/pom.xml index 8fc3f36994d..a999292979d 100644 --- a/java/c/pom.xml +++ b/java/c/pom.xml @@ -48,6 +48,10 @@ org.slf4j slf4j-api + + org.immutables + value + org.apache.arrow arrow-memory-unsafe diff --git a/java/c/src/main/java/module-info.java b/java/c/src/main/java/module-info.java new file mode 100644 index 00000000000..d5e6679a69d --- /dev/null +++ b/java/c/src/main/java/module-info.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module org.apache.arrow.c { + exports org.apache.arrow.c; + exports org.apache.arrow.c.jni; + + requires flatbuffers.java; + requires jdk.unsupported; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; + requires org.slf4j; +} From 724c700bc8c5bab639ab62c16e8a470ce186c93a Mon Sep 17 00:00:00 2001 From: James Duong Date: Wed, 13 Dec 2023 13:47:28 -0800 Subject: [PATCH 16/54] Make arrow-c-data an open module This fixes problems loading the native library --- java/c/src/main/java/module-info.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/c/src/main/java/module-info.java b/java/c/src/main/java/module-info.java index d5e6679a69d..0a62c9b9875 100644 --- a/java/c/src/main/java/module-info.java +++ b/java/c/src/main/java/module-info.java @@ -15,7 +15,7 @@ * limitations under the License. */ -module org.apache.arrow.c { +open module org.apache.arrow.c { exports org.apache.arrow.c; exports org.apache.arrow.c.jni; From 54cb6be97088cdcce7ca87591a063862fde3d1c2 Mon Sep 17 00:00:00 2001 From: James Duong Date: Wed, 13 Dec 2023 14:16:20 -0800 Subject: [PATCH 17/54] Modularize the Orc adapter - add module-info.java. Set to be an open module for native libraries. - add reference to immutables to pom.xml --- java/adapter/orc/pom.xml | 4 ++++ .../orc/src/main/java/module-info.java | 24 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 java/adapter/orc/src/main/java/module-info.java diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml index a42a458e207..605b9871639 100644 --- a/java/adapter/orc/pom.xml +++ b/java/adapter/orc/pom.xml @@ -31,6 +31,10 @@ compile ${arrow.vector.classifier} + + org.immutables + value + org.apache.orc orc-core diff --git a/java/adapter/orc/src/main/java/module-info.java b/java/adapter/orc/src/main/java/module-info.java new file mode 100644 index 00000000000..d18a978e93f --- /dev/null +++ b/java/adapter/orc/src/main/java/module-info.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +open module org.apache.arrow.adapter.orc { + exports org.apache.arrow.adapter.orc; + + requires hadoop.client.api; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; +} From d17b958ae57397c99271ece88eadaaa3381be90f Mon Sep 17 00:00:00 2001 From: James Duong Date: Wed, 13 Dec 2023 14:59:30 -0800 Subject: [PATCH 18/54] Modularize gandiva - Add module-info.java and make it an open module - Add dependency on org.immutables.value --- java/gandiva/pom.xml | 4 +++ java/gandiva/src/main/java/module-info.java | 29 +++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 java/gandiva/src/main/java/module-info.java diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml index e837a09ff83..330c156a034 100644 --- a/java/gandiva/pom.xml +++ b/java/gandiva/pom.xml @@ -34,6 +34,10 @@ org.apache.arrow arrow-memory-core + + org.immutables + value + org.apache.arrow arrow-memory-netty diff --git a/java/gandiva/src/main/java/module-info.java b/java/gandiva/src/main/java/module-info.java new file mode 100644 index 00000000000..533717d91f7 --- /dev/null +++ b/java/gandiva/src/main/java/module-info.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +open module org.apache.arrow.gandiva { + exports org.apache.arrow.gandiva.expression; + exports org.apache.arrow.gandiva.exceptions; + exports org.apache.arrow.gandiva.evaluator; + exports org.apache.arrow.gandiva.ipc; + + requires com.google.common; + requires org.apache.arrow.format; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; + requires org.slf4j; + requires protobuf.java; +} \ No newline at end of file From b638339c611e5a6f3a93da2ae5780b351718ea34 Mon Sep 17 00:00:00 2001 From: James Duong Date: Wed, 13 Dec 2023 14:25:50 -0800 Subject: [PATCH 19/54] Modularize dataset (not working due to collision with arrow-c-data for the location of the native lib) --- java/dataset/pom.xml | 13 +++++++++ java/dataset/src/main/java/module-info.java | 29 +++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 java/dataset/src/main/java/module-info.java diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml index bb5636b7454..13aa59a1424 100644 --- a/java/dataset/pom.xml +++ b/java/dataset/pom.xml @@ -47,6 +47,10 @@ arrow-c-data compile + + org.immutables + value + org.apache.arrow arrow-memory-netty @@ -161,6 +165,15 @@ + + maven-surefire-plugin + + false + + ${project.basedir}/../../testing/data + + + org.xolstice.maven.plugins protobuf-maven-plugin diff --git a/java/dataset/src/main/java/module-info.java b/java/dataset/src/main/java/module-info.java new file mode 100644 index 00000000000..1672d12ffec --- /dev/null +++ b/java/dataset/src/main/java/module-info.java @@ -0,0 +1,29 @@ +/* + + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +open module org.apache.arrow.dataset { + exports org.apache.arrow.dataset.file; + exports org.apache.arrow.dataset.source; + exports org.apache.arrow.dataset.jni; + exports org.apache.arrow.dataset.substrait; + exports org.apache.arrow.dataset.scanner; + + requires org.apache.arrow.c; + requires org.apache.arrow.memory.core; + requires org.apache.arrow.vector; +} From e1b79003a696c9f7c1a92e9b9089194ada6c1488 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 5 Jan 2024 16:59:07 -0800 Subject: [PATCH 20/54] Remove reference to failureaccess modue --- java/flight/flight-core/src/main/java/module-info.java | 1 - 1 file changed, 1 deletion(-) diff --git a/java/flight/flight-core/src/main/java/module-info.java b/java/flight/flight-core/src/main/java/module-info.java index e76f22d6168..f6bf5b73b09 100644 --- a/java/flight/flight-core/src/main/java/module-info.java +++ b/java/flight/flight-core/src/main/java/module-info.java @@ -26,7 +26,6 @@ requires com.fasterxml.jackson.databind; requires com.google.common; requires com.google.errorprone.annotations; - requires failureaccess; requires io.grpc; requires io.grpc.internal; requires io.grpc.netty; From 656a5d0a28f81c204547f819fe670d3941bce21d Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 10:47:27 -0800 Subject: [PATCH 21/54] Move native binaries to a sub-folder based on library name Move native binaries to a sub-folder based on the library name instead of directly to a folder based on architecture. This is because JPMS is treating any folder as potentially a package name and if we have the same folder name appear across modules it gets treated as a package conflict. --- java/adapter/orc/pom.xml | 21 ++++++++++++++++++- .../apache/arrow/adapter/orc/OrcJniUtils.java | 2 +- java/c/pom.xml | 21 ++++++++++++++++++- .../org/apache/arrow/c/jni/JniLoader.java | 2 +- java/dataset/pom.xml | 19 ++++++++++++++++- .../apache/arrow/dataset/jni/JniLoader.java | 2 +- java/gandiva/pom.xml | 19 ++++++++++++++++- .../arrow/gandiva/evaluator/JniLoader.java | 2 +- java/pom.xml | 5 +++++ 9 files changed, 85 insertions(+), 8 deletions(-) diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml index 605b9871639..12ee1065ca4 100644 --- a/java/adapter/orc/pom.xml +++ b/java/adapter/orc/pom.xml @@ -131,11 +131,30 @@ - ${arrow.cpp.build.dir} + ${arrow.cpp.build.dir}/arrow_orc_jni **/*arrow_orc_jni.* + + + com.coderplus.maven.plugins + copy-rename-maven-plugin + + + copy-file + generate-sources + + copy + + + ${arrow.cpp.build.dir} + ${arrow.cpp.build.dir}/arrow_orc_jni + + + + + diff --git a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java index 2701c228709..9b599234bdf 100644 --- a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java +++ b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java @@ -39,7 +39,7 @@ static void loadOrcAdapterLibraryFromJar() synchronized (OrcJniUtils.class) { if (!isLoaded) { final String libraryToLoad = - getNormalizedArch() + "/" + System.mapLibraryName(LIBRARY_NAME); + LIBRARY_NAME + "/" + getNormalizedArch() + "/" + System.mapLibraryName(LIBRARY_NAME); final File libraryFile = moveFileFromJarToTemp(System.getProperty("java.io.tmpdir"), libraryToLoad, LIBRARY_NAME); System.load(libraryFile.getAbsolutePath()); diff --git a/java/c/pom.xml b/java/c/pom.xml index a999292979d..2fd39a7ecbe 100644 --- a/java/c/pom.xml +++ b/java/c/pom.xml @@ -76,12 +76,31 @@ - ${arrow.c.jni.dist.dir} + ${arrow.c.jni.dist.dir}/arrow_cdata_jni **/*arrow_cdata_jni.* + + + com.coderplus.maven.plugins + copy-rename-maven-plugin + + + copy-file + generate-sources + + copy + + + ${arrow.cpp.build.dir} + ${arrow.cpp.build.dir}/arrow_cdata_jni + + + + + diff --git a/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java b/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java index e4354613492..ef9f432cf00 100644 --- a/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java +++ b/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java @@ -80,7 +80,7 @@ private synchronized void loadRemaining() { private void load(String name) { final String libraryToLoad = - getNormalizedArch() + "/" + System.mapLibraryName(name); + name + "/" + getNormalizedArch() + "/" + System.mapLibraryName(name); try { File temp = File.createTempFile("jnilib-", ".tmp", new File(System.getProperty("java.io.tmpdir"))); temp.deleteOnExit(); diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml index 13aa59a1424..8d566c0be22 100644 --- a/java/dataset/pom.xml +++ b/java/dataset/pom.xml @@ -157,7 +157,7 @@ - ${arrow.cpp.build.dir} + ${arrow.cpp.build.dir}/arrow_dataset_jni **/*arrow_dataset_jni.* @@ -165,6 +165,23 @@ + + com.coderplus.maven.plugins + copy-rename-maven-plugin + + + copy-file + generate-sources + + copy + + + ${arrow.cpp.build.dir} + ${arrow.cpp.build.dir}/arrow_dataset_jni + + + + maven-surefire-plugin diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java index a3b31c73e85..cf2f8fe29e8 100644 --- a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java +++ b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java @@ -79,7 +79,7 @@ private synchronized void loadRemaining() { private void load(String name) { final String libraryToLoad = - getNormalizedArch() + "/" + System.mapLibraryName(name); + name + "/" + getNormalizedArch() + "/" + System.mapLibraryName(name); try { File temp = File.createTempFile("jnilib-", ".tmp", new File(System.getProperty("java.io.tmpdir"))); temp.deleteOnExit(); diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml index 330c156a034..46736078d55 100644 --- a/java/gandiva/pom.xml +++ b/java/gandiva/pom.xml @@ -114,7 +114,7 @@ - ${arrow.cpp.build.dir} + ${arrow.cpp.build.dir}/gandiva_jni **/*gandiva_jni.* @@ -129,6 +129,23 @@ + + com.coderplus.maven.plugins + copy-rename-maven-plugin + + + copy-file + generate-sources + + copy + + + ${arrow.cpp.build.dir} + ${arrow.cpp.build.dir}/gandiva_jni + + + + org.xolstice.maven.plugins protobuf-maven-plugin diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java index 2528989f378..57748e9c8e1 100644 --- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java +++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java @@ -71,7 +71,7 @@ private static JniLoader setupInstance() throws GandivaException { private static void loadGandivaLibraryFromJar(final String tmpDir) throws IOException, GandivaException { final String libraryToLoad = - getNormalizedArch() + "/" + System.mapLibraryName(LIBRARY_NAME); + LIBRARY_NAME + "/" + getNormalizedArch() + "/" + System.mapLibraryName(LIBRARY_NAME); final File libraryFile = moveFileFromJarToTemp(tmpDir, libraryToLoad, LIBRARY_NAME); System.load(libraryFile.getAbsolutePath()); } diff --git a/java/pom.xml b/java/pom.xml index b2513d58626..4b0b11feb98 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -605,6 +605,11 @@ maven-site-plugin 3.7.1 + + com.coderplus.maven.plugins + copy-rename-maven-plugin + 1.0 + From 9147a7907537141b8b8726c1d870517e27b3084d Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 10:47:27 -0800 Subject: [PATCH 22/54] Move native binaries to a sub-folder based on library name Move native binaries to a sub-folder based on the library name instead of directly to a folder based on architecture. This is because JPMS is treating any folder as potentially a package name and if we have the same folder name appear across modules it gets treated as a package conflict. --- java/adapter/orc/CMakeLists.txt | 2 ++ java/c/CMakeLists.txt | 2 ++ java/dataset/CMakeLists.txt | 2 ++ java/gandiva/CMakeLists.txt | 2 ++ java/pom.xml | 10 +++++----- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/java/adapter/orc/CMakeLists.txt b/java/adapter/orc/CMakeLists.txt index a9b3a480279..c1f57eb8808 100644 --- a/java/adapter/orc/CMakeLists.txt +++ b/java/adapter/orc/CMakeLists.txt @@ -37,6 +37,8 @@ set_property(TARGET arrow_java_jni_orc PROPERTY OUTPUT_NAME "arrow_orc_jni") target_link_libraries(arrow_java_jni_orc arrow_java_jni_orc_headers jni Arrow::arrow_static) +set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_orc_jni/${CMAKE_ARCH_DIR}") + install(TARGETS arrow_java_jni_orc LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/java/c/CMakeLists.txt b/java/c/CMakeLists.txt index 8ff208aaeb0..9ae1b31be2d 100644 --- a/java/c/CMakeLists.txt +++ b/java/c/CMakeLists.txt @@ -30,6 +30,8 @@ add_library(arrow_java_jni_cdata SHARED src/main/cpp/jni_wrapper.cc) set_property(TARGET arrow_java_jni_cdata PROPERTY OUTPUT_NAME "arrow_cdata_jni") target_link_libraries(arrow_java_jni_cdata arrow_java_jni_cdata_headers jni) +set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_cdata_jni/${CMAKE_ARCH_DIR}") + install(TARGETS arrow_java_jni_cdata LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/java/dataset/CMakeLists.txt b/java/dataset/CMakeLists.txt index ede3ee7330d..9a10a4e60e0 100644 --- a/java/dataset/CMakeLists.txt +++ b/java/dataset/CMakeLists.txt @@ -47,6 +47,8 @@ if(BUILD_TESTING) add_test(NAME arrow-java-jni-dataset-test COMMAND arrow-java-jni-dataset-test) endif() +set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_dataset_jni/${CMAKE_ARCH_DIR}") + install(TARGETS arrow_java_jni_dataset LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/java/gandiva/CMakeLists.txt b/java/gandiva/CMakeLists.txt index 2aa8d92959e..84147e58dba 100644 --- a/java/gandiva/CMakeLists.txt +++ b/java/gandiva/CMakeLists.txt @@ -84,6 +84,8 @@ if(CXX_LINKER_SUPPORTS_VERSION_SCRIPT) ) endif() +set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/gandiva_jni/${CMAKE_ARCH_DIR}") + install(TARGETS arrow_java_jni_gandiva LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/java/pom.xml b/java/pom.xml index 4b0b11feb98..5f9cfe8811f 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -1060,7 +1060,7 @@ -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib/${os.detected.arch} + -DCMAKE_ARCH_DIR=${os.detected.arch} -DCMAKE_INSTALL_PREFIX=${arrow.c.jni.dist.dir} ../ @@ -1133,7 +1133,7 @@ -DARROW_SUBSTRAIT=${ARROW_DATASET} -DARROW_USE_CCACHE=ON -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib/${os.detected.arch} + -DCMAKE_ARCH_DIR=${os.detected.arch} -DCMAKE_INSTALL_PREFIX=java-dist -DCMAKE_UNITY_BUILD=ON @@ -1174,7 +1174,7 @@ -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib/${os.detected.arch} + -DCMAKE_ARCH_DIR=${os.detected.arch} -DCMAKE_INSTALL_PREFIX=${arrow.dataset.jni.dist.dir} -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist/lib/${os.detected.arch}/cmake -DProtobuf_USE_STATIC_LIBS=ON @@ -1253,7 +1253,7 @@ -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib/${os.detected.arch} + -DCMAKE_ARCH_DIR=${os.detected.arch} -DCMAKE_INSTALL_PREFIX=java-dist -DCMAKE_UNITY_BUILD=ON -GNinja @@ -1295,7 +1295,7 @@ -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib/${os.detected.arch} + -DCMAKE_ARCH_DIR=${os.detected.arch} -DCMAKE_INSTALL_PREFIX=${arrow.dataset.jni.dist.dir} -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist/lib/${os.detected.arch}/cmake From e97e6b3634c8e497cf8ea7a3bcb0f96bfa6804d4 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 13:05:53 -0800 Subject: [PATCH 23/54] Rename error-prone-jdk11+ profiles to jdk11+ --- java/adapter/jdbc/pom.xml | 2 +- java/flight/flight-core/pom.xml | 2 +- java/flight/flight-sql/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index 1cb1fb744fb..e964aa1871a 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -92,7 +92,7 @@ - error-prone-jdk11+ + jdk11+ [11,] diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index 7caa039eb74..ec3034d14e2 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -317,7 +317,7 @@ - error-prone-jdk11+ + jdk11+ [11,] diff --git a/java/flight/flight-sql/pom.xml b/java/flight/flight-sql/pom.xml index 7ccab8e813c..5ae1a0a23f3 100644 --- a/java/flight/flight-sql/pom.xml +++ b/java/flight/flight-sql/pom.xml @@ -116,7 +116,7 @@ - error-prone-jdk11+ + jdk11+ [11,] From ed30423a948e11b3ccdacb85e67b8600e866e254 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 13:06:20 -0800 Subject: [PATCH 24/54] Add Jackson databind module when running dataset tests --- java/dataset/pom.xml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml index 8d566c0be22..faf910a6680 100644 --- a/java/dataset/pom.xml +++ b/java/dataset/pom.xml @@ -212,4 +212,30 @@ + + + jdk11+ + + [11,] + + !m2e.version + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + ${project.basedir}/../../testing/data + + --add-reads=org.apache.arrow.dataset=com.fasterxml.jackson.databind --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + + + + + From 4e4ef552c30dde5002e37aa3fa6ee096f93967f6 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 13:14:12 -0800 Subject: [PATCH 25/54] Make loading test schemas compatible with modules in Dataset --- .../arrow/dataset/ParquetWriteSupport.java | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/java/dataset/src/test/java/org/apache/arrow/dataset/ParquetWriteSupport.java b/java/dataset/src/test/java/org/apache/arrow/dataset/ParquetWriteSupport.java index 75f905877cd..2352a65e8fb 100644 --- a/java/dataset/src/test/java/org/apache/arrow/dataset/ParquetWriteSupport.java +++ b/java/dataset/src/test/java/org/apache/arrow/dataset/ParquetWriteSupport.java @@ -18,8 +18,8 @@ package org.apache.arrow.dataset; import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.io.InputStream; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -47,7 +47,7 @@ public class ParquetWriteSupport implements AutoCloseable { public ParquetWriteSupport(String schemaName, File outputFolder) throws Exception { - avroSchema = readSchemaFromFile(schemaName); + avroSchema = getSchema(schemaName); path = outputFolder.getPath() + "/" + "generated-" + random.nextLong() + ".parquet"; uri = "file://" + path; writer = AvroParquetWriter @@ -56,10 +56,23 @@ public ParquetWriteSupport(String schemaName, File outputFolder) throws Exceptio .build(); } - private static Schema readSchemaFromFile(String schemaName) throws Exception { - Path schemaPath = Paths.get(ParquetWriteSupport.class.getResource("/").getPath(), - "avroschema", schemaName); - return new org.apache.avro.Schema.Parser().parse(schemaPath.toFile()); + public static Schema getSchema(String schemaName) throws Exception { + try { + // Attempt to use JDK 9 behavior of getting the module then the resource stream from the module. + // Note that this code is caller-sensitive. + Method getModuleMethod = Class.class.getMethod("getModule"); + Object module = getModuleMethod.invoke(ParquetWriteSupport.class); + Method getResourceAsStreamFromModule = module.getClass().getMethod("getResourceAsStream", String.class); + try (InputStream is = (InputStream) getResourceAsStreamFromModule.invoke(module, "/avroschema/" + schemaName)) { + return new Schema.Parser() + .parse(is); + } + } catch (NoSuchMethodException ex) { + // Use JDK8 behavior. + try (InputStream is = ParquetWriteSupport.class.getResourceAsStream("/avroschema/" + schemaName)) { + return new Schema.Parser().parse(is); + } + } } public static ParquetWriteSupport writeTempFile(String schemaName, File outputFolder, From aebac892d88d4d092f6dfbe636d8e9d7178e9425 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 13:15:53 -0800 Subject: [PATCH 26/54] Allow dataset tests to use reflection on native functions --- java/dataset/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml index faf910a6680..187c5a055cf 100644 --- a/java/dataset/pom.xml +++ b/java/dataset/pom.xml @@ -231,7 +231,7 @@ ${project.basedir}/../../testing/data - --add-reads=org.apache.arrow.dataset=com.fasterxml.jackson.databind --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + --add-reads=org.apache.arrow.dataset=com.fasterxml.jackson.databind --add-opens=java.base/java.nio=org.apache.arrow.dataset,org.apache.arrow.memory.core,ALL-UNNAMED From a0f416e578f42913040a2218d9978a56e8f7fe89 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 13:20:48 -0800 Subject: [PATCH 27/54] Revert "Allow dataset tests to use reflection on native functions" This reverts commit 379bf53d3eecf86f7e9819c0843edf0b287e57ae. --- java/dataset/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml index 187c5a055cf..faf910a6680 100644 --- a/java/dataset/pom.xml +++ b/java/dataset/pom.xml @@ -231,7 +231,7 @@ ${project.basedir}/../../testing/data - --add-reads=org.apache.arrow.dataset=com.fasterxml.jackson.databind --add-opens=java.base/java.nio=org.apache.arrow.dataset,org.apache.arrow.memory.core,ALL-UNNAMED + --add-reads=org.apache.arrow.dataset=com.fasterxml.jackson.databind --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED From 167e57c85e6c19153b544b6a9feccc1292fcb505 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 13:23:50 -0800 Subject: [PATCH 28/54] Update required flags - Add a friendly exception to dataset - Update tester_java.py for flight flags - Update documentation --- .../archery/integration/tester_java.py | 3 +- docs/source/java/install.rst | 35 ++++++++++++++++++- .../jni/DirectReservationListener.java | 6 +++- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/dev/archery/archery/integration/tester_java.py b/dev/archery/archery/integration/tester_java.py index 857fe0c50af..dea642fa9c2 100644 --- a/dev/archery/archery/integration/tester_java.py +++ b/dev/archery/archery/integration/tester_java.py @@ -258,7 +258,8 @@ def __init__(self, *args, **kwargs): # Java 9+ self._java_opts.append( '--add-opens=java.base/java.nio=' - 'org.apache.arrow.memory.core,ALL-UNNAMED') + 'org.apache.arrow.memory.core,ALL-UNNAMED' + '--add-reads=org.apache.arrow.flight.core=ALL-UNNAMED') def _run(self, arrow_path=None, json_path=None, command='VALIDATE'): cmd = ( diff --git a/docs/source/java/install.rst b/docs/source/java/install.rst index b7484536f23..783687fb1f7 100644 --- a/docs/source/java/install.rst +++ b/docs/source/java/install.rst @@ -43,7 +43,40 @@ adding ``--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED $ env _JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" java -jar ... Otherwise, you may see errors like ``module java.base does not "opens -java.nio" to unnamed module``. +java.nio" to unnamed module`` or ``module java.base does not "opens +java.nio" to org.apache.arrow.memory.core`` + +Note that the command has changed from Arrow 15 and earlier. If you are still using the flags from that version +(``--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED``) you will see the +``module java.base does not "opens java.nio" to org.apache.arrow.memory.core`` error. + +If you are using flight-core or dependent modules, you will need to mark that flight-core can read unnamed modules. +Modifying the command above for Flight: + +.. code-block:: shell + + # Directly on the command line + $ java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED -jar ... + # Indirectly via environment variables + $ env _JAVA_OPTIONS="--add-reads=org.apache.arrow.flight.core=ALL-UNNAMED --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" java -jar ... + +Otherwise, you may see errors like ``java.lang.IllegalAccessError: superclass access check failed: class +org.apache.arrow.flight.ArrowMessage$ArrowBufRetainingCompositeByteBuf (in module org.apache.arrow.flight.core) +cannot access class io.netty.buffer.CompositeByteBuf (in unnamed module ...) because module +org.apache.arrow.flight.core does not read unnamed module ... + +Finally, if you are using arrow-dataset, you'll also need to report that JDK internals need to be exposed. +Modifying the command above for arrow-memory: +.. code-block:: shell + + # Directly on the command line + $ java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED -jar ... + # Indirectly via environment variables + $ env _JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.dataset,org.apache.arrow.memory.core,ALL-UNNAMED" java -jar ... + +Otherwise you may see errors such as ``java.lang.RuntimeException: java.lang.reflect.InaccessibleObjectException: +Unable to make static void java.nio.Bits.reserveMemory(long,long) accessible: module +java.base does not "opens java.nio" to module org.apache.arrow.dataset`` If using Maven and Surefire for unit testing, :ref:`this argument must be added to Surefire as well `. diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java index eb26400cbf8..f61637e0800 100644 --- a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java +++ b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java @@ -40,7 +40,11 @@ private DirectReservationListener() { methodUnreserve = this.getDeclaredMethodBaseOnJDKVersion(classBits, "unreserveMemory"); methodUnreserve.setAccessible(true); } catch (Exception e) { - throw new RuntimeException(e); + final RuntimeException failure = new RuntimeException( + "Failed to initialize DirectReservationListener. When starting Java you must include " + + "`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` " + + "(See https://arrow.apache.org/docs/java/install.html)", e); + failure.printStackTrace(); } } From 1a97294366ef49f9013769a12aeffa1fe52a85c5 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 13:46:57 -0800 Subject: [PATCH 29/54] Fix line length in CMakeLists changes --- java/adapter/orc/CMakeLists.txt | 3 ++- java/c/CMakeLists.txt | 3 ++- java/dataset/CMakeLists.txt | 3 ++- java/gandiva/CMakeLists.txt | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/java/adapter/orc/CMakeLists.txt b/java/adapter/orc/CMakeLists.txt index c1f57eb8808..d4cafd241fb 100644 --- a/java/adapter/orc/CMakeLists.txt +++ b/java/adapter/orc/CMakeLists.txt @@ -37,7 +37,8 @@ set_property(TARGET arrow_java_jni_orc PROPERTY OUTPUT_NAME "arrow_orc_jni") target_link_libraries(arrow_java_jni_orc arrow_java_jni_orc_headers jni Arrow::arrow_static) -set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_orc_jni/${CMAKE_ARCH_DIR}") +set(CMAKE_INSTALL_LIBDIR + "${CMAKE_INSTALL_PREFIX}/lib/arrow_orc_jni/${CMAKE_ARCH_DIR}") install(TARGETS arrow_java_jni_orc LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/java/c/CMakeLists.txt b/java/c/CMakeLists.txt index 9ae1b31be2d..0c5f1488fdc 100644 --- a/java/c/CMakeLists.txt +++ b/java/c/CMakeLists.txt @@ -30,7 +30,8 @@ add_library(arrow_java_jni_cdata SHARED src/main/cpp/jni_wrapper.cc) set_property(TARGET arrow_java_jni_cdata PROPERTY OUTPUT_NAME "arrow_cdata_jni") target_link_libraries(arrow_java_jni_cdata arrow_java_jni_cdata_headers jni) -set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_cdata_jni/${CMAKE_ARCH_DIR}") +set(CMAKE_INSTALL_LIBDIR + "${CMAKE_INSTALL_PREFIX}/lib/arrow_cdata_jni/${CMAKE_ARCH_DIR}") install(TARGETS arrow_java_jni_cdata LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/java/dataset/CMakeLists.txt b/java/dataset/CMakeLists.txt index 9a10a4e60e0..4c40e9fad2e 100644 --- a/java/dataset/CMakeLists.txt +++ b/java/dataset/CMakeLists.txt @@ -47,7 +47,8 @@ if(BUILD_TESTING) add_test(NAME arrow-java-jni-dataset-test COMMAND arrow-java-jni-dataset-test) endif() -set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_dataset_jni/${CMAKE_ARCH_DIR}") +set(CMAKE_INSTALL_LIBDIR + "${CMAKE_INSTALL_PREFIX}/lib/arrow_dataset_jni/${CMAKE_ARCH_DIR}") install(TARGETS arrow_java_jni_dataset LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/java/gandiva/CMakeLists.txt b/java/gandiva/CMakeLists.txt index 84147e58dba..0de71d55310 100644 --- a/java/gandiva/CMakeLists.txt +++ b/java/gandiva/CMakeLists.txt @@ -84,7 +84,8 @@ if(CXX_LINKER_SUPPORTS_VERSION_SCRIPT) ) endif() -set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/gandiva_jni/${CMAKE_ARCH_DIR}") +set(CMAKE_INSTALL_LIBDIR + "${CMAKE_INSTALL_PREFIX}/lib/gandiva_jni/${CMAKE_ARCH_DIR}") install(TARGETS arrow_java_jni_gandiva LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} From 6f544e91b009f9bae703982a627b91c17c1c4692 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 14:00:23 -0800 Subject: [PATCH 30/54] Archery linting on CMake files --- java/adapter/orc/CMakeLists.txt | 3 +-- java/c/CMakeLists.txt | 3 +-- java/dataset/CMakeLists.txt | 2 +- java/gandiva/CMakeLists.txt | 3 +-- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/java/adapter/orc/CMakeLists.txt b/java/adapter/orc/CMakeLists.txt index d4cafd241fb..c1f57eb8808 100644 --- a/java/adapter/orc/CMakeLists.txt +++ b/java/adapter/orc/CMakeLists.txt @@ -37,8 +37,7 @@ set_property(TARGET arrow_java_jni_orc PROPERTY OUTPUT_NAME "arrow_orc_jni") target_link_libraries(arrow_java_jni_orc arrow_java_jni_orc_headers jni Arrow::arrow_static) -set(CMAKE_INSTALL_LIBDIR - "${CMAKE_INSTALL_PREFIX}/lib/arrow_orc_jni/${CMAKE_ARCH_DIR}") +set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_orc_jni/${CMAKE_ARCH_DIR}") install(TARGETS arrow_java_jni_orc LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/java/c/CMakeLists.txt b/java/c/CMakeLists.txt index 0c5f1488fdc..9ae1b31be2d 100644 --- a/java/c/CMakeLists.txt +++ b/java/c/CMakeLists.txt @@ -30,8 +30,7 @@ add_library(arrow_java_jni_cdata SHARED src/main/cpp/jni_wrapper.cc) set_property(TARGET arrow_java_jni_cdata PROPERTY OUTPUT_NAME "arrow_cdata_jni") target_link_libraries(arrow_java_jni_cdata arrow_java_jni_cdata_headers jni) -set(CMAKE_INSTALL_LIBDIR - "${CMAKE_INSTALL_PREFIX}/lib/arrow_cdata_jni/${CMAKE_ARCH_DIR}") +set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_cdata_jni/${CMAKE_ARCH_DIR}") install(TARGETS arrow_java_jni_cdata LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/java/dataset/CMakeLists.txt b/java/dataset/CMakeLists.txt index 4c40e9fad2e..620dfb10b27 100644 --- a/java/dataset/CMakeLists.txt +++ b/java/dataset/CMakeLists.txt @@ -48,7 +48,7 @@ if(BUILD_TESTING) endif() set(CMAKE_INSTALL_LIBDIR - "${CMAKE_INSTALL_PREFIX}/lib/arrow_dataset_jni/${CMAKE_ARCH_DIR}") + "${CMAKE_INSTALL_PREFIX}/lib/arrow_dataset_jni/${CMAKE_ARCH_DIR}") install(TARGETS arrow_java_jni_dataset LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/java/gandiva/CMakeLists.txt b/java/gandiva/CMakeLists.txt index 0de71d55310..84147e58dba 100644 --- a/java/gandiva/CMakeLists.txt +++ b/java/gandiva/CMakeLists.txt @@ -84,8 +84,7 @@ if(CXX_LINKER_SUPPORTS_VERSION_SCRIPT) ) endif() -set(CMAKE_INSTALL_LIBDIR - "${CMAKE_INSTALL_PREFIX}/lib/gandiva_jni/${CMAKE_ARCH_DIR}") +set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/gandiva_jni/${CMAKE_ARCH_DIR}") install(TARGETS arrow_java_jni_gandiva LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} From 9fee14b3e401e98b18704827af9a68057677c8be Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 14:26:14 -0800 Subject: [PATCH 31/54] Attempt to fix tool calls --- ci/scripts/java_jni_macos_build.sh | 8 ++++---- ci/scripts/java_jni_manylinux_build.sh | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index d66c39a37c5..b0fc52e42cd 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -138,8 +138,8 @@ archery linking check-dependencies \ --allow libncurses \ --allow libobjc \ --allow libz \ - libarrow_cdata_jni.dylib \ - libarrow_dataset_jni.dylib \ - libarrow_orc_jni.dylib \ - libgandiva_jni.dylib + arrow_cdata_jni/libarrow_cdata_jni.dylib \ + arrow_dataset_jni/libarrow_dataset_jni.dylib \ + arrow_orc_jni/libarrow_orc_jni.dylib \ + gandiva_jni/libgandiva_jni.dylib popd diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh index 03939715e39..a15eb89f658 100755 --- a/ci/scripts/java_jni_manylinux_build.sh +++ b/ci/scripts/java_jni_manylinux_build.sh @@ -164,8 +164,8 @@ archery linking check-dependencies \ --allow libstdc++ \ --allow libz \ --allow linux-vdso \ - libarrow_cdata_jni.so \ - libarrow_dataset_jni.so \ - libarrow_orc_jni.so \ - libgandiva_jni.so + arrow_cdata_jni/libarrow_cdata_jni.so \ + arrow_dataset_jni/libarrow_dataset_jni.so \ + arrow_orc_jni/libarrow_orc_jni.so \ + gandiva_jni/libgandiva_jni.so popd From 4f06a3d2ee6cea5eae8488dbf71f8059ef0d44c3 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 14:45:40 -0800 Subject: [PATCH 32/54] Fix a missed throw call --- .../org/apache/arrow/dataset/jni/DirectReservationListener.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java index f61637e0800..1a0325f9e67 100644 --- a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java +++ b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java @@ -45,6 +45,7 @@ private DirectReservationListener() { "`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` " + "(See https://arrow.apache.org/docs/java/install.html)", e); failure.printStackTrace(); + throw failure; } } From 4fc31b6b06b3bf10d3520e3c33f29d7716aa9c94 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 14:53:37 -0800 Subject: [PATCH 33/54] Correct the exception message --- .../org/apache/arrow/dataset/jni/DirectReservationListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java index 1a0325f9e67..3922e90335d 100644 --- a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java +++ b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/DirectReservationListener.java @@ -42,7 +42,7 @@ private DirectReservationListener() { } catch (Exception e) { final RuntimeException failure = new RuntimeException( "Failed to initialize DirectReservationListener. When starting Java you must include " + - "`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` " + + "`--add-opens=java.base/java.nio=org.apache.arrow.dataset,org.apache.arrow.memory.core,ALL-UNNAMED` " + "(See https://arrow.apache.org/docs/java/install.html)", e); failure.printStackTrace(); throw failure; From 6a9a5dbe9f1bb073681ec398d17235396f657a4c Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 15:43:27 -0800 Subject: [PATCH 34/54] Revert accidental inclusion of copy-rename plugin --- java/adapter/orc/pom.xml | 21 +-------------------- java/c/pom.xml | 21 +-------------------- java/dataset/pom.xml | 19 +------------------ java/gandiva/pom.xml | 19 +------------------ java/pom.xml | 5 ----- 5 files changed, 4 insertions(+), 81 deletions(-) diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml index 12ee1065ca4..605b9871639 100644 --- a/java/adapter/orc/pom.xml +++ b/java/adapter/orc/pom.xml @@ -131,30 +131,11 @@ - ${arrow.cpp.build.dir}/arrow_orc_jni + ${arrow.cpp.build.dir} **/*arrow_orc_jni.* - - - com.coderplus.maven.plugins - copy-rename-maven-plugin - - - copy-file - generate-sources - - copy - - - ${arrow.cpp.build.dir} - ${arrow.cpp.build.dir}/arrow_orc_jni - - - - - diff --git a/java/c/pom.xml b/java/c/pom.xml index 2fd39a7ecbe..a999292979d 100644 --- a/java/c/pom.xml +++ b/java/c/pom.xml @@ -76,31 +76,12 @@ - ${arrow.c.jni.dist.dir}/arrow_cdata_jni + ${arrow.c.jni.dist.dir} **/*arrow_cdata_jni.* - - - com.coderplus.maven.plugins - copy-rename-maven-plugin - - - copy-file - generate-sources - - copy - - - ${arrow.cpp.build.dir} - ${arrow.cpp.build.dir}/arrow_cdata_jni - - - - - diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml index faf910a6680..a18f443b7e1 100644 --- a/java/dataset/pom.xml +++ b/java/dataset/pom.xml @@ -157,7 +157,7 @@ - ${arrow.cpp.build.dir}/arrow_dataset_jni + ${arrow.cpp.build.dir} **/*arrow_dataset_jni.* @@ -165,23 +165,6 @@ - - com.coderplus.maven.plugins - copy-rename-maven-plugin - - - copy-file - generate-sources - - copy - - - ${arrow.cpp.build.dir} - ${arrow.cpp.build.dir}/arrow_dataset_jni - - - - maven-surefire-plugin diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml index 46736078d55..330c156a034 100644 --- a/java/gandiva/pom.xml +++ b/java/gandiva/pom.xml @@ -114,7 +114,7 @@ - ${arrow.cpp.build.dir}/gandiva_jni + ${arrow.cpp.build.dir} **/*gandiva_jni.* @@ -129,23 +129,6 @@ - - com.coderplus.maven.plugins - copy-rename-maven-plugin - - - copy-file - generate-sources - - copy - - - ${arrow.cpp.build.dir} - ${arrow.cpp.build.dir}/gandiva_jni - - - - org.xolstice.maven.plugins protobuf-maven-plugin diff --git a/java/pom.xml b/java/pom.xml index 5f9cfe8811f..e2109f22ca1 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -605,11 +605,6 @@ maven-site-plugin 3.7.1 - - com.coderplus.maven.plugins - copy-rename-maven-plugin - 1.0 - From 62954acf70c1787588da427cfd08310cc371215e Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 15:49:17 -0800 Subject: [PATCH 35/54] Correct the test that binaries exist. --- dev/tasks/java-jars/github.yml | 38 +++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 7ee68e77ee6..9c24867aaf7 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -208,29 +208,29 @@ jobs: run: | set -x - test -f arrow/java-dist/x86_64/libarrow_cdata_jni.so - test -f arrow/java-dist/x86_64/libarrow_dataset_jni.so - test -f arrow/java-dist/x86_64/libarrow_orc_jni.so - test -f arrow/java-dist/x86_64/libgandiva_jni.so + test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so + test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so + test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.so + test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.so - test -f arrow/java-dist/aarch_64/libarrow_cdata_jni.so - test -f arrow/java-dist/aarch_64/libarrow_dataset_jni.so - test -f arrow/java-dist/aarch_64/libarrow_orc_jni.so - test -f arrow/java-dist/aarch_64/libgandiva_jni.so + test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so + test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so + test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.so + test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.so - test -f arrow/java-dist/x86_64/libarrow_cdata_jni.dylib - test -f arrow/java-dist/x86_64/libarrow_dataset_jni.dylib - test -f arrow/java-dist/x86_64/libarrow_orc_jni.dylib - test -f arrow/java-dist/x86_64/libgandiva_jni.dylib + test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib + test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib + test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib + test -f arrow/java-dist/gandiva-jni/x86_64/libgandiva_jni.dylib - test -f arrow/java-dist/aarch_64/libarrow_cdata_jni.dylib - test -f arrow/java-dist/aarch_64/libarrow_dataset_jni.dylib - test -f arrow/java-dist/aarch_64/libarrow_orc_jni.dylib - test -f arrow/java-dist/aarch_64/libgandiva_jni.dylib + test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib + test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib + test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib + test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.dylib - test -f arrow/java-dist/x86_64/arrow_cdata_jni.dll - test -f arrow/java-dist/x86_64/arrow_dataset_jni.dll - test -f arrow/java-dist/x86_64/arrow_orc_jni.dll + test -f arrow/java-dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll + test -f arrow/java-dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll + test -f arrow/java-dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll - name: Build bundled jar run: | set -e From 0a4c8f4710996aa30dc9802a4272793d3c9b0528 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 16:37:52 -0800 Subject: [PATCH 36/54] Fix typo in path. --- dev/tasks/java-jars/github.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 9c24867aaf7..086d1fdbe81 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -221,7 +221,7 @@ jobs: test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib - test -f arrow/java-dist/gandiva-jni/x86_64/libgandiva_jni.dylib + test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.dylib test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib From 6eb48acdd2a07556ce952b92614f0bbf15c888a8 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 11 Jan 2024 16:49:07 -0800 Subject: [PATCH 37/54] Correct CMake calls in CI to build to the right directory --- ci/scripts/java_jni_macos_build.sh | 2 +- ci/scripts/java_jni_manylinux_build.sh | 2 +- ci/scripts/java_jni_windows_build.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index b0fc52e42cd..51a2d4d1dee 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -82,7 +82,7 @@ cmake \ -DARROW_S3=${ARROW_S3} \ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_ARCH_DIR=${normalized_arch} \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ -DGTest_SOURCE=BUNDLED \ diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh index a15eb89f658..9be129e08c4 100755 --- a/ci/scripts/java_jni_manylinux_build.sh +++ b/ci/scripts/java_jni_manylinux_build.sh @@ -91,7 +91,7 @@ cmake \ -DARROW_S3=${ARROW_S3} \ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_ARCH_DIR=${normalized_arch} \ -DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ -DGTest_SOURCE=BUNDLED \ diff --git a/ci/scripts/java_jni_windows_build.sh b/ci/scripts/java_jni_windows_build.sh index 778ee969679..e56827ce910 100755 --- a/ci/scripts/java_jni_windows_build.sh +++ b/ci/scripts/java_jni_windows_build.sh @@ -72,7 +72,7 @@ cmake \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_ZSTD=ON \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_ARCH_DIR=x86_64 \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ -GNinja \ From 8faa72a9dbfc78edecb2b32232980950e822e6e9 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 12 Jan 2024 14:36:33 -0800 Subject: [PATCH 38/54] Attempt to fix the JNI library output directories in CI --- ci/scripts/java_jni_build.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ci/scripts/java_jni_build.sh b/ci/scripts/java_jni_build.sh index 320c98c04df..6534ea74b0c 100755 --- a/ci/scripts/java_jni_build.sh +++ b/ci/scripts/java_jni_build.sh @@ -24,6 +24,18 @@ arrow_install_dir=${2} build_dir=${3}/java_jni # The directory where the final binaries will be stored when scripts finish dist_dir=${4} +normalized_arch=$(arch) +case ${normalized_arch} in + aarch64) + normalized_arch=aarch_64 + ;; + i386) + normalized_arch=x86_64 + ;; + arm64) + normalized_arch=aarch_64 + ;; + esac prefix_dir="${build_dir}/java-jni" @@ -56,7 +68,7 @@ cmake \ -DBUILD_TESTING=${ARROW_JAVA_BUILD_TESTS} \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_PREFIX_PATH=${arrow_install_dir} \ - -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_ARCH_DIR=${normalized_arch} \ -DCMAKE_INSTALL_PREFIX=${prefix_dir} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ -DProtobuf_USE_STATIC_LIBS=ON \ From 5263ad198a8fe2e5450f93245d00dc8dd1d3a62f Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 12 Jan 2024 14:49:40 -0800 Subject: [PATCH 39/54] Fix the checking of dependencies --- ci/scripts/java_jni_macos_build.sh | 8 ++++---- ci/scripts/java_jni_manylinux_build.sh | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index 51a2d4d1dee..6c367fafe3a 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -138,8 +138,8 @@ archery linking check-dependencies \ --allow libncurses \ --allow libobjc \ --allow libz \ - arrow_cdata_jni/libarrow_cdata_jni.dylib \ - arrow_dataset_jni/libarrow_dataset_jni.dylib \ - arrow_orc_jni/libarrow_orc_jni.dylib \ - gandiva_jni/libgandiva_jni.dylib + arrow_cdata_jni/${normalized_arch}/libarrow_cdata_jni.dylib \ + arrow_dataset_jni/${normalized_arch}/libarrow_dataset_jni.dylib \ + arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.dylib \ + gandiva_jni/${normalized_arch}/libgandiva_jni.dylib popd diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh index 9be129e08c4..214f1a943e6 100755 --- a/ci/scripts/java_jni_manylinux_build.sh +++ b/ci/scripts/java_jni_manylinux_build.sh @@ -164,8 +164,8 @@ archery linking check-dependencies \ --allow libstdc++ \ --allow libz \ --allow linux-vdso \ - arrow_cdata_jni/libarrow_cdata_jni.so \ - arrow_dataset_jni/libarrow_dataset_jni.so \ - arrow_orc_jni/libarrow_orc_jni.so \ - gandiva_jni/libgandiva_jni.so + arrow_cdata_jni/${normalized_arch}/libarrow_cdata_jni.so \ + arrow_dataset_jni/${normalized_arch}/libarrow_dataset_jni.so \ + arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.so \ + gandiva_jni/${normalized_arch}/libgandiva_jni.so popd From e50d1140d54af916f36572b07d8e8f4232a52ab8 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 12 Jan 2024 16:25:46 -0800 Subject: [PATCH 40/54] More fixes related to CI and the arch directory --- ci/scripts/integration_arrow_build.sh | 2 +- ci/scripts/java_jni_macos_build.sh | 2 +- ci/scripts/java_jni_manylinux_build.sh | 2 +- ci/scripts/java_jni_windows_build.sh | 2 +- docker-compose.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/scripts/integration_arrow_build.sh b/ci/scripts/integration_arrow_build.sh index 02f593bf77b..e5c31527aed 100755 --- a/ci/scripts/integration_arrow_build.sh +++ b/ci/scripts/integration_arrow_build.sh @@ -46,7 +46,7 @@ if [ "${ARROW_INTEGRATION_JAVA}" == "ON" ]; then export ARROW_JAVA_CDATA="ON" export JAVA_JNI_CMAKE_ARGS="-DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF -DARROW_JAVA_JNI_ENABLE_C=ON" - ${arrow_dir}/ci/scripts/java_jni_build.sh ${arrow_dir} ${ARROW_HOME} ${build_dir} /tmp/dist/java/$(arch) + ${arrow_dir}/ci/scripts/java_jni_build.sh ${arrow_dir} ${ARROW_HOME} ${build_dir} /tmp/dist/java ${arrow_dir}/ci/scripts/java_build.sh ${arrow_dir} ${build_dir} /tmp/dist/java fi diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index 6c367fafe3a..6e52389007f 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -31,7 +31,7 @@ case ${normalized_arch} in ;; esac # The directory where the final binaries will be stored when scripts finish -dist_dir=${3}/${normalized_arch} +dist_dir=${3} echo "=== Clear output directories and leftovers ===" # Clear output directories and leftovers diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh index 214f1a943e6..709a64c42fd 100755 --- a/ci/scripts/java_jni_manylinux_build.sh +++ b/ci/scripts/java_jni_manylinux_build.sh @@ -28,7 +28,7 @@ case ${normalized_arch} in ;; esac # The directory where the final binaries will be stored when scripts finish -dist_dir=${3}/${normalized_arch} +dist_dir=${3} echo "=== Clear output directories and leftovers ===" # Clear output directories and leftovers diff --git a/ci/scripts/java_jni_windows_build.sh b/ci/scripts/java_jni_windows_build.sh index e56827ce910..6aa057d2aaa 100755 --- a/ci/scripts/java_jni_windows_build.sh +++ b/ci/scripts/java_jni_windows_build.sh @@ -22,7 +22,7 @@ set -ex arrow_dir=${1} build_dir=${2} # The directory where the final binaries will be stored when scripts finish -dist_dir=${3}/x86_64 +dist_dir=${3} echo "=== Clear output directories and leftovers ===" # Clear output directories and leftovers diff --git a/docker-compose.yml b/docker-compose.yml index 14eff67f389..a08345c198f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1343,7 +1343,7 @@ services: command: [ "/arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/python_build.sh /arrow /build && - /arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java/$$(arch) && + /arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java/ && /arrow/ci/scripts/java_build.sh /arrow /build /tmp/dist/java && /arrow/ci/scripts/java_cdata_integration.sh /arrow /tmp/dist/java" ] From 2bb186488f9e33a5308fecf528eb063f451642fa Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 12 Jan 2024 17:29:02 -0800 Subject: [PATCH 41/54] Update documentation about CMAKE_ARCH_DIR option --- docs/source/developers/java/building.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/developers/java/building.rst b/docs/source/developers/java/building.rst index 0e831915e09..f6a3fde47fd 100644 --- a/docs/source/developers/java/building.rst +++ b/docs/source/developers/java/building.rst @@ -166,7 +166,7 @@ CMake -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF \ -DBUILD_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_LIBDIR=lib/ \ + -DCMAKE_ARCH_DIR= ^ \ -DCMAKE_INSTALL_PREFIX=java-dist $ cmake --build java-cdata --target install --config Release $ ls -latr java-dist/lib @@ -186,7 +186,7 @@ CMake -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF ^ -DBUILD_TESTING=OFF ^ -DCMAKE_BUILD_TYPE=Release ^ - -DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^ + -DCMAKE_ARCH_DIR=x86_64 ^ -DCMAKE_INSTALL_PREFIX=java-dist $ cmake --build java-cdata --target install --config Release $ dir "java-dist/bin" From 09929ba9fdf8914cb72a11e19dba76e6b663ade9 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 12 Jan 2024 17:34:07 -0800 Subject: [PATCH 42/54] Fix Windows output directories --- java/adapter/orc/CMakeLists.txt | 1 + java/c/CMakeLists.txt | 1 + java/dataset/CMakeLists.txt | 3 +++ java/gandiva/CMakeLists.txt | 1 + 4 files changed, 6 insertions(+) diff --git a/java/adapter/orc/CMakeLists.txt b/java/adapter/orc/CMakeLists.txt index c1f57eb8808..71fd6461d8c 100644 --- a/java/adapter/orc/CMakeLists.txt +++ b/java/adapter/orc/CMakeLists.txt @@ -38,6 +38,7 @@ target_link_libraries(arrow_java_jni_orc arrow_java_jni_orc_headers jni Arrow::arrow_static) set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_orc_jni/${CMAKE_ARCH_DIR}") +set(CMAKE_INSTALL_BINDIR "${CMAKE_INSTALL_PREFIX}/bin/arrow_orc_jni/${CMAKE_ARCH_DIR}") install(TARGETS arrow_java_jni_orc LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/java/c/CMakeLists.txt b/java/c/CMakeLists.txt index 9ae1b31be2d..83591ea5053 100644 --- a/java/c/CMakeLists.txt +++ b/java/c/CMakeLists.txt @@ -31,6 +31,7 @@ set_property(TARGET arrow_java_jni_cdata PROPERTY OUTPUT_NAME "arrow_cdata_jni") target_link_libraries(arrow_java_jni_cdata arrow_java_jni_cdata_headers jni) set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_cdata_jni/${CMAKE_ARCH_DIR}") +set(CMAKE_INSTALL_BINDIR "${CMAKE_INSTALL_PREFIX}/bin/arrow_cdata_jni/${CMAKE_ARCH_DIR}") install(TARGETS arrow_java_jni_cdata LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/java/dataset/CMakeLists.txt b/java/dataset/CMakeLists.txt index 620dfb10b27..07e0d0c2a0e 100644 --- a/java/dataset/CMakeLists.txt +++ b/java/dataset/CMakeLists.txt @@ -50,6 +50,9 @@ endif() set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_dataset_jni/${CMAKE_ARCH_DIR}") +set(CMAKE_INSTALL_BINDIR + "${CMAKE_INSTALL_PREFIX}/bin/arrow_dataset_jni/${CMAKE_ARCH_DIR}") + install(TARGETS arrow_java_jni_dataset LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/java/gandiva/CMakeLists.txt b/java/gandiva/CMakeLists.txt index 84147e58dba..3da4ae82904 100644 --- a/java/gandiva/CMakeLists.txt +++ b/java/gandiva/CMakeLists.txt @@ -85,6 +85,7 @@ if(CXX_LINKER_SUPPORTS_VERSION_SCRIPT) endif() set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/gandiva_jni/${CMAKE_ARCH_DIR}") +set(CMAKE_INSTALL_BINDIR "${CMAKE_INSTALL_PREFIX}/bin/gandiva_jni/${CMAKE_ARCH_DIR}") install(TARGETS arrow_java_jni_gandiva LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} From 632fd020fd209682aaf8444a8cbf6c7e6f7f797c Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 12 Jan 2024 17:34:20 -0800 Subject: [PATCH 43/54] Fix missing whitespace when running python integration tester --- dev/archery/archery/integration/tester_java.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/archery/archery/integration/tester_java.py b/dev/archery/archery/integration/tester_java.py index dea642fa9c2..83dab4914b9 100644 --- a/dev/archery/archery/integration/tester_java.py +++ b/dev/archery/archery/integration/tester_java.py @@ -258,7 +258,7 @@ def __init__(self, *args, **kwargs): # Java 9+ self._java_opts.append( '--add-opens=java.base/java.nio=' - 'org.apache.arrow.memory.core,ALL-UNNAMED' + 'org.apache.arrow.memory.core,ALL-UNNAMED ' '--add-reads=org.apache.arrow.flight.core=ALL-UNNAMED') def _run(self, arrow_path=None, json_path=None, command='VALIDATE'): From aaeff8fbd1bf6f6e89468cdda0991a0dee040d9b Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 12 Jan 2024 23:02:13 -0800 Subject: [PATCH 44/54] dummy commit --- dev/archery/archery/integration/tester_java.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/archery/archery/integration/tester_java.py b/dev/archery/archery/integration/tester_java.py index 83dab4914b9..4bd5dd0a2f7 100644 --- a/dev/archery/archery/integration/tester_java.py +++ b/dev/archery/archery/integration/tester_java.py @@ -243,7 +243,7 @@ class JavaTester(Tester): C_DATA_ARRAY_EXPORTER = True C_DATA_ARRAY_IMPORTER = True - name = 'Java' + name = 'Java' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) From a16a7c0eac69f575978ad2f35786be3bb970d341 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 12 Jan 2024 23:02:20 -0800 Subject: [PATCH 45/54] Revert "dummy commit" This reverts commit d795f47a71c0bc391f4d27ca37edf901a177560b. --- dev/archery/archery/integration/tester_java.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/archery/archery/integration/tester_java.py b/dev/archery/archery/integration/tester_java.py index 4bd5dd0a2f7..83dab4914b9 100644 --- a/dev/archery/archery/integration/tester_java.py +++ b/dev/archery/archery/integration/tester_java.py @@ -243,7 +243,7 @@ class JavaTester(Tester): C_DATA_ARRAY_EXPORTER = True C_DATA_ARRAY_IMPORTER = True - name = 'Java' + name = 'Java' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) From fa922eb4ac47e936dc6c8bd3873ac3581058adc5 Mon Sep 17 00:00:00 2001 From: James Duong Date: Sat, 13 Jan 2024 07:55:50 -0800 Subject: [PATCH 46/54] Update building.rst --- docs/source/developers/java/building.rst | 58 ++++++++++++------------ 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/docs/source/developers/java/building.rst b/docs/source/developers/java/building.rst index f6a3fde47fd..5e7e327ae28 100644 --- a/docs/source/developers/java/building.rst +++ b/docs/source/developers/java/building.rst @@ -115,9 +115,8 @@ Maven $ export JAVA_HOME= $ java --version $ mvn generate-resources -Pgenerate-libs-cdata-all-os -N - $ ls -latr ../java-dist/lib/ - |__ libarrow_cdata_jni.dylib - |__ libarrow_cdata_jni.so + $ ls -latr ../java-dist/lib + |__ arrow_cdata_jni/ - To build only the JNI C Data Interface library (Windows): @@ -125,8 +124,8 @@ Maven $ cd arrow/java $ mvn generate-resources -Pgenerate-libs-cdata-all-os -N - $ dir "../java-dist/bin/x86_64" - |__ arrow_cdata_jni.dll + $ dir "../java-dist/bin" + |__ arrow_cdata_jni/ - To build all JNI libraries (macOS / Linux) except the JNI C Data Interface library: @@ -136,10 +135,10 @@ Maven $ export JAVA_HOME= $ java --version $ mvn generate-resources -Pgenerate-libs-jni-macos-linux -N - $ ls -latr java-dist/lib//*_{jni,java}.* - |__ libarrow_dataset_jni.dylib - |__ libarrow_orc_jni.dylib - |__ libgandiva_jni.dylib + $ ls -latr java-dist/lib + |__ arrow_dataset_jni/ + |__ arrow_orc_jni/ + |__ gandiva_jni/ - To build all JNI libraries (Windows) except the JNI C Data Interface library: @@ -147,8 +146,8 @@ Maven $ cd arrow/java $ mvn generate-resources -Pgenerate-libs-jni-windows -N - $ dir "../java-dist/bin/x86_64" - |__ arrow_dataset_jni.dll + $ dir "../java-dist/bin" + |__ arrow_dataset_jni/ CMake ~~~~~ @@ -166,12 +165,11 @@ CMake -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF \ -DBUILD_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_ARCH_DIR= ^ \ + -DCMAKE_ARCH_DIR= \ -DCMAKE_INSTALL_PREFIX=java-dist $ cmake --build java-cdata --target install --config Release $ ls -latr java-dist/lib - |__ libarrow_cdata_jni.dylib - |__ libarrow_cdata_jni.so + |__ arrow_cdata_jni/ - To build only the JNI C Data Interface library (Windows): @@ -190,7 +188,7 @@ CMake -DCMAKE_INSTALL_PREFIX=java-dist $ cmake --build java-cdata --target install --config Release $ dir "java-dist/bin" - |__ arrow_cdata_jni.dll + |__ arrow_cdata_jni/ - To build all JNI libraries (macOS / Linux) except the JNI C Data Interface library: @@ -222,7 +220,7 @@ CMake -DARROW_SUBSTRAIT=ON \ -DARROW_USE_CCACHE=ON \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_LIBDIR=lib/ \ + -DCMAKE_ARCH_DIR= \ -DCMAKE_INSTALL_PREFIX=java-dist \ -DCMAKE_UNITY_BUILD=ON $ cmake --build cpp-jni --target install --config Release @@ -233,16 +231,16 @@ CMake -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON \ -DBUILD_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_LIBDIR=lib/ \ + -DCMAKE_ARCH_DIR= \ -DCMAKE_INSTALL_PREFIX=java-dist \ -DCMAKE_PREFIX_PATH=$PWD/java-dist \ -DProtobuf_ROOT=$PWD/../cpp-jni/protobuf_ep-install \ -DProtobuf_USE_STATIC_LIBS=ON $ cmake --build java-jni --target install --config Release - $ ls -latr java-dist/lib//*_{jni,java}.* - |__ libarrow_dataset_jni.dylib - |__ libarrow_orc_jni.dylib - |__ libgandiva_jni.dylib + $ ls -latr java-dist/lib/ + |__ arrow_dataset_jni/ + |__ arrow_orc_jni/ + |__ gandiva_jni/ - To build all JNI libraries (Windows) except the JNI C Data Interface library: @@ -271,7 +269,7 @@ CMake -DARROW_WITH_ZLIB=ON ^ -DARROW_WITH_ZSTD=ON ^ -DCMAKE_BUILD_TYPE=Release ^ - -DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^ + -DCMAKE_ARCH_DIR=x86_64 ^ -DCMAKE_INSTALL_PREFIX=java-dist ^ -DCMAKE_UNITY_BUILD=ON ^ -GNinja @@ -288,13 +286,13 @@ CMake -DARROW_JAVA_JNI_ENABLE_ORC=ON ^ -DBUILD_TESTING=OFF ^ -DCMAKE_BUILD_TYPE=Release ^ - -DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^ + -DCMAKE_ARCH_DIR=x86_64 ^ -DCMAKE_INSTALL_PREFIX=java-dist ^ -DCMAKE_PREFIX_PATH=$PWD/java-dist $ cmake --build java-jni --target install --config Release $ dir "java-dist/bin" - |__ arrow_orc_jni.dll - |__ arrow_dataset_jni.dll + |__ arrow_orc_jni/ + |__ arrow_dataset_jni/ Archery ~~~~~~~ @@ -303,11 +301,11 @@ Archery $ cd arrow $ archery docker run java-jni-manylinux-2014 - $ ls -latr java-dist// - |__ libarrow_cdata_jni.so - |__ libarrow_dataset_jni.so - |__ libarrow_orc_jni.so - |__ libgandiva_jni.so + $ ls -latr java-dist + |__ arrow_cdata_jni/ + |__ arrow_dataset_jni/ + |__ arrow_orc_jni/ + |__ gandiva_jni/ Building Java JNI Modules ------------------------- From 77a6893bc11b7f24c5d3404f4749e3c28a91b05f Mon Sep 17 00:00:00 2001 From: James Duong Date: Sat, 13 Jan 2024 08:04:45 -0800 Subject: [PATCH 47/54] Fix the integration test command-line --- dev/archery/archery/integration/tester_java.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/archery/archery/integration/tester_java.py b/dev/archery/archery/integration/tester_java.py index 83dab4914b9..8e7a0bb99f9 100644 --- a/dev/archery/archery/integration/tester_java.py +++ b/dev/archery/archery/integration/tester_java.py @@ -258,7 +258,8 @@ def __init__(self, *args, **kwargs): # Java 9+ self._java_opts.append( '--add-opens=java.base/java.nio=' - 'org.apache.arrow.memory.core,ALL-UNNAMED ' + 'org.apache.arrow.memory.core,ALL-UNNAMED') + self._java_opts.append( '--add-reads=org.apache.arrow.flight.core=ALL-UNNAMED') def _run(self, arrow_path=None, json_path=None, command='VALIDATE'): From 0d89a579eaf4e1f7572c63ef1d3829886195e873 Mon Sep 17 00:00:00 2001 From: James Duong Date: Mon, 15 Jan 2024 08:33:58 -0800 Subject: [PATCH 48/54] Remove flight-grpc from tasks.yaml Module is combined with flight-core now. --- dev/tasks/tasks.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 04faef427e2..ca45d48bcd4 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -810,13 +810,6 @@ tasks: - flight-core-{no_rc_snapshot_version}-tests.jar - flight-core-{no_rc_snapshot_version}.jar - flight-core-{no_rc_snapshot_version}.pom - - flight-grpc-{no_rc_snapshot_version}-cyclonedx.json - - flight-grpc-{no_rc_snapshot_version}-cyclonedx.xml - - flight-grpc-{no_rc_snapshot_version}-javadoc.jar - - flight-grpc-{no_rc_snapshot_version}-sources.jar - - flight-grpc-{no_rc_snapshot_version}-tests.jar - - flight-grpc-{no_rc_snapshot_version}.jar - - flight-grpc-{no_rc_snapshot_version}.pom - flight-integration-tests-{no_rc_snapshot_version}-cyclonedx.json - flight-integration-tests-{no_rc_snapshot_version}-cyclonedx.xml - flight-integration-tests-{no_rc_snapshot_version}-jar-with-dependencies.jar From d45cbc2100c13049f097e8d7921c9a5a4c1ca0d3 Mon Sep 17 00:00:00 2001 From: James Duong Date: Mon, 15 Jan 2024 08:34:19 -0800 Subject: [PATCH 49/54] Rename CMAKE_ARCH_DIR to ARROW_JAVA_JNI_ARCH_DIR --- ci/scripts/java_jni_build.sh | 2 +- ci/scripts/java_jni_macos_build.sh | 2 +- ci/scripts/java_jni_manylinux_build.sh | 2 +- ci/scripts/java_jni_windows_build.sh | 2 +- docs/source/developers/java/building.rst | 12 ++++++------ java/adapter/orc/CMakeLists.txt | 6 ++++-- java/c/CMakeLists.txt | 6 ++++-- java/dataset/CMakeLists.txt | 4 ++-- java/gandiva/CMakeLists.txt | 6 ++++-- java/pom.xml | 10 +++++----- 10 files changed, 29 insertions(+), 23 deletions(-) diff --git a/ci/scripts/java_jni_build.sh b/ci/scripts/java_jni_build.sh index 6534ea74b0c..9c3f031b61d 100755 --- a/ci/scripts/java_jni_build.sh +++ b/ci/scripts/java_jni_build.sh @@ -68,7 +68,7 @@ cmake \ -DBUILD_TESTING=${ARROW_JAVA_BUILD_TESTS} \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_PREFIX_PATH=${arrow_install_dir} \ - -DCMAKE_ARCH_DIR=${normalized_arch} \ + -DARROW_JAVA_JNI_ARCH_DIR=${normalized_arch} \ -DCMAKE_INSTALL_PREFIX=${prefix_dir} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ -DProtobuf_USE_STATIC_LIBS=ON \ diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index 6e52389007f..5d9c3ce15d9 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -82,7 +82,7 @@ cmake \ -DARROW_S3=${ARROW_S3} \ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_ARCH_DIR=${normalized_arch} \ + -DARROW_JAVA_JNI_ARCH_DIR=${normalized_arch} \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ -DGTest_SOURCE=BUNDLED \ diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh index 709a64c42fd..572cc79a232 100755 --- a/ci/scripts/java_jni_manylinux_build.sh +++ b/ci/scripts/java_jni_manylinux_build.sh @@ -91,7 +91,7 @@ cmake \ -DARROW_S3=${ARROW_S3} \ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_ARCH_DIR=${normalized_arch} \ + -DARROW_JAVA_JNI_ARCH_DIR=${normalized_arch} \ -DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ -DGTest_SOURCE=BUNDLED \ diff --git a/ci/scripts/java_jni_windows_build.sh b/ci/scripts/java_jni_windows_build.sh index 6aa057d2aaa..b0916a9eb8b 100755 --- a/ci/scripts/java_jni_windows_build.sh +++ b/ci/scripts/java_jni_windows_build.sh @@ -72,7 +72,7 @@ cmake \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_ZSTD=ON \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_ARCH_DIR=x86_64 \ + -DARROW_JAVA_JNI_ARCH_DIR=x86_64 \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ -GNinja \ diff --git a/docs/source/developers/java/building.rst b/docs/source/developers/java/building.rst index 5e7e327ae28..6e17b948834 100644 --- a/docs/source/developers/java/building.rst +++ b/docs/source/developers/java/building.rst @@ -165,7 +165,7 @@ CMake -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF \ -DBUILD_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_ARCH_DIR= \ + -DARROW_JAVA_JNI_ARCH_DIR= \ -DCMAKE_INSTALL_PREFIX=java-dist $ cmake --build java-cdata --target install --config Release $ ls -latr java-dist/lib @@ -184,7 +184,7 @@ CMake -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF ^ -DBUILD_TESTING=OFF ^ -DCMAKE_BUILD_TYPE=Release ^ - -DCMAKE_ARCH_DIR=x86_64 ^ + -DARROW_JAVA_JNI_ARCH_DIR=x86_64 ^ -DCMAKE_INSTALL_PREFIX=java-dist $ cmake --build java-cdata --target install --config Release $ dir "java-dist/bin" @@ -220,7 +220,7 @@ CMake -DARROW_SUBSTRAIT=ON \ -DARROW_USE_CCACHE=ON \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_ARCH_DIR= \ + -DARROW_JAVA_JNI_ARCH_DIR= \ -DCMAKE_INSTALL_PREFIX=java-dist \ -DCMAKE_UNITY_BUILD=ON $ cmake --build cpp-jni --target install --config Release @@ -231,7 +231,7 @@ CMake -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON \ -DBUILD_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_ARCH_DIR= \ + -DARROW_JAVA_JNI_ARCH_DIR= \ -DCMAKE_INSTALL_PREFIX=java-dist \ -DCMAKE_PREFIX_PATH=$PWD/java-dist \ -DProtobuf_ROOT=$PWD/../cpp-jni/protobuf_ep-install \ @@ -269,7 +269,7 @@ CMake -DARROW_WITH_ZLIB=ON ^ -DARROW_WITH_ZSTD=ON ^ -DCMAKE_BUILD_TYPE=Release ^ - -DCMAKE_ARCH_DIR=x86_64 ^ + -DARROW_JAVA_JNI_ARCH_DIR=x86_64 ^ -DCMAKE_INSTALL_PREFIX=java-dist ^ -DCMAKE_UNITY_BUILD=ON ^ -GNinja @@ -286,7 +286,7 @@ CMake -DARROW_JAVA_JNI_ENABLE_ORC=ON ^ -DBUILD_TESTING=OFF ^ -DCMAKE_BUILD_TYPE=Release ^ - -DCMAKE_ARCH_DIR=x86_64 ^ + -DARROW_JAVA_JNI_ARCH_DIR=x86_64 ^ -DCMAKE_INSTALL_PREFIX=java-dist ^ -DCMAKE_PREFIX_PATH=$PWD/java-dist $ cmake --build java-jni --target install --config Release diff --git a/java/adapter/orc/CMakeLists.txt b/java/adapter/orc/CMakeLists.txt index 71fd6461d8c..92545941f0e 100644 --- a/java/adapter/orc/CMakeLists.txt +++ b/java/adapter/orc/CMakeLists.txt @@ -37,8 +37,10 @@ set_property(TARGET arrow_java_jni_orc PROPERTY OUTPUT_NAME "arrow_orc_jni") target_link_libraries(arrow_java_jni_orc arrow_java_jni_orc_headers jni Arrow::arrow_static) -set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_orc_jni/${CMAKE_ARCH_DIR}") -set(CMAKE_INSTALL_BINDIR "${CMAKE_INSTALL_PREFIX}/bin/arrow_orc_jni/${CMAKE_ARCH_DIR}") +set(CMAKE_INSTALL_LIBDIR + "${CMAKE_INSTALL_PREFIX}/lib/arrow_orc_jni/${ARROW_JAVA_JNI_ARCH_DIR}") +set(CMAKE_INSTALL_BINDIR + "${CMAKE_INSTALL_PREFIX}/bin/arrow_orc_jni/${ARROW_JAVA_JNI_ARCH_DIR}") install(TARGETS arrow_java_jni_orc LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/java/c/CMakeLists.txt b/java/c/CMakeLists.txt index 83591ea5053..b8f87ef545f 100644 --- a/java/c/CMakeLists.txt +++ b/java/c/CMakeLists.txt @@ -30,8 +30,10 @@ add_library(arrow_java_jni_cdata SHARED src/main/cpp/jni_wrapper.cc) set_property(TARGET arrow_java_jni_cdata PROPERTY OUTPUT_NAME "arrow_cdata_jni") target_link_libraries(arrow_java_jni_cdata arrow_java_jni_cdata_headers jni) -set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_cdata_jni/${CMAKE_ARCH_DIR}") -set(CMAKE_INSTALL_BINDIR "${CMAKE_INSTALL_PREFIX}/bin/arrow_cdata_jni/${CMAKE_ARCH_DIR}") +set(CMAKE_INSTALL_LIBDIR + "${CMAKE_INSTALL_PREFIX}/lib/arrow_cdata_jni/${ARROW_JAVA_JNI_ARCH_DIR}") +set(CMAKE_INSTALL_BINDIR + "${CMAKE_INSTALL_PREFIX}/bin/arrow_cdata_jni/${ARROW_JAVA_JNI_ARCH_DIR}") install(TARGETS arrow_java_jni_cdata LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/java/dataset/CMakeLists.txt b/java/dataset/CMakeLists.txt index 07e0d0c2a0e..9ca5f4d75c0 100644 --- a/java/dataset/CMakeLists.txt +++ b/java/dataset/CMakeLists.txt @@ -48,10 +48,10 @@ if(BUILD_TESTING) endif() set(CMAKE_INSTALL_LIBDIR - "${CMAKE_INSTALL_PREFIX}/lib/arrow_dataset_jni/${CMAKE_ARCH_DIR}") + "${CMAKE_INSTALL_PREFIX}/lib/arrow_dataset_jni/${ARROW_JAVA_JNI_ARCH_DIR}") set(CMAKE_INSTALL_BINDIR - "${CMAKE_INSTALL_PREFIX}/bin/arrow_dataset_jni/${CMAKE_ARCH_DIR}") + "${CMAKE_INSTALL_PREFIX}/bin/arrow_dataset_jni/${ARROW_JAVA_JNI_ARCH_DIR}") install(TARGETS arrow_java_jni_dataset LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/java/gandiva/CMakeLists.txt b/java/gandiva/CMakeLists.txt index 3da4ae82904..097eedffc47 100644 --- a/java/gandiva/CMakeLists.txt +++ b/java/gandiva/CMakeLists.txt @@ -84,8 +84,10 @@ if(CXX_LINKER_SUPPORTS_VERSION_SCRIPT) ) endif() -set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/gandiva_jni/${CMAKE_ARCH_DIR}") -set(CMAKE_INSTALL_BINDIR "${CMAKE_INSTALL_PREFIX}/bin/gandiva_jni/${CMAKE_ARCH_DIR}") +set(CMAKE_INSTALL_LIBDIR + "${CMAKE_INSTALL_PREFIX}/lib/gandiva_jni/${ARROW_JAVA_JNI_ARCH_DIR}") +set(CMAKE_INSTALL_BINDIR + "${CMAKE_INSTALL_PREFIX}/bin/gandiva_jni/${ARROW_JAVA_JNI_ARCH_DIR}") install(TARGETS arrow_java_jni_gandiva LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/java/pom.xml b/java/pom.xml index e2109f22ca1..4bef553e58b 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -1055,7 +1055,7 @@ -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release - -DCMAKE_ARCH_DIR=${os.detected.arch} + -DARROW_JAVA_JNI_ARCH_DIR=${os.detected.arch} -DCMAKE_INSTALL_PREFIX=${arrow.c.jni.dist.dir} ../ @@ -1128,7 +1128,7 @@ -DARROW_SUBSTRAIT=${ARROW_DATASET} -DARROW_USE_CCACHE=ON -DCMAKE_BUILD_TYPE=Release - -DCMAKE_ARCH_DIR=${os.detected.arch} + -DARROW_JAVA_JNI_ARCH_DIR=${os.detected.arch} -DCMAKE_INSTALL_PREFIX=java-dist -DCMAKE_UNITY_BUILD=ON @@ -1169,7 +1169,7 @@ -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release - -DCMAKE_ARCH_DIR=${os.detected.arch} + -DARROW_JAVA_JNI_ARCH_DIR=${os.detected.arch} -DCMAKE_INSTALL_PREFIX=${arrow.dataset.jni.dist.dir} -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist/lib/${os.detected.arch}/cmake -DProtobuf_USE_STATIC_LIBS=ON @@ -1248,7 +1248,7 @@ -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DCMAKE_BUILD_TYPE=Release - -DCMAKE_ARCH_DIR=${os.detected.arch} + -DARROW_JAVA_JNI_ARCH_DIR=${os.detected.arch} -DCMAKE_INSTALL_PREFIX=java-dist -DCMAKE_UNITY_BUILD=ON -GNinja @@ -1290,7 +1290,7 @@ -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release - -DCMAKE_ARCH_DIR=${os.detected.arch} + -DARROW_JAVA_JNI_ARCH_DIR=${os.detected.arch} -DCMAKE_INSTALL_PREFIX=${arrow.dataset.jni.dist.dir} -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist/lib/${os.detected.arch}/cmake From 184467ac5da30720b5cb79d372199df7c662ccf2 Mon Sep 17 00:00:00 2001 From: James Duong Date: Mon, 15 Jan 2024 12:27:21 -0800 Subject: [PATCH 50/54] Removed a reference to flight-grpc from docs --- docs/source/java/overview.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/source/java/overview.rst b/docs/source/java/overview.rst index 4b30b8e000f..9d9cbad8a26 100644 --- a/docs/source/java/overview.rst +++ b/docs/source/java/overview.rst @@ -56,9 +56,6 @@ but some modules are JNI bindings to the C++ library. * - flight-core - (Experimental) An RPC mechanism for transferring ValueVectors. - Native - * - flight-grpc - - (Experimental) Contains utility class to expose Flight gRPC service and client. - - Native * - flight-sql - (Experimental) Contains utility classes to expose Flight SQL semantics for clients and servers over Arrow Flight. - Native From aaedda7d3c6bf22f4cd313bfe7f8711064286788 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 18 Jan 2024 11:08:21 -0800 Subject: [PATCH 51/54] Change build variable name Use ARROW_JNI_JAVA__[BIN|LIB]DIR instead of overriding CMAKE_INSTALL_[BIN|LIB]DIR --- java/adapter/orc/CMakeLists.txt | 8 ++++---- java/c/CMakeLists.txt | 8 ++++---- java/dataset/CMakeLists.txt | 8 ++++---- java/gandiva/CMakeLists.txt | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/java/adapter/orc/CMakeLists.txt b/java/adapter/orc/CMakeLists.txt index 92545941f0e..d29856ff8cd 100644 --- a/java/adapter/orc/CMakeLists.txt +++ b/java/adapter/orc/CMakeLists.txt @@ -37,11 +37,11 @@ set_property(TARGET arrow_java_jni_orc PROPERTY OUTPUT_NAME "arrow_orc_jni") target_link_libraries(arrow_java_jni_orc arrow_java_jni_orc_headers jni Arrow::arrow_static) -set(CMAKE_INSTALL_LIBDIR +set(ARROW_JAVA_JNI_ORC_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_orc_jni/${ARROW_JAVA_JNI_ARCH_DIR}") -set(CMAKE_INSTALL_BINDIR +set(ARROW_JAVA_JNI_ORC_BINDIR "${CMAKE_INSTALL_PREFIX}/bin/arrow_orc_jni/${ARROW_JAVA_JNI_ARCH_DIR}") install(TARGETS arrow_java_jni_orc - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + LIBRARY DESTINATION ${ARROW_JAVA_JNI_ORC_LIBDIR} + RUNTIME DESTINATION ${ARROW_JAVA_JNI_ORC_BINDIR}) diff --git a/java/c/CMakeLists.txt b/java/c/CMakeLists.txt index b8f87ef545f..8e307672d0e 100644 --- a/java/c/CMakeLists.txt +++ b/java/c/CMakeLists.txt @@ -30,11 +30,11 @@ add_library(arrow_java_jni_cdata SHARED src/main/cpp/jni_wrapper.cc) set_property(TARGET arrow_java_jni_cdata PROPERTY OUTPUT_NAME "arrow_cdata_jni") target_link_libraries(arrow_java_jni_cdata arrow_java_jni_cdata_headers jni) -set(CMAKE_INSTALL_LIBDIR +set(ARROW_JAVA_JNI_CDATA_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_cdata_jni/${ARROW_JAVA_JNI_ARCH_DIR}") -set(CMAKE_INSTALL_BINDIR +set(ARROW_JAVA_JNI_CDATA_BINDIR "${CMAKE_INSTALL_PREFIX}/bin/arrow_cdata_jni/${ARROW_JAVA_JNI_ARCH_DIR}") install(TARGETS arrow_java_jni_cdata - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + LIBRARY DESTINATION ${ARROW_JAVA_JNI_CDATA_LIBDIR} + RUNTIME DESTINATION ${ARROW_JAVA_JNI_CDATA_BINDIR}) diff --git a/java/dataset/CMakeLists.txt b/java/dataset/CMakeLists.txt index 9ca5f4d75c0..348850c3be5 100644 --- a/java/dataset/CMakeLists.txt +++ b/java/dataset/CMakeLists.txt @@ -47,12 +47,12 @@ if(BUILD_TESTING) add_test(NAME arrow-java-jni-dataset-test COMMAND arrow-java-jni-dataset-test) endif() -set(CMAKE_INSTALL_LIBDIR +set(ARROW_JAVA_JNI_DATASET_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_dataset_jni/${ARROW_JAVA_JNI_ARCH_DIR}") -set(CMAKE_INSTALL_BINDIR +set(ARROW_JAVA_JNI_DATASET_BINDIR "${CMAKE_INSTALL_PREFIX}/bin/arrow_dataset_jni/${ARROW_JAVA_JNI_ARCH_DIR}") install(TARGETS arrow_java_jni_dataset - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + LIBRARY DESTINATION ${ARROW_JAVA_JNI_DATASET_LIBDIR} + RUNTIME DESTINATION ${ARROW_JAVA_JNI_DATASET_BINDIR}) diff --git a/java/gandiva/CMakeLists.txt b/java/gandiva/CMakeLists.txt index 097eedffc47..369829d7a30 100644 --- a/java/gandiva/CMakeLists.txt +++ b/java/gandiva/CMakeLists.txt @@ -84,11 +84,11 @@ if(CXX_LINKER_SUPPORTS_VERSION_SCRIPT) ) endif() -set(CMAKE_INSTALL_LIBDIR +set(ARROW_JAVA_JNI_GANDIVA_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/gandiva_jni/${ARROW_JAVA_JNI_ARCH_DIR}") -set(CMAKE_INSTALL_BINDIR +set(ARROW_JAVA_JNI_GANDIVA_BINDIR "${CMAKE_INSTALL_PREFIX}/bin/gandiva_jni/${ARROW_JAVA_JNI_ARCH_DIR}") install(TARGETS arrow_java_jni_gandiva - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + LIBRARY DESTINATION ${ARROW_JAVA_JNI_GANDIVA_LIBDIR} + RUNTIME DESTINATION ${ARROW_JAVA_JNI_GANDIVA_BINDIR}) From 3607ed8c467f276a1377c9e174ac07d0233bebe0 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 18 Jan 2024 12:03:27 -0800 Subject: [PATCH 52/54] Derive arch in CMakeLists.txt instead of scripts --- ci/scripts/java_jni_build.sh | 16 +--------------- java/CMakeLists.txt | 14 ++++++++++++++ java/c/CMakeLists.txt | 8 ++++---- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/ci/scripts/java_jni_build.sh b/ci/scripts/java_jni_build.sh index 9c3f031b61d..cbb754b3e48 100755 --- a/ci/scripts/java_jni_build.sh +++ b/ci/scripts/java_jni_build.sh @@ -24,19 +24,6 @@ arrow_install_dir=${2} build_dir=${3}/java_jni # The directory where the final binaries will be stored when scripts finish dist_dir=${4} -normalized_arch=$(arch) -case ${normalized_arch} in - aarch64) - normalized_arch=aarch_64 - ;; - i386) - normalized_arch=x86_64 - ;; - arm64) - normalized_arch=aarch_64 - ;; - esac - prefix_dir="${build_dir}/java-jni" echo "=== Clear output directories and leftovers ===" @@ -68,8 +55,7 @@ cmake \ -DBUILD_TESTING=${ARROW_JAVA_BUILD_TESTS} \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_PREFIX_PATH=${arrow_install_dir} \ - -DARROW_JAVA_JNI_ARCH_DIR=${normalized_arch} \ - -DCMAKE_INSTALL_PREFIX=${prefix_dir} \ +\ -DCMAKE_INSTALL_PREFIX=${prefix_dir} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ -DProtobuf_USE_STATIC_LIBS=ON \ -GNinja \ diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 04fa51ff98c..e4c4dfd95fb 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -74,6 +74,20 @@ if(BUILD_TESTING) GTest::gtest_main) endif() +# If the user hasn't set ARROW_JAVA_JNI_ARCH_DIR, derive the normalized +# operating system from the host processor. +if("${ARROW_JAVA_JNI_ARCH_DIR}" STREQUAL "") + if("${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + set(ARROW_JAVA_JNI_ARCH_DIR "aarch_64") + elseif("${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "i386") + set(ARROW_JAVA_JNI_ARCH_DIR "x86_64") + elseif("${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "arm64") + set(ARROW_JAVA_JNI_ARCH_DIR "aarch_64") + else() + set(ARROW_JAVA_JNI_ARCH_DIR "${CMAKE_HOST_SYSTEM_PROCESSOR}") + endif() +endif() + if(ARROW_JAVA_JNI_ENABLE_C) add_subdirectory(c) endif() diff --git a/java/c/CMakeLists.txt b/java/c/CMakeLists.txt index 8e307672d0e..83909c5e13e 100644 --- a/java/c/CMakeLists.txt +++ b/java/c/CMakeLists.txt @@ -30,11 +30,11 @@ add_library(arrow_java_jni_cdata SHARED src/main/cpp/jni_wrapper.cc) set_property(TARGET arrow_java_jni_cdata PROPERTY OUTPUT_NAME "arrow_cdata_jni") target_link_libraries(arrow_java_jni_cdata arrow_java_jni_cdata_headers jni) -set(ARROW_JAVA_JNI_CDATA_LIBDIR +set(ARROW_JAVA_JNI_C_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib/arrow_cdata_jni/${ARROW_JAVA_JNI_ARCH_DIR}") -set(ARROW_JAVA_JNI_CDATA_BINDIR +set(ARROW_JAVA_JNI_C_BINDIR "${CMAKE_INSTALL_PREFIX}/bin/arrow_cdata_jni/${ARROW_JAVA_JNI_ARCH_DIR}") install(TARGETS arrow_java_jni_cdata - LIBRARY DESTINATION ${ARROW_JAVA_JNI_CDATA_LIBDIR} - RUNTIME DESTINATION ${ARROW_JAVA_JNI_CDATA_BINDIR}) + LIBRARY DESTINATION ${ARROW_JAVA_JNI_C_LIBDIR} + RUNTIME DESTINATION ${ARROW_JAVA_JNI_C_BINDIR}) From c5723d1a2358854f65b57541ff48c67385397c49 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 19 Jan 2024 11:47:10 -0800 Subject: [PATCH 53/54] PR suggestions Remove explicit setting of ARROW_JAVA_JNI_ARCH_DIR in build system. Clarify that this should get auto-detected. --- ci/scripts/java_jni_build.sh | 2 +- ci/scripts/java_jni_manylinux_build.sh | 1 - ci/scripts/java_jni_windows_build.sh | 1 - docs/source/developers/java/building.rst | 6 ------ java/CMakeLists.txt | 13 +++++++------ java/pom.xml | 5 ----- 6 files changed, 8 insertions(+), 20 deletions(-) diff --git a/ci/scripts/java_jni_build.sh b/ci/scripts/java_jni_build.sh index cbb754b3e48..d989351ab7e 100755 --- a/ci/scripts/java_jni_build.sh +++ b/ci/scripts/java_jni_build.sh @@ -55,7 +55,7 @@ cmake \ -DBUILD_TESTING=${ARROW_JAVA_BUILD_TESTS} \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_PREFIX_PATH=${arrow_install_dir} \ -\ -DCMAKE_INSTALL_PREFIX=${prefix_dir} \ + -DCMAKE_INSTALL_PREFIX=${prefix_dir} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ -DProtobuf_USE_STATIC_LIBS=ON \ -GNinja \ diff --git a/ci/scripts/java_jni_manylinux_build.sh b/ci/scripts/java_jni_manylinux_build.sh index 572cc79a232..da4987d307c 100755 --- a/ci/scripts/java_jni_manylinux_build.sh +++ b/ci/scripts/java_jni_manylinux_build.sh @@ -91,7 +91,6 @@ cmake \ -DARROW_S3=${ARROW_S3} \ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DARROW_JAVA_JNI_ARCH_DIR=${normalized_arch} \ -DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ -DGTest_SOURCE=BUNDLED \ diff --git a/ci/scripts/java_jni_windows_build.sh b/ci/scripts/java_jni_windows_build.sh index b0916a9eb8b..39288f4a9d0 100755 --- a/ci/scripts/java_jni_windows_build.sh +++ b/ci/scripts/java_jni_windows_build.sh @@ -72,7 +72,6 @@ cmake \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_ZSTD=ON \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DARROW_JAVA_JNI_ARCH_DIR=x86_64 \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ -GNinja \ diff --git a/docs/source/developers/java/building.rst b/docs/source/developers/java/building.rst index 6e17b948834..27e2de97328 100644 --- a/docs/source/developers/java/building.rst +++ b/docs/source/developers/java/building.rst @@ -165,7 +165,6 @@ CMake -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF \ -DBUILD_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DARROW_JAVA_JNI_ARCH_DIR= \ -DCMAKE_INSTALL_PREFIX=java-dist $ cmake --build java-cdata --target install --config Release $ ls -latr java-dist/lib @@ -184,7 +183,6 @@ CMake -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF ^ -DBUILD_TESTING=OFF ^ -DCMAKE_BUILD_TYPE=Release ^ - -DARROW_JAVA_JNI_ARCH_DIR=x86_64 ^ -DCMAKE_INSTALL_PREFIX=java-dist $ cmake --build java-cdata --target install --config Release $ dir "java-dist/bin" @@ -220,7 +218,6 @@ CMake -DARROW_SUBSTRAIT=ON \ -DARROW_USE_CCACHE=ON \ -DCMAKE_BUILD_TYPE=Release \ - -DARROW_JAVA_JNI_ARCH_DIR= \ -DCMAKE_INSTALL_PREFIX=java-dist \ -DCMAKE_UNITY_BUILD=ON $ cmake --build cpp-jni --target install --config Release @@ -231,7 +228,6 @@ CMake -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON \ -DBUILD_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DARROW_JAVA_JNI_ARCH_DIR= \ -DCMAKE_INSTALL_PREFIX=java-dist \ -DCMAKE_PREFIX_PATH=$PWD/java-dist \ -DProtobuf_ROOT=$PWD/../cpp-jni/protobuf_ep-install \ @@ -269,7 +265,6 @@ CMake -DARROW_WITH_ZLIB=ON ^ -DARROW_WITH_ZSTD=ON ^ -DCMAKE_BUILD_TYPE=Release ^ - -DARROW_JAVA_JNI_ARCH_DIR=x86_64 ^ -DCMAKE_INSTALL_PREFIX=java-dist ^ -DCMAKE_UNITY_BUILD=ON ^ -GNinja @@ -286,7 +281,6 @@ CMake -DARROW_JAVA_JNI_ENABLE_ORC=ON ^ -DBUILD_TESTING=OFF ^ -DCMAKE_BUILD_TYPE=Release ^ - -DARROW_JAVA_JNI_ARCH_DIR=x86_64 ^ -DCMAKE_INSTALL_PREFIX=java-dist ^ -DCMAKE_PREFIX_PATH=$PWD/java-dist $ cmake --build java-jni --target install --config Release diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index e4c4dfd95fb..2389c183cfd 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -74,17 +74,18 @@ if(BUILD_TESTING) GTest::gtest_main) endif() -# If the user hasn't set ARROW_JAVA_JNI_ARCH_DIR, derive the normalized -# operating system from the host processor. +# The ARROW_JAVA_JNI_ARCH_DIR will automatically be derived the normalized +# operating system from system processor. The user can override this variable +# if auto-detection fails. if("${ARROW_JAVA_JNI_ARCH_DIR}" STREQUAL "") - if("${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + if("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") set(ARROW_JAVA_JNI_ARCH_DIR "aarch_64") - elseif("${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "i386") + elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "i386") set(ARROW_JAVA_JNI_ARCH_DIR "x86_64") - elseif("${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "arm64") + elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "arm64") set(ARROW_JAVA_JNI_ARCH_DIR "aarch_64") else() - set(ARROW_JAVA_JNI_ARCH_DIR "${CMAKE_HOST_SYSTEM_PROCESSOR}") + set(ARROW_JAVA_JNI_ARCH_DIR "${CMAKE_SYSTEM_PROCESSOR}") endif() endif() diff --git a/java/pom.xml b/java/pom.xml index 4bef553e58b..6fc4df67af3 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -1055,7 +1055,6 @@ -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release - -DARROW_JAVA_JNI_ARCH_DIR=${os.detected.arch} -DCMAKE_INSTALL_PREFIX=${arrow.c.jni.dist.dir} ../ @@ -1128,7 +1127,6 @@ -DARROW_SUBSTRAIT=${ARROW_DATASET} -DARROW_USE_CCACHE=ON -DCMAKE_BUILD_TYPE=Release - -DARROW_JAVA_JNI_ARCH_DIR=${os.detected.arch} -DCMAKE_INSTALL_PREFIX=java-dist -DCMAKE_UNITY_BUILD=ON @@ -1169,7 +1167,6 @@ -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release - -DARROW_JAVA_JNI_ARCH_DIR=${os.detected.arch} -DCMAKE_INSTALL_PREFIX=${arrow.dataset.jni.dist.dir} -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist/lib/${os.detected.arch}/cmake -DProtobuf_USE_STATIC_LIBS=ON @@ -1248,7 +1245,6 @@ -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DCMAKE_BUILD_TYPE=Release - -DARROW_JAVA_JNI_ARCH_DIR=${os.detected.arch} -DCMAKE_INSTALL_PREFIX=java-dist -DCMAKE_UNITY_BUILD=ON -GNinja @@ -1290,7 +1286,6 @@ -DARROW_JAVA_JNI_ENABLE_DEFAULT=ON -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release - -DARROW_JAVA_JNI_ARCH_DIR=${os.detected.arch} -DCMAKE_INSTALL_PREFIX=${arrow.dataset.jni.dist.dir} -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist/lib/${os.detected.arch}/cmake From 0ece855474a810301d2e1b7c09451cd155a3fb26 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 19 Jan 2024 12:36:21 -0800 Subject: [PATCH 54/54] Fix Windows build --- ci/scripts/java_jni_macos_build.sh | 1 - java/CMakeLists.txt | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index 5d9c3ce15d9..4ecc029bdd3 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -82,7 +82,6 @@ cmake \ -DARROW_S3=${ARROW_S3} \ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DARROW_JAVA_JNI_ARCH_DIR=${normalized_arch} \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ -DGTest_SOURCE=BUNDLED \ diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 2389c183cfd..8b29f37d80a 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -84,6 +84,8 @@ if("${ARROW_JAVA_JNI_ARCH_DIR}" STREQUAL "") set(ARROW_JAVA_JNI_ARCH_DIR "x86_64") elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "arm64") set(ARROW_JAVA_JNI_ARCH_DIR "aarch_64") + elseif("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "AMD64") + set(ARROW_JAVA_JNI_ARCH_DIR "x86_64") else() set(ARROW_JAVA_JNI_ARCH_DIR "${CMAKE_SYSTEM_PROCESSOR}") endif()