From a97a6617eedb2a5765e7dc79c82480bb1531272d Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 23 Nov 2023 12:31:40 -0800 Subject: [PATCH 01/35] Fix CData related test failures --- java/c/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/java/c/pom.xml b/java/c/pom.xml index d66379d3566..8fc3f36994d 100644 --- a/java/c/pom.xml +++ b/java/c/pom.xml @@ -53,6 +53,11 @@ arrow-memory-unsafe test + + org.apache.arrow + arrow-format + test + com.google.guava guava From b605164281ef5233d67fe2f0a01db2ab0989a00e Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 23 Nov 2023 12:32:25 -0800 Subject: [PATCH 02/35] Standardize use of maven-shade-plugin Also set all components to use the latest version already used in the codebase (3.4.1) --- java/flight/flight-core/pom.xml | 1 - java/flight/flight-sql-jdbc-driver/pom.xml | 1 - java/performance/pom.xml | 1 - java/pom.xml | 5 +++++ java/vector/pom.xml | 1 - 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index 7b69179053d..1c9d3227b80 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -149,7 +149,6 @@ org.apache.maven.plugins maven-shade-plugin - 3.1.1 shade-main diff --git a/java/flight/flight-sql-jdbc-driver/pom.xml b/java/flight/flight-sql-jdbc-driver/pom.xml index 263538ba48b..d301f28341e 100644 --- a/java/flight/flight-sql-jdbc-driver/pom.xml +++ b/java/flight/flight-sql-jdbc-driver/pom.xml @@ -159,7 +159,6 @@ org.apache.maven.plugins maven-shade-plugin - 3.4.1 package diff --git a/java/performance/pom.xml b/java/performance/pom.xml index 102832491ec..94fba35278c 100644 --- a/java/performance/pom.xml +++ b/java/performance/pom.xml @@ -103,7 +103,6 @@ org.apache.maven.plugins maven-shade-plugin - 2.2 package diff --git a/java/pom.xml b/java/pom.xml index 2a9997b7012..1e2ab5e5233 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -408,6 +408,11 @@ maven-enforcer-plugin 3.0.0-M2 + + org.apache.maven.plugins + maven-shade-plugin + 3.4.1 + maven-surefire-plugin 3.0.0-M7 diff --git a/java/vector/pom.xml b/java/vector/pom.xml index 4c8bf1e594a..9cedcfdaa44 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -174,7 +174,6 @@ org.apache.maven.plugins maven-shade-plugin - 3.1.1 package From 7abc7730f231001fc0b16d54ebf349b874f0d7b3 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 23 Nov 2023 12:33:05 -0800 Subject: [PATCH 03/35] Update maven-shade-plugin to 3.5.1 --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 1e2ab5e5233..52803f12b6a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -411,7 +411,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.4.1 + 3.5.1 maven-surefire-plugin From 25be61094759bd7523d4a623b2f0c154d9ecd0bf Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 23 Nov 2023 13:38:42 -0800 Subject: [PATCH 04/35] Update CycloneDX plugin to 2.7.10 Needed because upgrading grpc-java past 1.58 breaks CycloneDX 2.7.6 and grpc-java 1.58 is needed for modules --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 52803f12b6a..2676d3531ba 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -361,7 +361,7 @@ org.cyclonedx cyclonedx-maven-plugin - 2.7.6 + 2.7.10 package From 0695e468492572469e09e950454be478ed8f8ed1 Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 23 Nov 2023 12:47:50 -0800 Subject: [PATCH 05/35] Update grpc-java to 1.59 1.58 fixes issues around module support --- java/flight/flight-core/pom.xml | 4 ---- java/flight/flight-grpc/pom.xml | 10 +++++----- java/pom.xml | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index 1c9d3227b80..30facf734b5 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -54,10 +54,6 @@ io.grpc grpc-core - - io.grpc - grpc-context - io.grpc grpc-protobuf diff --git a/java/flight/flight-grpc/pom.xml b/java/flight/flight-grpc/pom.xml index e7bb9508d24..727c3398f18 100644 --- a/java/flight/flight-grpc/pom.xml +++ b/java/flight/flight-grpc/pom.xml @@ -48,12 +48,12 @@ io.grpc - grpc-core + grpc-stub + + + io.grpc + grpc-inprocess - - io.grpc - grpc-stub - org.apache.arrow arrow-memory-core diff --git a/java/pom.xml b/java/pom.xml index 2676d3531ba..26bb3b64ef9 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -34,7 +34,7 @@ 1.7.25 31.1-jre 4.1.100.Final - 1.56.0 + 1.59.0 3.23.1 2.15.1 2.7.1 From 0d118ec934713fe04bcbbc056b313ab06fef95bd Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 23 Nov 2023 13:40:39 -0800 Subject: [PATCH 06/35] Update maven-depndency-plugin to 3.6.1 Needed to be able to handle module-info.class files Resolve various errors about test-only dependencies. Note that arrow-tools doesn't need a compile-dependency on jackson because it gets it through the shaded arrow-vector JAR. --- java/algorithm/pom.xml | 1 + java/flight/flight-grpc/pom.xml | 1 + java/flight/flight-sql/pom.xml | 2 ++ java/performance/pom.xml | 6 ++++++ java/pom.xml | 2 +- java/tools/pom.xml | 3 +++ 6 files changed, 14 insertions(+), 1 deletion(-) diff --git a/java/algorithm/pom.xml b/java/algorithm/pom.xml index 8c6a9fb0151..3e32d955ec4 100644 --- a/java/algorithm/pom.xml +++ b/java/algorithm/pom.xml @@ -31,6 +31,7 @@ arrow-vector ${project.version} test-jar + test org.apache.arrow diff --git a/java/flight/flight-grpc/pom.xml b/java/flight/flight-grpc/pom.xml index 727c3398f18..af765f8c436 100644 --- a/java/flight/flight-grpc/pom.xml +++ b/java/flight/flight-grpc/pom.xml @@ -53,6 +53,7 @@ io.grpc grpc-inprocess + test org.apache.arrow diff --git a/java/flight/flight-sql/pom.xml b/java/flight/flight-sql/pom.xml index 09100d9731a..25478e58d29 100644 --- a/java/flight/flight-sql/pom.xml +++ b/java/flight/flight-sql/pom.xml @@ -53,6 +53,7 @@ org.apache.arrow arrow-jdbc + test com.google.guava @@ -70,6 +71,7 @@ org.slf4j slf4j-api + test org.apache.derby diff --git a/java/performance/pom.xml b/java/performance/pom.xml index 94fba35278c..269ac72d833 100644 --- a/java/performance/pom.xml +++ b/java/performance/pom.xml @@ -26,6 +26,7 @@ org.openjdk.jmh jmh-core ${jmh.version} + test org.openjdk.jmh @@ -37,10 +38,12 @@ org.apache.arrow arrow-vector ${arrow.vector.classifier} + test org.apache.arrow arrow-memory-core + test org.apache.arrow @@ -51,10 +54,12 @@ org.apache.avro avro ${dep.avro.version} + test org.apache.arrow arrow-avro + test com.h2database @@ -65,6 +70,7 @@ org.apache.arrow arrow-jdbc + test org.apache.arrow diff --git a/java/pom.xml b/java/pom.xml index 26bb3b64ef9..d3c2422e3ad 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -378,7 +378,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.0.1 + 3.6.1 org.apache.rat diff --git a/java/tools/pom.xml b/java/tools/pom.xml index 128825c2243..2ebabe74e6b 100644 --- a/java/tools/pom.xml +++ b/java/tools/pom.xml @@ -37,6 +37,7 @@ com.google.guava guava + test commons-cli @@ -52,10 +53,12 @@ com.fasterxml.jackson.core jackson-core + provided com.fasterxml.jackson.core jackson-databind + provided org.slf4j From 607a9c15a781f2762b22103a5c0db0378b406e8a Mon Sep 17 00:00:00 2001 From: James Duong Date: Thu, 23 Nov 2023 14:57:24 -0800 Subject: [PATCH 07/35] Exclude module-info.java files from the build They currently don't build with JDK 8. --- java/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/java/pom.xml b/java/pom.xml index d3c2422e3ad..2b452f6d889 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -395,6 +395,7 @@ maven-compiler-plugin ${maven-compiler-plugin.version} + **/module-info.java org.immutables From a9dba8ac68e4f1e25d13ca1005f0519df4b136a5 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 24 Nov 2023 06:30:46 -0800 Subject: [PATCH 08/35] Refactor memory modules for JPMS support Based on #13072 - Avoid having multiple memory modules contribute to the same package - Introduce memory-netty-buffer-patch module for patching classes into Netty modules - Patch classes into netty's io.netty.buffer package (specifically use the JDK9 compiler to do this instead of inheriting JDK8). - Avoid using BaseAllocator in tests and use RootAllocator instead - Move TestBaseAllocator#testMemoryUsage() to a new test in memory-netty TestNettyAllocator because TestBaseAllocator is now in memory-core, but that specific test has Netty dependencies. --- .../apache/arrow/memory/CheckAllocator.java | 39 ++++- .../DefaultAllocationManagerOption.java | 13 +- .../memory/CountingAllocationListener.java | 4 + .../arrow/memory/TestBaseAllocator.java | 134 ++++++------------ .../arrow/memory/TestForeignAllocation.java | 7 + java/memory/memory-netty-buffer-patch/pom.xml | 44 ++++++ .../io/netty/buffer/ExpandableByteBuf.java | 0 .../java/io/netty/buffer/LargeBuffer.java | 0 .../netty/buffer/MutableWrappedByteBuf.java | 0 .../java/io/netty/buffer/NettyArrowBuf.java | 16 ++- .../netty/buffer/PooledByteBufAllocatorL.java | 7 +- .../buffer/UnsafeDirectLittleEndian.java | 0 .../memory/patch}/ArrowByteBufAllocator.java | 4 +- .../buffer/TestUnsafeDirectLittleEndian.java | 5 + java/memory/memory-netty/pom.xml | 23 +++ .../DefaultAllocationManagerFactory.java | 6 +- .../{ => netty}/NettyAllocationManager.java | 7 +- .../netty/buffer/TestExpandableByteBuf.java | 4 + .../io/netty/buffer/TestNettyArrowBuf.java | 5 +- .../{ => netty}/ITTestLargeArrowBuf.java | 5 +- .../TestAllocationManagerNetty.java | 4 +- .../memory/{ => netty}/TestEmptyArrowBuf.java | 9 +- .../memory/{ => netty}/TestEndianness.java | 4 +- .../TestNettyAllocationManager.java | 15 +- .../memory/netty/TestNettyAllocator.java | 75 ++++++++++ .../DefaultAllocationManagerFactory.java | 6 +- .../{ => unsafe}/UnsafeAllocationManager.java | 6 +- .../TestAllocationManagerUnsafe.java | 4 +- .../TestUnsafeAllocationManager.java | 13 +- java/memory/pom.xml | 1 + 30 files changed, 330 insertions(+), 130 deletions(-) rename java/memory/{memory-netty => memory-core}/src/test/java/org/apache/arrow/memory/CountingAllocationListener.java (95%) rename java/memory/{memory-netty => memory-core}/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java (91%) rename java/memory/{memory-netty => memory-core}/src/test/java/org/apache/arrow/memory/TestForeignAllocation.java (93%) create mode 100644 java/memory/memory-netty-buffer-patch/pom.xml rename java/memory/{memory-netty => memory-netty-buffer-patch}/src/main/java/io/netty/buffer/ExpandableByteBuf.java (100%) rename java/memory/{memory-netty => memory-netty-buffer-patch}/src/main/java/io/netty/buffer/LargeBuffer.java (100%) rename java/memory/{memory-netty => memory-netty-buffer-patch}/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java (100%) rename java/memory/{memory-netty => memory-netty-buffer-patch}/src/main/java/io/netty/buffer/NettyArrowBuf.java (96%) rename java/memory/{memory-netty => memory-netty-buffer-patch}/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java (97%) rename java/memory/{memory-netty => memory-netty-buffer-patch}/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java (100%) rename java/memory/{memory-netty/src/main/java/org/apache/arrow/memory => memory-netty-buffer-patch/src/main/java/org/apache/arrow/memory/patch}/ArrowByteBufAllocator.java (97%) rename java/memory/{memory-netty => memory-netty-buffer-patch}/src/test/java/io/netty/buffer/TestUnsafeDirectLittleEndian.java (95%) rename java/memory/memory-netty/src/main/java/org/apache/arrow/memory/{ => netty}/DefaultAllocationManagerFactory.java (87%) rename java/memory/memory-netty/src/main/java/org/apache/arrow/memory/{ => netty}/NettyAllocationManager.java (94%) rename java/memory/memory-netty/src/test/java/org/apache/arrow/memory/{ => netty}/ITTestLargeArrowBuf.java (93%) rename java/memory/memory-netty/src/test/java/org/apache/arrow/memory/{ => netty}/TestAllocationManagerNetty.java (90%) rename java/memory/memory-netty/src/test/java/org/apache/arrow/memory/{ => netty}/TestEmptyArrowBuf.java (90%) rename java/memory/memory-netty/src/test/java/org/apache/arrow/memory/{ => netty}/TestEndianness.java (92%) rename java/memory/memory-netty/src/test/java/org/apache/arrow/memory/{ => netty}/TestNettyAllocationManager.java (87%) create mode 100644 java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestNettyAllocator.java rename java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/{ => unsafe}/DefaultAllocationManagerFactory.java (87%) rename java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/{ => unsafe}/UnsafeAllocationManager.java (89%) rename java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/{ => unsafe}/TestAllocationManagerUnsafe.java (90%) rename java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/{ => unsafe}/TestUnsafeAllocationManager.java (82%) diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/CheckAllocator.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/CheckAllocator.java index 79b825aa2e8..dac4a3fcff5 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/CheckAllocator.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/CheckAllocator.java @@ -31,20 +31,35 @@ */ final class CheckAllocator { private static final Logger logger = LoggerFactory.getLogger(CheckAllocator.class); - private static final String ALLOCATOR_PATH = "org/apache/arrow/memory/DefaultAllocationManagerFactory.class"; + // unique package names needed by JPMS module naming + private static final String ALLOCATOR_PATH_CORE = + "org/apache/arrow/memory/DefaultAllocationManagerFactory.class"; + private static final String ALLOCATOR_PATH_UNSAFE = + "org/apache/arrow/memory/unsafe/DefaultAllocationManagerFactory.class"; + private static final String ALLOCATOR_PATH_NETTY = + "org/apache/arrow/memory/netty/DefaultAllocationManagerFactory.class"; private CheckAllocator() { - } static String check() { Set urls = scanClasspath(); URL rootAllocator = assertOnlyOne(urls); reportResult(rootAllocator); - return "org.apache.arrow.memory.DefaultAllocationManagerFactory"; + if (rootAllocator.getPath().contains("memory-core") || + rootAllocator.getPath().contains("/org/apache/arrow/memory/core/")) { + return "org.apache.arrow.memory.DefaultAllocationManagerFactory"; + } else if (rootAllocator.getPath().contains("memory-unsafe") || + rootAllocator.getPath().contains("/org/apache/arrow/memory/unsafe/")) { + return "org.apache.arrow.memory.unsafe.DefaultAllocationManagerFactory"; + } else if (rootAllocator.getPath().contains("memory-netty") || + rootAllocator.getPath().contains("/org/apache/arrow/memory/netty/")) { + return "org.apache.arrow.memory.netty.DefaultAllocationManagerFactory"; + } else { + throw new IllegalStateException("Unknown allocation manager type to infer. Current: " + rootAllocator.getPath()); + } } - private static Set scanClasspath() { // LinkedHashSet appropriate here because it preserves insertion order // during iteration @@ -53,9 +68,21 @@ private static Set scanClasspath() { ClassLoader allocatorClassLoader = CheckAllocator.class.getClassLoader(); Enumeration paths; if (allocatorClassLoader == null) { - paths = ClassLoader.getSystemResources(ALLOCATOR_PATH); + paths = ClassLoader.getSystemResources(ALLOCATOR_PATH_CORE); + if (!paths.hasMoreElements()) { + paths = ClassLoader.getSystemResources(ALLOCATOR_PATH_UNSAFE); + } + if (!paths.hasMoreElements()) { + paths = ClassLoader.getSystemResources(ALLOCATOR_PATH_NETTY); + } } else { - paths = allocatorClassLoader.getResources(ALLOCATOR_PATH); + paths = allocatorClassLoader.getResources(ALLOCATOR_PATH_CORE); + if (!paths.hasMoreElements()) { + paths = allocatorClassLoader.getResources(ALLOCATOR_PATH_UNSAFE); + } + if (!paths.hasMoreElements()) { + paths = allocatorClassLoader.getResources(ALLOCATOR_PATH_NETTY); + } } while (paths.hasMoreElements()) { URL path = paths.nextElement(); diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java index 15120c252fc..564d59994b7 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java @@ -19,6 +19,8 @@ import java.lang.reflect.Field; +import org.apache.arrow.util.VisibleForTesting; + /** * A class for choosing the default allocation manager. */ @@ -61,7 +63,12 @@ public enum AllocationManagerType { Unknown, } - static AllocationManagerType getDefaultAllocationManagerType() { + /** + * Returns the default allocation manager type. + * @return the default allocation manager type. + */ + @VisibleForTesting + public static AllocationManagerType getDefaultAllocationManagerType() { AllocationManagerType ret = AllocationManagerType.Unknown; try { @@ -115,7 +122,7 @@ private static AllocationManager.Factory getFactory(String clazzName) { private static AllocationManager.Factory getUnsafeFactory() { try { - return getFactory("org.apache.arrow.memory.UnsafeAllocationManager"); + return getFactory("org.apache.arrow.memory.unsafe.UnsafeAllocationManager"); } catch (RuntimeException e) { throw new RuntimeException("Please add arrow-memory-unsafe to your classpath," + " No DefaultAllocationManager found to instantiate an UnsafeAllocationManager", e); @@ -124,7 +131,7 @@ private static AllocationManager.Factory getUnsafeFactory() { private static AllocationManager.Factory getNettyFactory() { try { - return getFactory("org.apache.arrow.memory.NettyAllocationManager"); + return getFactory("org.apache.arrow.memory.netty.NettyAllocationManager"); } catch (RuntimeException e) { throw new RuntimeException("Please add arrow-memory-netty to your classpath," + " No DefaultAllocationManager found to instantiate an NettyAllocationManager", e); diff --git a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/CountingAllocationListener.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/CountingAllocationListener.java similarity index 95% rename from java/memory/memory-netty/src/test/java/org/apache/arrow/memory/CountingAllocationListener.java rename to java/memory/memory-core/src/test/java/org/apache/arrow/memory/CountingAllocationListener.java index 78c78c8ad8c..f1dd7e92c5c 100644 --- a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/CountingAllocationListener.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/CountingAllocationListener.java @@ -17,6 +17,10 @@ package org.apache.arrow.memory; +import org.apache.arrow.memory.AllocationListener; +import org.apache.arrow.memory.AllocationOutcome; +import org.apache.arrow.memory.BufferAllocator; + /** * Counting allocation listener. * It counts the number of times it has been invoked, and how much memory allocation it has seen diff --git a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java similarity index 91% rename from java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java rename to java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java index 7613d073f8c..365c84f5a2b 100644 --- a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java @@ -31,22 +31,16 @@ import java.util.Collection; import java.util.Collections; import java.util.Iterator; -import java.util.stream.Collectors; import org.apache.arrow.memory.AllocationOutcomeDetails.Entry; import org.apache.arrow.memory.rounding.RoundingPolicy; import org.apache.arrow.memory.rounding.SegmentRoundingPolicy; import org.apache.arrow.memory.util.AssertionUtil; +import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; import org.junit.jupiter.api.Assertions; -import org.slf4j.LoggerFactory; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.Logger; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.read.ListAppender; -import io.netty.buffer.PooledByteBufAllocatorL; import sun.misc.Unsafe; public class TestBaseAllocator { @@ -448,73 +442,73 @@ public ArrowBuf empty() { @Test public void testRootAllocator_listeners() throws Exception { CountingAllocationListener l1 = new CountingAllocationListener(); - assertEquals(0, l1.getNumPreCalls()); - assertEquals(0, l1.getNumCalls()); - assertEquals(0, l1.getNumReleaseCalls()); - assertEquals(0, l1.getNumChildren()); - assertEquals(0, l1.getTotalMem()); + Assert.assertEquals(0, l1.getNumPreCalls()); + Assert.assertEquals(0, l1.getNumCalls()); + Assert.assertEquals(0, l1.getNumReleaseCalls()); + Assert.assertEquals(0, l1.getNumChildren()); + Assert.assertEquals(0, l1.getTotalMem()); CountingAllocationListener l2 = new CountingAllocationListener(); - assertEquals(0, l2.getNumPreCalls()); - assertEquals(0, l2.getNumCalls()); - assertEquals(0, l2.getNumReleaseCalls()); - assertEquals(0, l2.getNumChildren()); - assertEquals(0, l2.getTotalMem()); + Assert.assertEquals(0, l2.getNumPreCalls()); + Assert.assertEquals(0, l2.getNumCalls()); + Assert.assertEquals(0, l2.getNumReleaseCalls()); + Assert.assertEquals(0, l2.getNumChildren()); + Assert.assertEquals(0, l2.getTotalMem()); // root and first-level child share the first listener // second-level and third-level child share the second listener try (final RootAllocator rootAllocator = new RootAllocator(l1, MAX_ALLOCATION)) { try (final BufferAllocator c1 = rootAllocator.newChildAllocator("c1", 0, MAX_ALLOCATION)) { - assertEquals(1, l1.getNumChildren()); + Assert.assertEquals(1, l1.getNumChildren()); final ArrowBuf buf1 = c1.buffer(16); assertNotNull("allocation failed", buf1); - assertEquals(1, l1.getNumPreCalls()); - assertEquals(1, l1.getNumCalls()); - assertEquals(0, l1.getNumReleaseCalls()); - assertEquals(16, l1.getTotalMem()); + Assert.assertEquals(1, l1.getNumPreCalls()); + Assert.assertEquals(1, l1.getNumCalls()); + Assert.assertEquals(0, l1.getNumReleaseCalls()); + Assert.assertEquals(16, l1.getTotalMem()); buf1.getReferenceManager().release(); try (final BufferAllocator c2 = c1.newChildAllocator("c2", l2, 0, MAX_ALLOCATION)) { - assertEquals(2, l1.getNumChildren()); // c1 got a new child, so c1's listener (l1) is notified - assertEquals(0, l2.getNumChildren()); + Assert.assertEquals(2, l1.getNumChildren()); // c1 got a new child, so c1's listener (l1) is notified + Assert.assertEquals(0, l2.getNumChildren()); final ArrowBuf buf2 = c2.buffer(32); assertNotNull("allocation failed", buf2); - assertEquals(1, l1.getNumCalls()); - assertEquals(16, l1.getTotalMem()); - assertEquals(1, l2.getNumPreCalls()); - assertEquals(1, l2.getNumCalls()); - assertEquals(0, l2.getNumReleaseCalls()); - assertEquals(32, l2.getTotalMem()); + Assert.assertEquals(1, l1.getNumCalls()); + Assert.assertEquals(16, l1.getTotalMem()); + Assert.assertEquals(1, l2.getNumPreCalls()); + Assert.assertEquals(1, l2.getNumCalls()); + Assert.assertEquals(0, l2.getNumReleaseCalls()); + Assert.assertEquals(32, l2.getTotalMem()); buf2.getReferenceManager().release(); try (final BufferAllocator c3 = c2.newChildAllocator("c3", 0, MAX_ALLOCATION)) { - assertEquals(2, l1.getNumChildren()); - assertEquals(1, l2.getNumChildren()); + Assert.assertEquals(2, l1.getNumChildren()); + Assert.assertEquals(1, l2.getNumChildren()); final ArrowBuf buf3 = c3.buffer(64); assertNotNull("allocation failed", buf3); - assertEquals(1, l1.getNumPreCalls()); - assertEquals(1, l1.getNumCalls()); - assertEquals(1, l1.getNumReleaseCalls()); - assertEquals(16, l1.getTotalMem()); - assertEquals(2, l2.getNumPreCalls()); - assertEquals(2, l2.getNumCalls()); - assertEquals(1, l2.getNumReleaseCalls()); - assertEquals(32 + 64, l2.getTotalMem()); + Assert.assertEquals(1, l1.getNumPreCalls()); + Assert.assertEquals(1, l1.getNumCalls()); + Assert.assertEquals(1, l1.getNumReleaseCalls()); + Assert.assertEquals(16, l1.getTotalMem()); + Assert.assertEquals(2, l2.getNumPreCalls()); + Assert.assertEquals(2, l2.getNumCalls()); + Assert.assertEquals(1, l2.getNumReleaseCalls()); + Assert.assertEquals(32 + 64, l2.getTotalMem()); buf3.getReferenceManager().release(); } - assertEquals(2, l1.getNumChildren()); - assertEquals(0, l2.getNumChildren()); // third-level child removed + Assert.assertEquals(2, l1.getNumChildren()); + Assert.assertEquals(0, l2.getNumChildren()); // third-level child removed } - assertEquals(1, l1.getNumChildren()); // second-level child removed - assertEquals(0, l2.getNumChildren()); + Assert.assertEquals(1, l1.getNumChildren()); // second-level child removed + Assert.assertEquals(0, l2.getNumChildren()); } - assertEquals(0, l1.getNumChildren()); // first-level child removed + Assert.assertEquals(0, l1.getNumChildren()); // first-level child removed - assertEquals(2, l2.getNumReleaseCalls()); + Assert.assertEquals(2, l2.getNumReleaseCalls()); } } @Test public void testRootAllocator_listenerAllocationFail() throws Exception { CountingAllocationListener l1 = new CountingAllocationListener(); - assertEquals(0, l1.getNumCalls()); - assertEquals(0, l1.getTotalMem()); + Assert.assertEquals(0, l1.getNumCalls()); + Assert.assertEquals(0, l1.getTotalMem()); // Test attempts to allocate too much from a child whose limit is set to half of the max // allocation. The listener's callback triggers, expanding the child allocator's limit, so then // the allocation succeeds. @@ -527,14 +521,14 @@ public void testRootAllocator_listenerAllocationFail() throws Exception { } catch (OutOfMemoryException e) { // expected } - assertEquals(0, l1.getNumCalls()); - assertEquals(0, l1.getTotalMem()); + Assert.assertEquals(0, l1.getNumCalls()); + Assert.assertEquals(0, l1.getTotalMem()); l1.setExpandOnFail(c1, MAX_ALLOCATION); ArrowBuf arrowBuf = c1.buffer(MAX_ALLOCATION); assertNotNull("allocation failed", arrowBuf); - assertEquals(1, l1.getNumCalls()); - assertEquals(MAX_ALLOCATION, l1.getTotalMem()); + Assert.assertEquals(1, l1.getNumCalls()); + Assert.assertEquals(MAX_ALLOCATION, l1.getTotalMem()); arrowBuf.getReferenceManager().release(); } } @@ -1098,42 +1092,6 @@ public void testMemoryLeakWithReservation() throws Exception { } } - @Test - public void testMemoryUsage() { - ListAppender memoryLogsAppender = new ListAppender<>(); - Logger logger = (Logger) LoggerFactory.getLogger("arrow.allocator"); - try { - logger.setLevel(Level.TRACE); - logger.addAppender(memoryLogsAppender); - memoryLogsAppender.start(); - try (ArrowBuf buf = new ArrowBuf(ReferenceManager.NO_OP, null, - 1024, new PooledByteBufAllocatorL().empty.memoryAddress())) { - buf.memoryAddress(); - } - boolean result = false; - long startTime = System.currentTimeMillis(); - while ((System.currentTimeMillis() - startTime) < 10000) { // 10 seconds maximum for time to read logs - result = memoryLogsAppender.list.stream() - .anyMatch( - log -> log.toString().contains("Memory Usage: \n") && - log.toString().contains("Large buffers outstanding: ") && - log.toString().contains("Normal buffers outstanding: ") && - log.getLevel().equals(Level.TRACE) - ); - if (result) { - break; - } - } - assertTrue("Log messages are:\n" + - memoryLogsAppender.list.stream().map(ILoggingEvent::toString).collect(Collectors.joining("\n")), - result); - } finally { - memoryLogsAppender.stop(); - logger.detachAppender(memoryLogsAppender); - logger.setLevel(null); - } - } - @Test public void testOverlimit() { try (BufferAllocator allocator = new RootAllocator(1024)) { diff --git a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestForeignAllocation.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestForeignAllocation.java similarity index 93% rename from java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestForeignAllocation.java rename to java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestForeignAllocation.java index ec049ca6923..46e94fad37b 100644 --- a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestForeignAllocation.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestForeignAllocation.java @@ -23,6 +23,13 @@ import java.util.ArrayList; import java.util.List; +import org.apache.arrow.memory.AllocationListener; +import org.apache.arrow.memory.AllocationOutcome; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.ForeignAllocation; +import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.memory.util.MemoryUtil; import org.junit.After; import org.junit.Before; diff --git a/java/memory/memory-netty-buffer-patch/pom.xml b/java/memory/memory-netty-buffer-patch/pom.xml new file mode 100644 index 00000000000..1d4407c638d --- /dev/null +++ b/java/memory/memory-netty-buffer-patch/pom.xml @@ -0,0 +1,44 @@ + + + + + arrow-memory + org.apache.arrow + 15.0.0-SNAPSHOT + + 4.0.0 + + arrow-memory-netty-buffer-patch + Arrow Memory - Netty Buffer + Netty Buffer needed to patch that is consumed by Arrow Memory Netty + + + + org.apache.arrow + arrow-memory-core + + + io.netty + netty-buffer + + + io.netty + netty-common + + + org.slf4j + slf4j-api + + + diff --git a/java/memory/memory-netty/src/main/java/io/netty/buffer/ExpandableByteBuf.java b/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/ExpandableByteBuf.java similarity index 100% rename from java/memory/memory-netty/src/main/java/io/netty/buffer/ExpandableByteBuf.java rename to java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/ExpandableByteBuf.java diff --git a/java/memory/memory-netty/src/main/java/io/netty/buffer/LargeBuffer.java b/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/LargeBuffer.java similarity index 100% rename from java/memory/memory-netty/src/main/java/io/netty/buffer/LargeBuffer.java rename to java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/LargeBuffer.java diff --git a/java/memory/memory-netty/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java b/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java similarity index 100% rename from java/memory/memory-netty/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java rename to java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java diff --git a/java/memory/memory-netty/src/main/java/io/netty/buffer/NettyArrowBuf.java b/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/NettyArrowBuf.java similarity index 96% rename from java/memory/memory-netty/src/main/java/io/netty/buffer/NettyArrowBuf.java rename to java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/NettyArrowBuf.java index 71e4b7cb6d5..466444c7d53 100644 --- a/java/memory/memory-netty/src/main/java/io/netty/buffer/NettyArrowBuf.java +++ b/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/NettyArrowBuf.java @@ -17,8 +17,6 @@ package io.netty.buffer; -import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -29,10 +27,12 @@ import java.nio.channels.ScatteringByteChannel; import org.apache.arrow.memory.ArrowBuf; -import org.apache.arrow.memory.ArrowByteBufAllocator; import org.apache.arrow.memory.BoundsChecking; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.patch.ArrowByteBufAllocator; +import org.apache.arrow.memory.util.LargeMemoryUtil; import org.apache.arrow.util.Preconditions; +import org.apache.arrow.util.VisibleForTesting; import io.netty.util.internal.PlatformDependent; @@ -264,7 +264,7 @@ public ByteBuffer nioBuffer(long index, int length) { * @return ByteBuffer */ private ByteBuffer getDirectBuffer(long index) { - return PlatformDependent.directBuffer(addr(index), checkedCastToInt(length - index)); + return PlatformDependent.directBuffer(addr(index), LargeMemoryUtil.checkedCastToInt(length - index)); } @Override @@ -580,11 +580,13 @@ public NettyArrowBuf setMedium(int index, int value) { } @Override + @VisibleForTesting protected void _setInt(int index, int value) { setInt(index, value); } @Override + @VisibleForTesting protected void _setIntLE(int index, int value) { this.chk(index, 4); PlatformDependent.putInt(this.addr(index), Integer.reverseBytes(value)); @@ -620,9 +622,9 @@ public static NettyArrowBuf unwrapBuffer(ArrowBuf buf) { final NettyArrowBuf nettyArrowBuf = new NettyArrowBuf( buf, buf.getReferenceManager().getAllocator(), - checkedCastToInt(buf.capacity())); - nettyArrowBuf.readerIndex(checkedCastToInt(buf.readerIndex())); - nettyArrowBuf.writerIndex(checkedCastToInt(buf.writerIndex())); + LargeMemoryUtil.checkedCastToInt(buf.capacity())); + nettyArrowBuf.readerIndex(LargeMemoryUtil.checkedCastToInt(buf.readerIndex())); + nettyArrowBuf.writerIndex(LargeMemoryUtil.checkedCastToInt(buf.writerIndex())); return nettyArrowBuf; } diff --git a/java/memory/memory-netty/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java similarity index 97% rename from java/memory/memory-netty/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java rename to java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java index 06c6669cfd1..bc95b9df545 100644 --- a/java/memory/memory-netty/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java +++ b/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java @@ -17,13 +17,12 @@ package io.netty.buffer; -import static org.apache.arrow.memory.util.AssertionUtil.ASSERT_ENABLED; - import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.util.concurrent.atomic.AtomicLong; import org.apache.arrow.memory.OutOfMemoryException; +import org.apache.arrow.memory.util.AssertionUtil; import org.apache.arrow.memory.util.LargeMemoryUtil; import io.netty.util.internal.OutOfDirectMemoryError; @@ -51,7 +50,7 @@ public PooledByteBufAllocatorL() { } /** - * Returns a {@linkplain io.netty.buffer.UnsafeDirectLittleEndian} of the given size. + * Returns a {@linkplain UnsafeDirectLittleEndian} of the given size. */ public UnsafeDirectLittleEndian allocate(long size) { try { @@ -183,7 +182,7 @@ private UnsafeDirectLittleEndian newDirectBufferL(int initialCapacity, int maxCa fail(); } - if (!ASSERT_ENABLED) { + if (!AssertionUtil.ASSERT_ENABLED) { return new UnsafeDirectLittleEndian((PooledUnsafeDirectByteBuf) buf); } diff --git a/java/memory/memory-netty/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java similarity index 100% rename from java/memory/memory-netty/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java rename to java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java diff --git a/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java b/java/memory/memory-netty-buffer-patch/src/main/java/org/apache/arrow/memory/patch/ArrowByteBufAllocator.java similarity index 97% rename from java/memory/memory-netty/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java rename to java/memory/memory-netty-buffer-patch/src/main/java/org/apache/arrow/memory/patch/ArrowByteBufAllocator.java index ff40b49ff6f..6ce08b5a590 100644 --- a/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java +++ b/java/memory/memory-netty-buffer-patch/src/main/java/org/apache/arrow/memory/patch/ArrowByteBufAllocator.java @@ -15,7 +15,9 @@ * limitations under the License. */ -package org.apache.arrow.memory; +package org.apache.arrow.memory.patch; + +import org.apache.arrow.memory.BufferAllocator; import io.netty.buffer.AbstractByteBufAllocator; import io.netty.buffer.ByteBuf; diff --git a/java/memory/memory-netty/src/test/java/io/netty/buffer/TestUnsafeDirectLittleEndian.java b/java/memory/memory-netty-buffer-patch/src/test/java/io/netty/buffer/TestUnsafeDirectLittleEndian.java similarity index 95% rename from java/memory/memory-netty/src/test/java/io/netty/buffer/TestUnsafeDirectLittleEndian.java rename to java/memory/memory-netty-buffer-patch/src/test/java/io/netty/buffer/TestUnsafeDirectLittleEndian.java index c2bd95bb3d9..043c2c1605a 100644 --- a/java/memory/memory-netty/src/test/java/io/netty/buffer/TestUnsafeDirectLittleEndian.java +++ b/java/memory/memory-netty-buffer-patch/src/test/java/io/netty/buffer/TestUnsafeDirectLittleEndian.java @@ -28,6 +28,11 @@ import org.junit.Test; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.LargeBuffer; +import io.netty.buffer.Unpooled; +import io.netty.buffer.UnsafeDirectLittleEndian; + public class TestUnsafeDirectLittleEndian { private static final boolean LITTLE_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN; diff --git a/java/memory/memory-netty/pom.xml b/java/memory/memory-netty/pom.xml index 307f6ad81a0..06b5e97fc2e 100644 --- a/java/memory/memory-netty/pom.xml +++ b/java/memory/memory-netty/pom.xml @@ -26,9 +26,15 @@ org.apache.arrow arrow-memory-core + + org.apache.arrow + arrow-memory-netty-buffer-patch + ${project.version} + io.netty netty-buffer + provided io.netty @@ -37,6 +43,7 @@ org.slf4j slf4j-api + test ch.qos.logback @@ -74,4 +81,20 @@ + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + 9 + + --patch-module=io.netty.buffer=${project.basedir}/../memory-netty-buffer-patch/target/arrow-memory-netty-buffer-patch-${project.version}.jar + + + + + diff --git a/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java b/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/netty/DefaultAllocationManagerFactory.java similarity index 87% rename from java/memory/memory-netty/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java rename to java/memory/memory-netty/src/main/java/org/apache/arrow/memory/netty/DefaultAllocationManagerFactory.java index 10cfb5c1648..8ece77178f0 100644 --- a/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java +++ b/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/netty/DefaultAllocationManagerFactory.java @@ -15,7 +15,11 @@ * limitations under the License. */ -package org.apache.arrow.memory; +package org.apache.arrow.memory.netty; + +import org.apache.arrow.memory.AllocationManager; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; /** * The default Allocation Manager Factory for a module. diff --git a/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java b/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/netty/NettyAllocationManager.java similarity index 94% rename from java/memory/memory-netty/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java rename to java/memory/memory-netty/src/main/java/org/apache/arrow/memory/netty/NettyAllocationManager.java index 20004778307..58354d0c2ee 100644 --- a/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java +++ b/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/netty/NettyAllocationManager.java @@ -15,7 +15,12 @@ * limitations under the License. */ -package org.apache.arrow.memory; +package org.apache.arrow.memory.netty; + +import org.apache.arrow.memory.AllocationManager; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.ReferenceManager; import io.netty.buffer.PooledByteBufAllocatorL; import io.netty.buffer.UnsafeDirectLittleEndian; diff --git a/java/memory/memory-netty/src/test/java/io/netty/buffer/TestExpandableByteBuf.java b/java/memory/memory-netty/src/test/java/io/netty/buffer/TestExpandableByteBuf.java index b39cca8e8e7..67557b65a62 100644 --- a/java/memory/memory-netty/src/test/java/io/netty/buffer/TestExpandableByteBuf.java +++ b/java/memory/memory-netty/src/test/java/io/netty/buffer/TestExpandableByteBuf.java @@ -23,6 +23,10 @@ import org.junit.Assert; import org.junit.Test; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ExpandableByteBuf; +import io.netty.buffer.NettyArrowBuf; + public class TestExpandableByteBuf { @Test diff --git a/java/memory/memory-netty/src/test/java/io/netty/buffer/TestNettyArrowBuf.java b/java/memory/memory-netty/src/test/java/io/netty/buffer/TestNettyArrowBuf.java index 45d3b41e8a6..f18bccb4c9d 100644 --- a/java/memory/memory-netty/src/test/java/io/netty/buffer/TestNettyArrowBuf.java +++ b/java/memory/memory-netty/src/test/java/io/netty/buffer/TestNettyArrowBuf.java @@ -20,12 +20,15 @@ import java.nio.ByteBuffer; import org.apache.arrow.memory.ArrowBuf; -import org.apache.arrow.memory.ArrowByteBufAllocator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.memory.patch.ArrowByteBufAllocator; import org.junit.Assert; import org.junit.Test; +import io.netty.buffer.CompositeByteBuf; +import io.netty.buffer.NettyArrowBuf; + public class TestNettyArrowBuf { @Test diff --git a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/ITTestLargeArrowBuf.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/ITTestLargeArrowBuf.java similarity index 93% rename from java/memory/memory-netty/src/test/java/org/apache/arrow/memory/ITTestLargeArrowBuf.java rename to java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/ITTestLargeArrowBuf.java index fa8d510e361..71dba73d289 100644 --- a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/ITTestLargeArrowBuf.java +++ b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/ITTestLargeArrowBuf.java @@ -15,10 +15,13 @@ * limitations under the License. */ -package org.apache.arrow.memory; +package org.apache.arrow.memory.netty; import static org.junit.Assert.assertEquals; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestAllocationManagerNetty.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestAllocationManagerNetty.java similarity index 90% rename from java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestAllocationManagerNetty.java rename to java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestAllocationManagerNetty.java index 2dbd56480b8..7f1e34ddc5f 100644 --- a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestAllocationManagerNetty.java +++ b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestAllocationManagerNetty.java @@ -15,10 +15,12 @@ * limitations under the License. */ -package org.apache.arrow.memory; +package org.apache.arrow.memory.netty; import static org.junit.Assert.assertEquals; +import org.apache.arrow.memory.AllocationManager; +import org.apache.arrow.memory.DefaultAllocationManagerOption; import org.junit.Test; /** diff --git a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEmptyArrowBuf.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestEmptyArrowBuf.java similarity index 90% rename from java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEmptyArrowBuf.java rename to java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestEmptyArrowBuf.java index 3fd7ce74aab..b9948083e6f 100644 --- a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEmptyArrowBuf.java +++ b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestEmptyArrowBuf.java @@ -15,10 +15,13 @@ * limitations under the License. */ -package org.apache.arrow.memory; +package org.apache.arrow.memory.netty; import static org.junit.Assert.assertEquals; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.ReferenceManager; +import org.apache.arrow.memory.RootAllocator; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -47,8 +50,8 @@ public static void afterClass() { public void testZeroBuf() { // Exercise the historical log inside the empty ArrowBuf. This is initialized statically, and there is a circular // dependency between ArrowBuf and BaseAllocator, so if the initialization happens in the wrong order, the - // historical log will be null even though BaseAllocator.DEBUG is true. - allocator.getEmpty().print(new StringBuilder(), 0, BaseAllocator.Verbosity.LOG_WITH_STACKTRACE); + // historical log will be null even though RootAllocator.DEBUG is true. + allocator.getEmpty().print(new StringBuilder(), 0, RootAllocator.Verbosity.LOG_WITH_STACKTRACE); } @Test diff --git a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEndianness.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestEndianness.java similarity index 92% rename from java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEndianness.java rename to java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestEndianness.java index dcaeb24889e..a782523cbc6 100644 --- a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEndianness.java +++ b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestEndianness.java @@ -15,12 +15,14 @@ * limitations under the License. */ -package org.apache.arrow.memory; +package org.apache.arrow.memory.netty; import static org.junit.Assert.assertEquals; import java.nio.ByteOrder; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; import org.junit.Test; import io.netty.buffer.ByteBuf; diff --git a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestNettyAllocationManager.java similarity index 87% rename from java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java rename to java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestNettyAllocationManager.java index 1b64cd73363..39692c96ceb 100644 --- a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java +++ b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestNettyAllocationManager.java @@ -15,13 +15,18 @@ * limitations under the License. */ -package org.apache.arrow.memory; +package org.apache.arrow.memory.netty; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import org.apache.arrow.memory.AllocationManager; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.BufferLedger; +import org.apache.arrow.memory.RootAllocator; import org.junit.Test; /** @@ -31,8 +36,8 @@ public class TestNettyAllocationManager { static int CUSTOMIZED_ALLOCATION_CUTOFF_VALUE = 1024; - private BaseAllocator createCustomizedAllocator() { - return new RootAllocator(BaseAllocator.configBuilder() + private RootAllocator createCustomizedAllocator() { + return new RootAllocator(RootAllocator.configBuilder() .allocationManagerFactory(new AllocationManager.Factory() { @Override public AllocationManager create(BufferAllocator accountingAllocator, long size) { @@ -65,7 +70,7 @@ private void readWriteArrowBuf(ArrowBuf buffer) { @Test public void testSmallBufferAllocation() { final long bufSize = CUSTOMIZED_ALLOCATION_CUTOFF_VALUE - 512L; - try (BaseAllocator allocator = createCustomizedAllocator(); + try (RootAllocator allocator = createCustomizedAllocator(); ArrowBuf buffer = allocator.buffer(bufSize)) { assertTrue(buffer.getReferenceManager() instanceof BufferLedger); @@ -89,7 +94,7 @@ public void testSmallBufferAllocation() { @Test public void testLargeBufferAllocation() { final long bufSize = CUSTOMIZED_ALLOCATION_CUTOFF_VALUE + 1024L; - try (BaseAllocator allocator = createCustomizedAllocator(); + try (RootAllocator allocator = createCustomizedAllocator(); ArrowBuf buffer = allocator.buffer(bufSize)) { assertTrue(buffer.getReferenceManager() instanceof BufferLedger); BufferLedger bufferLedger = (BufferLedger) buffer.getReferenceManager(); diff --git a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestNettyAllocator.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestNettyAllocator.java new file mode 100644 index 00000000000..b9525d0fe9d --- /dev/null +++ b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestNettyAllocator.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory.netty; + +import static org.junit.Assert.assertTrue; + +import java.util.stream.Collectors; + +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.ReferenceManager; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import io.netty.buffer.PooledByteBufAllocatorL; + +/** + * Test netty allocators. + */ +public class TestNettyAllocator { + + @Test + public void testMemoryUsage() { + ListAppender memoryLogsAppender = new ListAppender<>(); + Logger logger = (Logger) LoggerFactory.getLogger("arrow.allocator"); + try { + logger.setLevel(Level.TRACE); + logger.addAppender(memoryLogsAppender); + memoryLogsAppender.start(); + try (ArrowBuf buf = new ArrowBuf(ReferenceManager.NO_OP, null, + 1024, new PooledByteBufAllocatorL().empty.memoryAddress())) { + buf.memoryAddress(); + } + boolean result = false; + long startTime = System.currentTimeMillis(); + while ((System.currentTimeMillis() - startTime) < 10000) { // 10 seconds maximum for time to read logs + result = memoryLogsAppender.list.stream() + .anyMatch( + log -> log.toString().contains("Memory Usage: \n") && + log.toString().contains("Large buffers outstanding: ") && + log.toString().contains("Normal buffers outstanding: ") && + log.getLevel().equals(Level.TRACE) + ); + if (result) { + break; + } + } + assertTrue("Log messages are:\n" + + memoryLogsAppender.list.stream().map(ILoggingEvent::toString).collect(Collectors.joining("\n")), + result); + } finally { + memoryLogsAppender.stop(); + logger.detachAppender(memoryLogsAppender); + logger.setLevel(null); + } + } +} diff --git a/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java b/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/unsafe/DefaultAllocationManagerFactory.java similarity index 87% rename from java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java rename to java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/unsafe/DefaultAllocationManagerFactory.java index 720c3d02d23..dfb6c706856 100644 --- a/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java +++ b/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/unsafe/DefaultAllocationManagerFactory.java @@ -15,7 +15,11 @@ * limitations under the License. */ -package org.apache.arrow.memory; +package org.apache.arrow.memory.unsafe; + +import org.apache.arrow.memory.AllocationManager; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; /** * The default Allocation Manager Factory for a module. diff --git a/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java b/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/unsafe/UnsafeAllocationManager.java similarity index 89% rename from java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java rename to java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/unsafe/UnsafeAllocationManager.java index b10aba3598d..3468a6ec65c 100644 --- a/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java +++ b/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/unsafe/UnsafeAllocationManager.java @@ -15,8 +15,12 @@ * limitations under the License. */ -package org.apache.arrow.memory; +package org.apache.arrow.memory.unsafe; +import org.apache.arrow.memory.AllocationManager; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.ReferenceManager; import org.apache.arrow.memory.util.MemoryUtil; /** diff --git a/java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/TestAllocationManagerUnsafe.java b/java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/unsafe/TestAllocationManagerUnsafe.java similarity index 90% rename from java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/TestAllocationManagerUnsafe.java rename to java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/unsafe/TestAllocationManagerUnsafe.java index 33abe92e50f..f1ca96eea0f 100644 --- a/java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/TestAllocationManagerUnsafe.java +++ b/java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/unsafe/TestAllocationManagerUnsafe.java @@ -15,10 +15,12 @@ * limitations under the License. */ -package org.apache.arrow.memory; +package org.apache.arrow.memory.unsafe; import static org.junit.Assert.assertEquals; +import org.apache.arrow.memory.AllocationManager; +import org.apache.arrow.memory.DefaultAllocationManagerOption; import org.junit.Test; /** diff --git a/java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/TestUnsafeAllocationManager.java b/java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/unsafe/TestUnsafeAllocationManager.java similarity index 82% rename from java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/TestUnsafeAllocationManager.java rename to java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/unsafe/TestUnsafeAllocationManager.java index c15882a37a6..5d74c398e26 100644 --- a/java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/TestUnsafeAllocationManager.java +++ b/java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/unsafe/TestUnsafeAllocationManager.java @@ -15,11 +15,16 @@ * limitations under the License. */ -package org.apache.arrow.memory; +package org.apache.arrow.memory.unsafe; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import org.apache.arrow.memory.AllocationManager; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferLedger; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.memory.unsafe.UnsafeAllocationManager; import org.junit.Test; /** @@ -27,8 +32,8 @@ */ public class TestUnsafeAllocationManager { - private BaseAllocator createUnsafeAllocator() { - return new RootAllocator(BaseAllocator.configBuilder().allocationManagerFactory(UnsafeAllocationManager.FACTORY) + private RootAllocator createUnsafeAllocator() { + return new RootAllocator(RootAllocator.configBuilder().allocationManagerFactory(UnsafeAllocationManager.FACTORY) .build()); } @@ -51,7 +56,7 @@ private void readWriteArrowBuf(ArrowBuf buffer) { @Test public void testBufferAllocation() { final long bufSize = 4096L; - try (BaseAllocator allocator = createUnsafeAllocator(); + try (RootAllocator allocator = createUnsafeAllocator(); ArrowBuf buffer = allocator.buffer(bufSize)) { assertTrue(buffer.getReferenceManager() instanceof BufferLedger); BufferLedger bufferLedger = (BufferLedger) buffer.getReferenceManager(); diff --git a/java/memory/pom.xml b/java/memory/pom.xml index c10263b97f1..55fbb90353f 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -23,6 +23,7 @@ memory-core memory-unsafe + memory-netty-buffer-patch memory-netty From a8bf81349ec2ceb765a58a03bf08064a226970ba Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 24 Nov 2023 08:44:53 -0800 Subject: [PATCH 09/35] Downgrade maven-dependency-plugin to 3.1.2 There appear to be false positives around Jackson usage in arrow-tools with versions greater than 3.1.2 up to at least 3.6.0. --- java/pom.xml | 7 ++++++- java/tools/pom.xml | 2 -- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 2b452f6d889..98f53fb7936 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -378,7 +378,12 @@ org.apache.maven.plugins maven-dependency-plugin - 3.6.1 + + 3.1.2 org.apache.rat diff --git a/java/tools/pom.xml b/java/tools/pom.xml index 2ebabe74e6b..4b43c513efd 100644 --- a/java/tools/pom.xml +++ b/java/tools/pom.xml @@ -53,12 +53,10 @@ com.fasterxml.jackson.core jackson-core - provided com.fasterxml.jackson.core jackson-databind - provided org.slf4j From da16115088608b860e7305a41a5d401dff16e929 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 24 Nov 2023 09:10:43 -0800 Subject: [PATCH 10/35] Downgrade maven-shade-plugin to 3.2.4 for arrow-vector only There are issues looking up FlatBuffers in the arrow-tools tests if arrow-vector gets shaded with a newer version up to at least 3.5.1. --- java/vector/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/java/vector/pom.xml b/java/vector/pom.xml index 9cedcfdaa44..17d8f312a52 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -174,6 +174,13 @@ org.apache.maven.plugins maven-shade-plugin + + 3.2.4 package From 0a21d54d6508929132cc7b290c9e10c4a72e63eb Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 24 Nov 2023 09:17:21 -0800 Subject: [PATCH 11/35] Patch Netty when the profile is JDK9 only Run the Netty module patch step only for profiles where JDK >=9 --- java/memory/memory-netty/pom.xml | 37 ++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/java/memory/memory-netty/pom.xml b/java/memory/memory-netty/pom.xml index 06b5e97fc2e..247fce7bb52 100644 --- a/java/memory/memory-netty/pom.xml +++ b/java/memory/memory-netty/pom.xml @@ -58,6 +58,27 @@ + + patch-netty-modules + + [9,] + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + 9 + + --patch-module=io.netty.buffer=${project.basedir}/../memory-netty-buffer-patch/target/arrow-memory-netty-buffer-patch-${project.version}.jar + + + + + + - 9 - - --patch-module=io.netty.buffer=${project.basedir}/../memory-netty-buffer-patch/target/arrow-memory-netty-buffer-patch-${project.version}.jar - - - - - From 02a0ee28cf92cda7051aded7dcb7a5a1b44d4e97 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 24 Nov 2023 10:05:18 -0800 Subject: [PATCH 12/35] Ignore TestBaseAllocator#testRootAllocator_getEmpty This test fails because we no longer use Netty's allocation manager factory since the test has moved into memory-core. To be fixed. --- .../src/test/java/org/apache/arrow/memory/TestBaseAllocator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java index 365c84f5a2b..f90e2214215 100644 --- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java @@ -112,6 +112,7 @@ public void testRootAllocator_closeWithOutstanding() throws Exception { } @Test + @Ignore public void testRootAllocator_getEmpty() throws Exception { try (final RootAllocator rootAllocator = new RootAllocator(MAX_ALLOCATION)) { From b0e9e7b5bc56c36ffbedd3efc00cd377e9605b17 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 24 Nov 2023 11:08:21 -0800 Subject: [PATCH 13/35] Downgrade the maven-shade-plugin to 3.2.4 in flight-core There are issues looking up gRPC in the flight-integration0tests tests if flight-core gets shaded with a newer version up to at least 3.5.1. --- java/flight/flight-core/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index 30facf734b5..8f41d2b65b7 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -145,6 +145,13 @@ org.apache.maven.plugins maven-shade-plugin + + 3.2.4 shade-main From 4301efd7becd3a24200469332366f2391fb0b2f0 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 24 Nov 2023 15:22:34 -0800 Subject: [PATCH 14/35] Add and integrate the module-info compiler plugin Note that some configuration from the arrow root POM is copied to the maven plugins module because it cannot be inherited without creating a cyclic depdendency. --- .../module-info-compiler-maven-plugin/pom.xml | 131 ++++++++ .../plugins/ModuleInfoCompilerPlugin.java | 93 ++++++ java/maven/pom.xml | 308 ++++++++++++++++++ java/pom.xml | 18 +- 4 files changed, 549 insertions(+), 1 deletion(-) create mode 100644 java/maven/module-info-compiler-maven-plugin/pom.xml create mode 100644 java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java create mode 100644 java/maven/pom.xml diff --git a/java/maven/module-info-compiler-maven-plugin/pom.xml b/java/maven/module-info-compiler-maven-plugin/pom.xml new file mode 100644 index 00000000000..be2a9f812d3 --- /dev/null +++ b/java/maven/module-info-compiler-maven-plugin/pom.xml @@ -0,0 +1,131 @@ + + + + 4.0.0 + + org.apache.arrow.maven.plugins + arrow-maven-plugins + 15.0.0-SNAPSHOT + + module-info-compiler-maven-plugin + 15.0.0-SNAPSHOT + maven-plugin + + Module Info Compiler Maven Plugin + + https://arrow.apache.org + + + ${maven.version} + + + + 3.3.9 + + + + + org.glavo + module-info-compiler + 2.0 + + + org.apache.maven + maven-plugin-api + ${maven.version} + provided + + + org.apache.maven + maven-core + ${maven.version} + provided + + + org.apache.maven + maven-artifact + ${maven.version} + provided + + + org.apache.maven + maven-model + ${maven.version} + provided + + + org.apache.maven.plugin-tools + maven-plugin-annotations + 3.6.0 + provided + + + + + + + + maven-clean-plugin + 3.1.0 + + + maven-plugin-plugin + 3.6.0 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + maven-invoker-plugin + 3.1.0 + + + + + + org.apache.maven.plugins + maven-plugin-plugin + 3.6.0 + + true + + + + mojo-descriptor + + descriptor + + + + help-goal + + helpmojo + + + + + + + + + \ No newline at end of file diff --git a/java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java b/java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java new file mode 100644 index 00000000000..8f1baf8f94b --- /dev/null +++ b/java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.maven.plugins; + +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.project.MavenProject; +import org.glavo.mic.ModuleInfoCompiler; + +/** + * Compiles the first module-info.java file in the project purely syntactically. + */ +@Mojo(name = "module-info-compile", defaultPhase = LifecyclePhase.COMPILE) +public class ModuleInfoCompilerPlugin extends AbstractMojo { + /** + * Source directories. + */ + @Parameter(defaultValue = "${project.compileSourceRoots}", property = "compileSourceRoots", + required = true) + private final List compileSourceRoots = new ArrayList<>(); + + @Parameter(defaultValue = "${project}", readonly = true, required = true) + private MavenProject project; + + @Override + public void execute() throws MojoExecutionException { + Optional moduleInfoFile = findFirstModuleInfo(compileSourceRoots); + if (moduleInfoFile.isPresent()) { + // The compiled module-info.class file goes into target/classes/module-info/main + Path outputDir = Path.of(project.getBuild().getOutputDirectory()); + + outputDir.toFile().mkdirs(); + Path targetPath = outputDir.resolve("module-info.class"); + + // Invoke the compiler, + ModuleInfoCompiler compiler = new ModuleInfoCompiler(); + try (Reader reader = new InputStreamReader(Files.newInputStream(moduleInfoFile.get().toPath()), + StandardCharsets.UTF_8); + OutputStream output = Files.newOutputStream(targetPath)) { + compiler.compile(reader, output); + } catch (IOException ex) { + throw new MojoExecutionException("Error compiling module-info.java", ex); + } + } + } + + /** + * Finds the first module-info.java file in the set of source directories. + */ + private Optional findFirstModuleInfo(List sourceDirectories) { + if (sourceDirectories == null) { + return Optional.empty(); + } + + return sourceDirectories.stream().map(Path::of) + .map(sourcePath -> + sourcePath.toFile().listFiles(file -> + file.getName().equals("module-info.java"))) + .filter(matchingFiles -> matchingFiles != null && matchingFiles.length != 0) + .map(matchingFiles -> matchingFiles[0]) + .findAny(); + } +} diff --git a/java/maven/pom.xml b/java/maven/pom.xml new file mode 100644 index 00000000000..f8c574d203c --- /dev/null +++ b/java/maven/pom.xml @@ -0,0 +1,308 @@ + + + + 4.0.0 + + org.apache.arrow.maven.plugins + arrow-maven-plugins + 15.0.0-SNAPSHOT + Arrow Maven Plugins + pom + + + module-info-compiler-maven-plugin + + + + true + + + + + + org.apache.rat + apache-rat-plugin + + + rat-checks + validate + + check + + + + + false + + **/dependency-reduced-pom.xml + **/*.log + **/*.css + **/*.js + **/*.md + **/*.eps + **/*.json + **/*.seq + **/*.parquet + **/*.sql + **/arrow-git.properties + **/*.csv + **/*.csvh + **/*.csvh-test + **/*.tsv + **/*.txt + **/*.ssv + **/arrow-*.conf + **/.buildpath + **/*.proto + **/*.fmpp + **/target/** + **/*.tdd + **/*.project + **/TAGS + **/*.checkstyle + **/.classpath + **/.factorypath + **/.settings/** + .*/** + **/*.patch + **/*.pb.cc + **/*.pb.h + **/*.linux + **/client/build/** + **/*.tbl + **/*.iml + **/flight.properties + **/*.idea/** + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + **/logging.properties + **/logback-test.xml + **/logback.out.xml + **/logback.xml + + + true + + true + true + + + org.apache.arrow + ${username} + https://arrow.apache.org/ + + + + + + + test-jar + + + true + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + UTF-8 + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + 2048m + false + true + + + + maven-enforcer-plugin + + + validate_java_and_maven_version + verify + + enforce + + false + + + + [3.3.0,4) + + + + + + avoid_bad_dependencies + verify + + enforce + + + + + + commons-logging + javax.servlet:servlet-api + org.mortbay.jetty:servlet-api + org.mortbay.jetty:servlet-api-2.5 + log4j:log4j + + + + + + + + + pl.project13.maven + git-commit-id-plugin + 4.0.5 + + + for-jars + true + + revision + + + target/classes/arrow-git.properties + + + + for-source-tarball + + revision + + false + + ./arrow-git.properties + + + + + + dd.MM.yyyy '@' HH:mm:ss z + false + false + true + false + + false + false + 7 + -dirty + true + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.1.0 + + + com.puppycrawl.tools + checkstyle + 8.19 + + + org.slf4j + jcl-over-slf4j + 1.7.5 + + + + + validate + validate + + check + + + + + ../dev/checkstyle/checkstyle.xml + ../dev/checkstyle/checkstyle.license + ../dev/checkstyle/suppressions.xml + true + UTF-8 + true + ${checkstyle.failOnViolation} + ${checkstyle.failOnViolation} + warning + xml + html + ${project.build.directory}/test/checkstyle-errors.xml + false + + + + org.apache.maven.plugins + maven-dependency-plugin + + + analyze + verify + + analyze-only + + + true + true + + + javax.annotation:javax.annotation-api:* + org.apache.hadoop:hadoop-client-api + + + + + + + org.cyclonedx + cyclonedx-maven-plugin + 2.7.10 + + + package + + makeBom + + + + + + + + diff --git a/java/pom.xml b/java/pom.xml index 98f53fb7936..0e85f89fc47 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -371,6 +371,17 @@ + + org.apache.arrow.maven.plugins + module-info-compiler-maven-plugin + + + + module-info-compile + + + + @@ -546,6 +557,11 @@ + + org.apache.arrow.maven.plugins + module-info-compiler-maven-plugin + ${project.version} + @@ -735,6 +751,7 @@ + maven bom format memory @@ -1236,7 +1253,6 @@ - From cd8b19e46427563fc3c3681774c25cb48eecc1a1 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 24 Nov 2023 15:49:10 -0800 Subject: [PATCH 15/35] Fix JDK8 compile issue with module-info-compiler and attempt to fix analyze issue --- java/maven/module-info-compiler-maven-plugin/pom.xml | 1 + .../apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/java/maven/module-info-compiler-maven-plugin/pom.xml b/java/maven/module-info-compiler-maven-plugin/pom.xml index be2a9f812d3..ac2b41621db 100644 --- a/java/maven/module-info-compiler-maven-plugin/pom.xml +++ b/java/maven/module-info-compiler-maven-plugin/pom.xml @@ -39,6 +39,7 @@ org.glavo module-info-compiler 2.0 + jar org.apache.maven diff --git a/java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java b/java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java index 8f1baf8f94b..93e99e9f164 100644 --- a/java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java +++ b/java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java @@ -25,6 +25,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -57,7 +58,7 @@ public void execute() throws MojoExecutionException { Optional moduleInfoFile = findFirstModuleInfo(compileSourceRoots); if (moduleInfoFile.isPresent()) { // The compiled module-info.class file goes into target/classes/module-info/main - Path outputDir = Path.of(project.getBuild().getOutputDirectory()); + Path outputDir = Paths.get(project.getBuild().getOutputDirectory()); outputDir.toFile().mkdirs(); Path targetPath = outputDir.resolve("module-info.class"); From 2ffc604a1e1a47a42bd3cf08575525e1aad82507 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 24 Nov 2023 15:52:25 -0800 Subject: [PATCH 16/35] Fix JDK8 issue with use of Path.of --- .../apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java b/java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java index 93e99e9f164..e0475c27ee0 100644 --- a/java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java +++ b/java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java @@ -83,7 +83,7 @@ private Optional findFirstModuleInfo(List sourceDirectories) { return Optional.empty(); } - return sourceDirectories.stream().map(Path::of) + return sourceDirectories.stream().map(Paths::get) .map(sourcePath -> sourcePath.toFile().listFiles(file -> file.getName().equals("module-info.java"))) From ba26997763b81be1962dfc640bc939109a953c79 Mon Sep 17 00:00:00 2001 From: James Duong Date: Fri, 24 Nov 2023 16:02:30 -0800 Subject: [PATCH 17/35] Skip the maven-dependency-plugin analyze check due to false positive Reports a used dependency as unused only on Windows 2022 CI runs --- .../module-info-compiler-maven-plugin/pom.xml | 23 ++++++++++++++++++- java/maven/pom.xml | 1 + 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/java/maven/module-info-compiler-maven-plugin/pom.xml b/java/maven/module-info-compiler-maven-plugin/pom.xml index ac2b41621db..b15d89eb5b3 100644 --- a/java/maven/module-info-compiler-maven-plugin/pom.xml +++ b/java/maven/module-info-compiler-maven-plugin/pom.xml @@ -39,7 +39,6 @@ org.glavo module-info-compiler 2.0 - jar org.apache.maven @@ -103,6 +102,28 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + analyze + verify + + analyze-only + + + + true + + + + org.apache.maven.plugins maven-plugin-plugin diff --git a/java/maven/pom.xml b/java/maven/pom.xml index f8c574d203c..3119fc34187 100644 --- a/java/maven/pom.xml +++ b/java/maven/pom.xml @@ -136,6 +136,7 @@ org.apache.maven.plugins maven-compiler-plugin + UTF-8 1.8 1.8 2048m From 5bc250133db2cc2412f73ccba316064c561c8fec Mon Sep 17 00:00:00 2001 From: James Duong Date: Sun, 26 Nov 2023 06:57:35 -0800 Subject: [PATCH 18/35] Fix non-inherited version --- java/maven/module-info-compiler-maven-plugin/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/java/maven/module-info-compiler-maven-plugin/pom.xml b/java/maven/module-info-compiler-maven-plugin/pom.xml index b15d89eb5b3..6cd72edb095 100644 --- a/java/maven/module-info-compiler-maven-plugin/pom.xml +++ b/java/maven/module-info-compiler-maven-plugin/pom.xml @@ -19,7 +19,6 @@ 15.0.0-SNAPSHOT module-info-compiler-maven-plugin - 15.0.0-SNAPSHOT maven-plugin Module Info Compiler Maven Plugin From 681362920a726e7cba32b7dd96a1c25b453c29a1 Mon Sep 17 00:00:00 2001 From: James Duong Date: Sun, 26 Nov 2023 07:00:00 -0800 Subject: [PATCH 19/35] Do not use dependency analyze when compiling plugins It is not correctly getting skipped. --- .../module-info-compiler-maven-plugin/pom.xml | 22 ------------------- java/maven/pom.xml | 22 ------------------- 2 files changed, 44 deletions(-) diff --git a/java/maven/module-info-compiler-maven-plugin/pom.xml b/java/maven/module-info-compiler-maven-plugin/pom.xml index 6cd72edb095..ce598fc7f0f 100644 --- a/java/maven/module-info-compiler-maven-plugin/pom.xml +++ b/java/maven/module-info-compiler-maven-plugin/pom.xml @@ -101,28 +101,6 @@ - - org.apache.maven.plugins - maven-dependency-plugin - - - analyze - verify - - analyze-only - - - - true - - - - org.apache.maven.plugins maven-plugin-plugin diff --git a/java/maven/pom.xml b/java/maven/pom.xml index 3119fc34187..86ac402732b 100644 --- a/java/maven/pom.xml +++ b/java/maven/pom.xml @@ -268,28 +268,6 @@ false - - org.apache.maven.plugins - maven-dependency-plugin - - - analyze - verify - - analyze-only - - - true - true - - - javax.annotation:javax.annotation-api:* - org.apache.hadoop:hadoop-client-api - - - - - org.cyclonedx cyclonedx-maven-plugin From 0598506e3f7242f955bf31b0e67cbbdcb2ae4c79 Mon Sep 17 00:00:00 2001 From: James Duong Date: Mon, 27 Nov 2023 10:38:19 -0800 Subject: [PATCH 20/35] Exclude module-info.java files from checkstyle Newer versions of the checkstyle plugin do support module-info.java files but require rewriting the checkstyle rules file. --- java/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/java/pom.xml b/java/pom.xml index 0e85f89fc47..09118f6d3d8 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -321,6 +321,7 @@ + **/module-info.java dev/checkstyle/checkstyle.xml dev/checkstyle/checkstyle.license dev/checkstyle/suppressions.xml From e2b33afb6c4fa1aa11e2f4fb9eece7b3d05ed1e6 Mon Sep 17 00:00:00 2001 From: James Duong Date: Mon, 27 Nov 2023 12:36:44 -0800 Subject: [PATCH 21/35] Add skip parameter and logging to module info compiler plugin --- .../arrow/maven/plugins/ModuleInfoCompilerPlugin.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java b/java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java index e0475c27ee0..0f91d0590bb 100644 --- a/java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java +++ b/java/maven/module-info-compiler-maven-plugin/src/main/java/org/apache/arrow/maven/plugins/ModuleInfoCompilerPlugin.java @@ -50,11 +50,19 @@ public class ModuleInfoCompilerPlugin extends AbstractMojo { required = true) private final List compileSourceRoots = new ArrayList<>(); + @Parameter(defaultValue = "false", property = "skip", required = false) + private boolean skip = false; + @Parameter(defaultValue = "${project}", readonly = true, required = true) private MavenProject project; @Override public void execute() throws MojoExecutionException { + if (skip) { + getLog().info("Skipping module-info-compiler-maven-plugin"); + return; + } + Optional moduleInfoFile = findFirstModuleInfo(compileSourceRoots); if (moduleInfoFile.isPresent()) { // The compiled module-info.class file goes into target/classes/module-info/main @@ -69,9 +77,12 @@ public void execute() throws MojoExecutionException { StandardCharsets.UTF_8); OutputStream output = Files.newOutputStream(targetPath)) { compiler.compile(reader, output); + getLog().info("Successfully wrote module-info.class file."); } catch (IOException ex) { throw new MojoExecutionException("Error compiling module-info.java", ex); } + } else { + getLog().info("No module-info.java file found. module-info.class file was not generated."); } } From b98bb385f68beb218f82f07e49c6cbffc9bef812 Mon Sep 17 00:00:00 2001 From: James Duong Date: Mon, 27 Nov 2023 12:37:40 -0800 Subject: [PATCH 22/35] Update maven-compiler-plugin to 3.11.0 for useModulePath flag Change the project to not use the module path for most compilation since most compilation will target JDK8. --- java/pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 09118f6d3d8..423e9b56737 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -45,7 +45,7 @@ true 9+181-r4173-1 2.22.0 - 3.10.1 + 3.11.0 5.5.0 5.2.0 @@ -413,6 +413,7 @@ ${maven-compiler-plugin.version} **/module-info.java + false org.immutables From ce34a33567d17420b0aec6782a5f7c1b16b74dfb Mon Sep 17 00:00:00 2001 From: James Duong Date: Tue, 28 Nov 2023 00:11:52 -0800 Subject: [PATCH 23/35] Remove Netty patching Not sure if this is necessary if we just use the plugin to compile module-info.java --- java/memory/memory-netty/pom.xml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/java/memory/memory-netty/pom.xml b/java/memory/memory-netty/pom.xml index 247fce7bb52..4ee4883d78d 100644 --- a/java/memory/memory-netty/pom.xml +++ b/java/memory/memory-netty/pom.xml @@ -58,27 +58,6 @@ - - patch-netty-modules - - [9,] - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - 9 - - --patch-module=io.netty.buffer=${project.basedir}/../memory-netty-buffer-patch/target/arrow-memory-netty-buffer-patch-${project.version}.jar - - - - - -