GH-38998: [Java] Build memory-core and memory-unsafe as JPMS modules

dev/archery/archery/integration/tester_java.py

@@ -40,7 +40,7 @@ def load_version_from_pom():
 _JAVA_OPTS = [
     "-Dio.netty.tryReflectionSetAccessible=true",
     "-Darrow.struct.conflict.policy=CONFLICT_APPEND",
-    "--add-opens=java.base/java.nio=ALL-UNNAMED",
+    "--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED",
     # GH-39113: avoid failures accessing files in `/tmp/hsperfdata_...`
     "-XX:-UsePerfData",
 ]
@@ -247,7 +247,8 @@ def __init__(self, *args, **kwargs):
         if 'Unrecognized option: --add-opens' not in proc.stderr:
             # Java 9+
             self._java_opts.append(
-                '--add-opens=java.base/java.nio=ALL-UNNAMED')
+                '--add-opens=java.base/java.nio='
+                'org.apache.arrow.memory.core,ALL-UNNAMED')
 
     def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
         cmd = (

docs/source/java/install.rst

@@ -33,14 +33,14 @@ Java modules are compatible with JDK 8 and above.
 Currently, JDK 8, 11, 17, and 21 are tested in CI.
 
 When using Java 9 or later, some JDK internals must be exposed by
-adding ``--add-opens=java.base/java.nio=ALL-UNNAMED`` to the ``java`` command:
+adding ``--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED`` to the ``java`` command:
 
 .. code-block:: shell
 
    # Directly on the command line
-   $ java --add-opens=java.base/java.nio=ALL-UNNAMED -jar ...
+   $ java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED -jar ...
    # Indirectly via environment variables
-   $ env _JAVA_OPTIONS="--add-opens=java.base/java.nio=ALL-UNNAMED" java -jar ...
+   $ env _JAVA_OPTIONS="--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED" java -jar ...
 
 Otherwise, you may see errors like ``module java.base does not "opens
 java.nio" to unnamed module``.

java/c/pom.xml

@@ -53,6 +53,11 @@
             <artifactId>arrow-memory-unsafe</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-format</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>

java/gandiva/pom.xml

@@ -79,7 +79,7 @@
                     <plugin>
                         <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-javadoc-plugin</artifactId>
-                        <version>2.9.1</version>
+                        <version>3.6.3</version>
                         <executions>
                             <execution>
                                 <id>attach-javadocs</id>

java/maven/module-info-compiler-maven-plugin/pom.xml

@@ -125,6 +125,4 @@
       </plugin>
     </plugins>
   </build>
-
-
-</project>
+</project>

java/memory/memory-core/pom.xml

@@ -46,7 +46,6 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-
         <configuration>
           <excludes>
             <!-- Test is only useful when NOT running with add-opens -->
@@ -58,6 +57,30 @@
   </build>
 
   <profiles>
+    <profile>
+      <id>jdk11+</id>
+      <activation>
+        <jdk>[11,]</jdk>
+        <property>
+          <name>!m2e.version</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration combine.self="override">
+              <argLine>--add-reads=org.apache.arrow.memory.core=ch.qos.logback.classic --add-opens=java.base/java.lang.reflect=org.apache.arrow.memory.core --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED</argLine>
+              <excludes>
+                <!-- Test is only useful when NOT running with add-opens -->
+                <exclude>**/TestOpens.java</exclude>
+              </excludes>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <profile>
       <id>opens-tests</id>
       <!-- Run tests WITHOUT add-opens to make sure we fail-fast -->

java/memory/memory-core/src/main/java/module-info.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module org.apache.arrow.memory.core {
+  exports org.apache.arrow.memory;
+  exports org.apache.arrow.memory.rounding;
+  exports org.apache.arrow.memory.util;
+  exports org.apache.arrow.memory.util.hash;
+  exports org.apache.arrow.util;
+  requires transitive jdk.unsupported;
+  requires jsr305;
+  requires org.immutables.value;
+  requires org.slf4j;
+}

java/memory/memory-core/src/main/java/org/apache/arrow/memory/CheckAllocator.java

@@ -31,20 +31,35 @@
  */
 final class CheckAllocator {
   private static final Logger logger = LoggerFactory.getLogger(CheckAllocator.class);
-  private static final String ALLOCATOR_PATH = "org/apache/arrow/memory/DefaultAllocationManagerFactory.class";
+  // unique package names needed by JPMS module naming
+  private static final String ALLOCATOR_PATH_CORE =
+      "org/apache/arrow/memory/DefaultAllocationManagerFactory.class";
+  private static final String ALLOCATOR_PATH_UNSAFE =
+      "org/apache/arrow/memory/unsafe/DefaultAllocationManagerFactory.class";
+  private static final String ALLOCATOR_PATH_NETTY =
+      "org/apache/arrow/memory/netty/DefaultAllocationManagerFactory.class";
 
   private CheckAllocator() {
-
   }
 
   static String check() {
     Set<URL> urls = scanClasspath();
     URL rootAllocator = assertOnlyOne(urls);
     reportResult(rootAllocator);
-    return "org.apache.arrow.memory.DefaultAllocationManagerFactory";
+    if (rootAllocator.getPath().contains("memory-core") ||
+        rootAllocator.getPath().contains("/org/apache/arrow/memory/core/")) {
+      return "org.apache.arrow.memory.DefaultAllocationManagerFactory";
+    } else if (rootAllocator.getPath().contains("memory-unsafe") ||
+        rootAllocator.getPath().contains("/org/apache/arrow/memory/unsafe/")) {
+      return "org.apache.arrow.memory.unsafe.DefaultAllocationManagerFactory";
+    } else if (rootAllocator.getPath().contains("memory-netty") ||
+        rootAllocator.getPath().contains("/org/apache/arrow/memory/netty/")) {
+      return "org.apache.arrow.memory.netty.DefaultAllocationManagerFactory";
+    } else {
+      throw new IllegalStateException("Unknown allocation manager type to infer. Current: " + rootAllocator.getPath());
+    }
   }
 
-
   private static Set<URL> scanClasspath() {
     // LinkedHashSet appropriate here because it preserves insertion order
     // during iteration
@@ -53,9 +68,21 @@ private static Set<URL> scanClasspath() {
       ClassLoader allocatorClassLoader = CheckAllocator.class.getClassLoader();
       Enumeration<URL> paths;
       if (allocatorClassLoader == null) {
-        paths = ClassLoader.getSystemResources(ALLOCATOR_PATH);
+        paths = ClassLoader.getSystemResources(ALLOCATOR_PATH_CORE);
+        if (!paths.hasMoreElements()) {
+          paths = ClassLoader.getSystemResources(ALLOCATOR_PATH_UNSAFE);
+        }
+        if (!paths.hasMoreElements()) {
+          paths = ClassLoader.getSystemResources(ALLOCATOR_PATH_NETTY);
+        }
       } else {
-        paths = allocatorClassLoader.getResources(ALLOCATOR_PATH);
+        paths = allocatorClassLoader.getResources(ALLOCATOR_PATH_CORE);
+        if (!paths.hasMoreElements()) {
+          paths = allocatorClassLoader.getResources(ALLOCATOR_PATH_UNSAFE);
+        }
+        if (!paths.hasMoreElements()) {
+          paths = allocatorClassLoader.getResources(ALLOCATOR_PATH_NETTY);
+        }
       }
       while (paths.hasMoreElements()) {
         URL path = paths.nextElement();

java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java

@@ -19,6 +19,7 @@
 
 import java.lang.reflect.Field;
 
+import org.apache.arrow.util.VisibleForTesting;
 import org.checkerframework.checker.nullness.qual.Nullable;
 
 
@@ -64,8 +65,13 @@ public enum AllocationManagerType {
     Unknown,
   }
 
+  /**
+   * Returns the default allocation manager type.
+   * @return the default allocation manager type.
+   */
   @SuppressWarnings("nullness:argument") //enum types valueOf are implicitly non-null
-  static AllocationManagerType getDefaultAllocationManagerType() {
+  @VisibleForTesting
+  public static AllocationManagerType getDefaultAllocationManagerType() {
     AllocationManagerType ret = AllocationManagerType.Unknown;
 
     try {
@@ -122,7 +128,7 @@ private static AllocationManager.Factory getFactory(String clazzName) {
 
   private static AllocationManager.Factory getUnsafeFactory() {
     try {
-      return getFactory("org.apache.arrow.memory.UnsafeAllocationManager");
+      return getFactory("org.apache.arrow.memory.unsafe.UnsafeAllocationManager");
     } catch (RuntimeException e) {
       throw new RuntimeException("Please add arrow-memory-unsafe to your classpath," +
           " No DefaultAllocationManager found to instantiate an UnsafeAllocationManager", e);
@@ -131,7 +137,7 @@ private static AllocationManager.Factory getUnsafeFactory() {
 
   private static AllocationManager.Factory getNettyFactory() {
     try {
-      return getFactory("org.apache.arrow.memory.NettyAllocationManager");
+      return getFactory("org.apache.arrow.memory.netty.NettyAllocationManager");
     } catch (RuntimeException e) {
       throw new RuntimeException("Please add arrow-memory-netty to your classpath," +
           " No DefaultAllocationManager found to instantiate an NettyAllocationManager", e);

java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java

@@ -148,7 +148,7 @@ public Object run() {
       // the static fields above get initialized
       final RuntimeException failure = new RuntimeException(
           "Failed to initialize MemoryUtil. You must start Java with " +
-              "`--add-opens=java.base/java.nio=ALL-UNNAMED` " +
+              "`--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED` " +
               "(See https://arrow.apache.org/docs/java/install.html)", e);
       failure.printStackTrace();
       throw failure;

java/memory/memory-netty/src/test/java/org/apache/arrow/memory/CountingAllocationListener.java → java/memory/memory-core/src/test/java/org/apache/arrow/memory/CountingAllocationListener.java

@@ -17,6 +17,11 @@
 
 package org.apache.arrow.memory;
 
+import org.apache.arrow.memory.AllocationListener;
+import org.apache.arrow.memory.AllocationOutcome;
+import org.apache.arrow.memory.BufferAllocator;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
 /**
  * Counting allocation listener.
  * It counts the number of times it has been invoked, and how much memory allocation it has seen
@@ -30,6 +35,7 @@ final class CountingAllocationListener implements AllocationListener {
   private long totalMem;
   private long currentMem;
   private boolean expandOnFail;
+  @Nullable
   BufferAllocator expandAlloc;
   long expandLimit;
 
@@ -58,6 +64,10 @@ public void onAllocation(long size) {
   @Override
   public boolean onFailedAllocation(long size, AllocationOutcome outcome) {
     if (expandOnFail) {
+      if (expandAlloc == null) {
+        throw new IllegalStateException("expandAlloc must be non-null because this " +
+            "listener is set to expand on failure.");
+      }
       expandAlloc.setLimit(expandLimit);
       return true;
     }

java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestArrowBuf.java

@@ -167,13 +167,15 @@ public void testEnabledHistoricalLog() {
       fieldDebug.set(null, true);
       try (BufferAllocator allocator = new RootAllocator(128)) {
         allocator.buffer(2);
-        Exception e = assertThrows(IllegalStateException.class, () -> allocator.close());
-        assertTrue(e.getMessage().contains("event log for:")); // JDK8, JDK11
+        Exception e = assertThrows(IllegalStateException.class, allocator::close);
+        assertTrue("Exception had the following message: " + e.getMessage(),
+            e.getMessage().contains("event log for:")); // JDK8, JDK11
       } finally {
         fieldDebug.set(null, false);
       }
     } catch (Exception e) {
-      assertTrue(e.toString().contains("java.lang.NoSuchFieldException: modifiers")); // JDK17+
+      assertTrue("Exception had the following toString(): " + e.toString(),
+          e.toString().contains("java.lang.NoSuchFieldException: modifiers")); // JDK17+
     } finally {
       ((Logger) LoggerFactory.getLogger("org.apache.arrow")).setLevel(null);
     }