diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index b8d2f6c0a00..d818269e205 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -77,6 +77,9 @@ groups: c-glib: - test-*c-glib* + java: + - "*java*" + python: - test-*python* diff --git a/docs/source/developers/java/building.rst b/docs/source/developers/java/building.rst index 8b2a504631f..8145292769f 100644 --- a/docs/source/developers/java/building.rst +++ b/docs/source/developers/java/building.rst @@ -32,9 +32,11 @@ Arrow Java uses the `Maven `_ build system. Building requires: -* JDK 8, 9, 10, 11, 17, or 18, but only JDK 8, 11 and 17 are tested in CI. +* JDK 8+ * Maven 3+ +Note: CI will test all supported JDK LTS versions, plus the latest non-LTS version. + Building ======== diff --git a/docs/source/java/overview.rst b/docs/source/java/overview.rst index 4b30b8e000f..dad9272378f 100644 --- a/docs/source/java/overview.rst +++ b/docs/source/java/overview.rst @@ -44,6 +44,9 @@ but some modules are JNI bindings to the C++ library. * - arrow-memory-netty - Memory management implementation based on Netty. - Native + * - arrow-memory-foreign + - (Experimental) Memory management implementation based on java.lang.foreign. Not released, can only be built from source. + - Native * - arrow-vector - An off-heap reference implementation for Arrow columnar data format. - Native diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java index 15120c252fc..05d2068ad91 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java @@ -55,6 +55,11 @@ public enum AllocationManagerType { */ Unsafe, + /** + * (Experimental) java.lang.foreign based allocation manager. + */ + Foreign, + /** * Unknown type. */ @@ -93,6 +98,9 @@ static AllocationManager.Factory getDefaultAllocationManagerFactory() { case Unsafe: DEFAULT_ALLOCATION_MANAGER_FACTORY = getUnsafeFactory(); break; + case Foreign: + DEFAULT_ALLOCATION_MANAGER_FACTORY = getForeignFactory(); + break; case Unknown: LOGGER.info("allocation manager type not specified, using netty as the default type"); DEFAULT_ALLOCATION_MANAGER_FACTORY = getFactory(CheckAllocator.check()); @@ -130,4 +138,13 @@ private static AllocationManager.Factory getNettyFactory() { " No DefaultAllocationManager found to instantiate an NettyAllocationManager", e); } } + + private static AllocationManager.Factory getForeignFactory() { + try { + return getFactory("org.apache.arrow.memory.JavaForeignAllocationManager"); + } catch (RuntimeException e) { + throw new RuntimeException("Please add arrow-memory-foreign to your classpath," + + " No DefaultAllocationManager found to instantiate an JavaForeignAllocationManager", e); + } + } } diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java index bfe496532b1..e4b781c32e1 100644 --- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java @@ -17,47 +17,21 @@ package org.apache.arrow.memory; -import org.apache.arrow.memory.util.MemoryUtil; - /** * The default Allocation Manager Factory for a module. * - * This is only used by tests and contains only a simplistic allocator method. - * */ public class DefaultAllocationManagerFactory implements AllocationManager.Factory { - public static final AllocationManager.Factory FACTORY = new DefaultAllocationManagerFactory(); - private static final ArrowBuf EMPTY = new ArrowBuf(ReferenceManager.NO_OP, - null, - 0, - MemoryUtil.UNSAFE.allocateMemory(0)); + public static final AllocationManager.Factory FACTORY = JavaForeignAllocationManager.FACTORY; @Override public AllocationManager create(BufferAllocator accountingAllocator, long size) { - return new AllocationManager(accountingAllocator) { - private final long allocatedSize = size; - private final long address = MemoryUtil.UNSAFE.allocateMemory(size); - - @Override - public long getSize() { - return allocatedSize; - } - - @Override - protected long memoryAddress() { - return address; - } - - @Override - protected void release0() { - MemoryUtil.UNSAFE.freeMemory(address); - } - }; + return FACTORY.create(accountingAllocator, size); } @Override public ArrowBuf empty() { - return EMPTY; + return FACTORY.empty(); } } diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/JavaForeignAllocationManager.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/JavaForeignAllocationManager.java new file mode 100644 index 00000000000..ce41e7db1e1 --- /dev/null +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/JavaForeignAllocationManager.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; + +/** + * Allocation manager based on java.lang.foreign API. + */ +public final class JavaForeignAllocationManager extends AllocationManager { + + private static final ArrowBuf EMPTY = new ArrowBuf(ReferenceManager.NO_OP, + null, + 0, + MemorySegment.NULL.address() + ); + + public static final AllocationManager.Factory FACTORY = new Factory() { + @Override + public AllocationManager create(BufferAllocator accountingAllocator, long size) { + return new JavaForeignAllocationManager(accountingAllocator, size); + } + + @Override + public ArrowBuf empty() { + return EMPTY; + } + }; + + private final Arena arena; + + private final MemorySegment allocatedMemorySegment; + + private final long allocatedSize; + + private final long allocatedAddress; + + JavaForeignAllocationManager(BufferAllocator accountingAllocator, long requestedSize) { + super(accountingAllocator); + arena = Arena.ofShared(); + allocatedMemorySegment = arena.allocate(requestedSize, /*byteAlignment*/ 8); + allocatedAddress = allocatedMemorySegment.address(); + allocatedSize = requestedSize; + } + + @Override + public long getSize() { + return allocatedSize; + } + + @Override + protected long memoryAddress() { + return allocatedAddress; + } + + @Override + protected void release0() { + arena.close(); + } + +} diff --git a/java/memory/memory-foreign/pom.xml b/java/memory/memory-foreign/pom.xml new file mode 100644 index 00000000000..576ac437391 --- /dev/null +++ b/java/memory/memory-foreign/pom.xml @@ -0,0 +1,30 @@ + + + + + arrow-memory + org.apache.arrow + 15.0.0-SNAPSHOT + + 4.0.0 + + arrow-memory-foreign + Arrow Memory - Foreign + Allocator and utils for allocating memory in Arrow based on java.lang.foreign + + + + org.apache.arrow + arrow-memory-core + + + diff --git a/java/memory/memory-foreign/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java b/java/memory/memory-foreign/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java new file mode 100644 index 00000000000..e4b781c32e1 --- /dev/null +++ b/java/memory/memory-foreign/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +/** + * The default Allocation Manager Factory for a module. + * + */ +public class DefaultAllocationManagerFactory implements AllocationManager.Factory { + + public static final AllocationManager.Factory FACTORY = JavaForeignAllocationManager.FACTORY; + + @Override + public AllocationManager create(BufferAllocator accountingAllocator, long size) { + return FACTORY.create(accountingAllocator, size); + } + + @Override + public ArrowBuf empty() { + return FACTORY.empty(); + } +} diff --git a/java/memory/memory-foreign/src/main/java/org/apache/arrow/memory/JavaForeignAllocationManager.java b/java/memory/memory-foreign/src/main/java/org/apache/arrow/memory/JavaForeignAllocationManager.java new file mode 100644 index 00000000000..ce41e7db1e1 --- /dev/null +++ b/java/memory/memory-foreign/src/main/java/org/apache/arrow/memory/JavaForeignAllocationManager.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; + +/** + * Allocation manager based on java.lang.foreign API. + */ +public final class JavaForeignAllocationManager extends AllocationManager { + + private static final ArrowBuf EMPTY = new ArrowBuf(ReferenceManager.NO_OP, + null, + 0, + MemorySegment.NULL.address() + ); + + public static final AllocationManager.Factory FACTORY = new Factory() { + @Override + public AllocationManager create(BufferAllocator accountingAllocator, long size) { + return new JavaForeignAllocationManager(accountingAllocator, size); + } + + @Override + public ArrowBuf empty() { + return EMPTY; + } + }; + + private final Arena arena; + + private final MemorySegment allocatedMemorySegment; + + private final long allocatedSize; + + private final long allocatedAddress; + + JavaForeignAllocationManager(BufferAllocator accountingAllocator, long requestedSize) { + super(accountingAllocator); + arena = Arena.ofShared(); + allocatedMemorySegment = arena.allocate(requestedSize, /*byteAlignment*/ 8); + allocatedAddress = allocatedMemorySegment.address(); + allocatedSize = requestedSize; + } + + @Override + public long getSize() { + return allocatedSize; + } + + @Override + protected long memoryAddress() { + return allocatedAddress; + } + + @Override + protected void release0() { + arena.close(); + } + +} diff --git a/java/memory/memory-foreign/src/test/java/org/apache/arrow/memory/TestAllocationManagerJavaForeign.java b/java/memory/memory-foreign/src/test/java/org/apache/arrow/memory/TestAllocationManagerJavaForeign.java new file mode 100644 index 00000000000..199f49efdb1 --- /dev/null +++ b/java/memory/memory-foreign/src/test/java/org/apache/arrow/memory/TestAllocationManagerJavaForeign.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +/** + * Test cases for {@link AllocationManager}. + */ +public class TestAllocationManagerJavaForeign { + + @Test + public void testAllocationManagerType() { + + // test Java Foreign allocation manager type + System.setProperty( + DefaultAllocationManagerOption.ALLOCATION_MANAGER_TYPE_PROPERTY_NAME, "Foreign"); + DefaultAllocationManagerOption.AllocationManagerType mgrType = + DefaultAllocationManagerOption.getDefaultAllocationManagerType(); + + assertEquals(DefaultAllocationManagerOption.AllocationManagerType.Foreign, mgrType); + + } +} diff --git a/java/memory/memory-foreign/src/test/java/org/apache/arrow/memory/TestJavaForeignAllocationManager.java b/java/memory/memory-foreign/src/test/java/org/apache/arrow/memory/TestJavaForeignAllocationManager.java new file mode 100644 index 00000000000..f50e6a44f25 --- /dev/null +++ b/java/memory/memory-foreign/src/test/java/org/apache/arrow/memory/TestJavaForeignAllocationManager.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** + * Test cases for {@link JavaForeignAllocationManager}. + */ +public class TestJavaForeignAllocationManager { + + private BaseAllocator createJavaForeignAllocator() { + return new RootAllocator(BaseAllocator.configBuilder() + .allocationManagerFactory(JavaForeignAllocationManager.FACTORY) + .build()); + } + + private void readWriteArrowBuf(ArrowBuf buffer) { + // write buffer + for (long i = 0; i < buffer.capacity() / 8; i++) { + buffer.setLong(i * 8, i); + } + + // read buffer + for (long i = 0; i < buffer.capacity() / 8; i++) { + long val = buffer.getLong(i * 8); + assertEquals(i, val); + } + } + + /** + * Test the memory allocation for {@link JavaForeignAllocationManager}. + */ + @Test + public void testBufferAllocation() { + final long bufSize = 4096L; + try (BaseAllocator allocator = createJavaForeignAllocator(); + ArrowBuf buffer = allocator.buffer(bufSize)) { + assertTrue(buffer.getReferenceManager() instanceof BufferLedger); + BufferLedger bufferLedger = (BufferLedger) buffer.getReferenceManager(); + + // make sure we are using the Java Foreign allocation manager + AllocationManager allocMgr = bufferLedger.getAllocationManager(); + assertTrue(allocMgr instanceof JavaForeignAllocationManager); + JavaForeignAllocationManager mgr = (JavaForeignAllocationManager) allocMgr; + + assertEquals(bufSize, mgr.getSize()); + readWriteArrowBuf(buffer); + } + } +} diff --git a/java/memory/pom.xml b/java/memory/pom.xml index c10263b97f1..c4e544e76e0 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -22,6 +22,7 @@ memory-core + memory-foreign memory-unsafe memory-netty diff --git a/java/pom.xml b/java/pom.xml index 2a9997b7012..0ecc7411f3a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -354,6 +354,10 @@ javax.annotation:javax.annotation-api:* org.apache.hadoop:hadoop-client-api + + com.google.guava:* + com.fasterxml.jackson.core:* + @@ -361,7 +365,7 @@ org.cyclonedx cyclonedx-maven-plugin - 2.7.6 + 2.7.10 package @@ -378,7 +382,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.0.1 + 3.6.1 org.apache.rat @@ -1226,6 +1230,60 @@ + + + jdk21+ + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 21 + 21 + UTF-8 + + -XDcompilePolicy=simple + -Xplugin:ErrorProne -XepExcludedPaths:.*/(target/generated-sources)/.* + -J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED + --enable-preview + + + + com.google.errorprone + error_prone_core + ${error_prone_core.version} + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-opens=java.base/java.nio=ALL-UNNAMED + --enable-preview + + Foreign + + + + + + +