From ecae6f171ea73171e18c24ca3e37fa190255cf6c Mon Sep 17 00:00:00 2001 From: Dane Pitkin Date: Wed, 4 Oct 2023 12:12:02 -0400 Subject: [PATCH 1/6] Add experimental arrow-memory-ffm module --- dev/tasks/tasks.yml | 7 ++ docs/source/java/overview.rst | 3 + .../DefaultAllocationManagerOption.java | 17 ++++ java/memory/memory-ffm/pom.xml | 31 ++++++++ .../DefaultAllocationManagerFactory.java | 37 +++++++++ .../arrow/memory/FfmAllocationManager.java | 77 +++++++++++++++++++ .../memory/TestAllocationManagerFfm.java | 41 ++++++++++ .../memory/TestFfmAllocationManager.java | 68 ++++++++++++++++ java/memory/pom.xml | 1 + java/vector/pom.xml | 17 ++++ 10 files changed, 299 insertions(+) create mode 100644 java/memory/memory-ffm/pom.xml create mode 100644 java/memory/memory-ffm/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java create mode 100644 java/memory/memory-ffm/src/main/java/org/apache/arrow/memory/FfmAllocationManager.java create mode 100644 java/memory/memory-ffm/src/test/java/org/apache/arrow/memory/TestAllocationManagerFfm.java create mode 100644 java/memory/memory-ffm/src/test/java/org/apache/arrow/memory/TestFfmAllocationManager.java diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index b8d2f6c0a00..ad34444ad8a 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -861,6 +861,13 @@ tasks: - arrow-memory-unsafe-{no_rc_snapshot_version}-tests.jar - arrow-memory-unsafe-{no_rc_snapshot_version}.jar - arrow-memory-unsafe-{no_rc_snapshot_version}.pom + - arrow-memory-ffm-{no_rc_snapshot_version}-cyclonedx.json + - arrow-memory-ffm-{no_rc_snapshot_version}-cyclonedx.xml + - arrow-memory-ffm-{no_rc_snapshot_version}-javadoc.jar + - arrow-memory-ffm-{no_rc_snapshot_version}-sources.jar + - arrow-memory-ffm-{no_rc_snapshot_version}-tests.jar + - arrow-memory-ffm-{no_rc_snapshot_version}.jar + - arrow-memory-ffm-{no_rc_snapshot_version}.pom - arrow-memory-{no_rc_snapshot_version}-cyclonedx.json - arrow-memory-{no_rc_snapshot_version}-cyclonedx.xml - arrow-memory-{no_rc_snapshot_version}.pom diff --git a/docs/source/java/overview.rst b/docs/source/java/overview.rst index 4b30b8e000f..55b564bce4c 100644 --- a/docs/source/java/overview.rst +++ b/docs/source/java/overview.rst @@ -44,6 +44,9 @@ but some modules are JNI bindings to the C++ library. * - arrow-memory-netty - Memory management implementation based on Netty. - Native + * - arrow-memory-ffm + - Memory management implementation based on java.lang.foreign. + - Native * - arrow-vector - An off-heap reference implementation for Arrow columnar data format. - Native diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java index 15120c252fc..15be33beace 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java @@ -55,6 +55,11 @@ public enum AllocationManagerType { */ Unsafe, + /** + * FFM based allocation manager. + */ + FFM, + /** * Unknown type. */ @@ -93,6 +98,9 @@ static AllocationManager.Factory getDefaultAllocationManagerFactory() { case Unsafe: DEFAULT_ALLOCATION_MANAGER_FACTORY = getUnsafeFactory(); break; + case FFM: + DEFAULT_ALLOCATION_MANAGER_FACTORY = getFfmFactory(); + break; case Unknown: LOGGER.info("allocation manager type not specified, using netty as the default type"); DEFAULT_ALLOCATION_MANAGER_FACTORY = getFactory(CheckAllocator.check()); @@ -130,4 +138,13 @@ private static AllocationManager.Factory getNettyFactory() { " No DefaultAllocationManager found to instantiate an NettyAllocationManager", e); } } + + private static AllocationManager.Factory getFfmFactory() { + try { + return getFactory("org.apache.arrow.memory.FfmAllocationManager"); + } catch (RuntimeException e) { + throw new RuntimeException("Please add arrow-memory-ffm to your classpath," + + " No DefaultAllocationManager found to instantiate an FfmAllocationManager", e); + } + } } diff --git a/java/memory/memory-ffm/pom.xml b/java/memory/memory-ffm/pom.xml new file mode 100644 index 00000000000..bb18a4144cb --- /dev/null +++ b/java/memory/memory-ffm/pom.xml @@ -0,0 +1,31 @@ + + + + + arrow-memory + org.apache.arrow + 14.0.0-SNAPSHOT + + 4.0.0 + + arrow-memory-ffm + Arrow Memory - FFM + Allocator and utils for allocating memory in Arrow based on java.lang.foreign + + + + org.apache.arrow + arrow-memory-core + ${project.version} + + + diff --git a/java/memory/memory-ffm/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java b/java/memory/memory-ffm/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java new file mode 100644 index 00000000000..0b96fed2f15 --- /dev/null +++ b/java/memory/memory-ffm/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +/** + * The default Allocation Manager Factory for a module. + * + */ +public class DefaultAllocationManagerFactory implements AllocationManager.Factory { + + public static final AllocationManager.Factory FACTORY = FfmAllocationManager.FACTORY; + + @Override + public AllocationManager create(BufferAllocator accountingAllocator, long size) { + return FACTORY.create(accountingAllocator, size); + } + + @Override + public ArrowBuf empty() { + return FACTORY.empty(); + } +} diff --git a/java/memory/memory-ffm/src/main/java/org/apache/arrow/memory/FfmAllocationManager.java b/java/memory/memory-ffm/src/main/java/org/apache/arrow/memory/FfmAllocationManager.java new file mode 100644 index 00000000000..96b69bd5c24 --- /dev/null +++ b/java/memory/memory-ffm/src/main/java/org/apache/arrow/memory/FfmAllocationManager.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; + +/** + * Allocation manager based on java.lang.foreign API. + */ +public final class FfmAllocationManager extends AllocationManager { + + private static final ArrowBuf EMPTY = new ArrowBuf(ReferenceManager.NO_OP, + null, + 0, + MemorySegment.NULL.address() + ); + + public static final AllocationManager.Factory FACTORY = new Factory() { + @Override + public AllocationManager create(BufferAllocator accountingAllocator, long size) { + return new FfmAllocationManager(accountingAllocator, size); + } + + @Override + public ArrowBuf empty() { + return EMPTY; + } + }; + + private final Arena arena; + + private final MemorySegment allocatedMemorySegment; + + private final long allocatedSize; + + private final long allocatedAddress; + + FfmAllocationManager(BufferAllocator accountingAllocator, long requestedSize) { + super(accountingAllocator); + arena = Arena.ofShared(); + allocatedMemorySegment = arena.allocate(requestedSize, /*byteAlignment*/ 8); + allocatedAddress = allocatedMemorySegment.address(); + allocatedSize = requestedSize; + } + + @Override + public long getSize() { + return allocatedSize; + } + + @Override + protected long memoryAddress() { + return allocatedAddress; + } + + @Override + protected void release0() { + arena.close(); + } + +} diff --git a/java/memory/memory-ffm/src/test/java/org/apache/arrow/memory/TestAllocationManagerFfm.java b/java/memory/memory-ffm/src/test/java/org/apache/arrow/memory/TestAllocationManagerFfm.java new file mode 100644 index 00000000000..27a4ada2ba7 --- /dev/null +++ b/java/memory/memory-ffm/src/test/java/org/apache/arrow/memory/TestAllocationManagerFfm.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +/** + * Test cases for {@link AllocationManager}. + */ +public class TestAllocationManagerFfm { + + @Test + public void testAllocationManagerType() { + + // test FFM allocation manager type + System.setProperty( + DefaultAllocationManagerOption.ALLOCATION_MANAGER_TYPE_PROPERTY_NAME, "FFM"); + DefaultAllocationManagerOption.AllocationManagerType mgrType = + DefaultAllocationManagerOption.getDefaultAllocationManagerType(); + + assertEquals(DefaultAllocationManagerOption.AllocationManagerType.FFM, mgrType); + + } +} diff --git a/java/memory/memory-ffm/src/test/java/org/apache/arrow/memory/TestFfmAllocationManager.java b/java/memory/memory-ffm/src/test/java/org/apache/arrow/memory/TestFfmAllocationManager.java new file mode 100644 index 00000000000..df616e4e6aa --- /dev/null +++ b/java/memory/memory-ffm/src/test/java/org/apache/arrow/memory/TestFfmAllocationManager.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** + * Test cases for {@link FfmAllocationManager}. + */ +public class TestFfmAllocationManager { + + private BaseAllocator createFfmAllocator() { + return new RootAllocator(BaseAllocator.configBuilder().allocationManagerFactory(FfmAllocationManager.FACTORY) + .build()); + } + + private void readWriteArrowBuf(ArrowBuf buffer) { + // write buffer + for (long i = 0; i < buffer.capacity() / 8; i++) { + buffer.setLong(i * 8, i); + } + + // read buffer + for (long i = 0; i < buffer.capacity() / 8; i++) { + long val = buffer.getLong(i * 8); + assertEquals(i, val); + } + } + + /** + * Test the memory allocation for {@link FfmAllocationManager}. + */ + @Test + public void testBufferAllocation() { + final long bufSize = 4096L; + try (BaseAllocator allocator = createFfmAllocator(); + ArrowBuf buffer = allocator.buffer(bufSize)) { + assertTrue(buffer.getReferenceManager() instanceof BufferLedger); + BufferLedger bufferLedger = (BufferLedger) buffer.getReferenceManager(); + + // make sure we are using FFM allocation manager + AllocationManager allocMgr = bufferLedger.getAllocationManager(); + assertTrue(allocMgr instanceof FfmAllocationManager); + FfmAllocationManager ffmMgr = (FfmAllocationManager) allocMgr; + + assertEquals(bufSize, ffmMgr.getSize()); + readWriteArrowBuf(buffer); + } + } +} diff --git a/java/memory/pom.xml b/java/memory/pom.xml index c10263b97f1..802947c18a9 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -22,6 +22,7 @@ memory-core + memory-ffm memory-unsafe memory-netty diff --git a/java/vector/pom.xml b/java/vector/pom.xml index 4c8bf1e594a..cc7fba70574 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -62,6 +62,10 @@ test + org.apache.arrow + arrow-memory-ffm + test + com.google.flatbuffers flatbuffers-java ${dep.fbs.version} @@ -129,6 +133,19 @@ netty + + run-unsafe2 + test + + test + + + + org.apache.arrow:arrow-memory-ffm + + ffm + + From 609e7ef68d3795fcbcb53456b1e28514ef70064e Mon Sep 17 00:00:00 2001 From: Dane Pitkin Date: Mon, 13 Nov 2023 15:00:25 -0500 Subject: [PATCH 2/6] only allow explicit build for arrow-memory-ffm --- dev/tasks/tasks.yml | 10 +++------- docs/source/developers/java/building.rst | 4 +++- docs/source/java/overview.rst | 2 +- java/bom/pom.xml | 1 + java/memory/memory-ffm/pom.xml | 3 +-- java/memory/pom.xml | 1 - java/pom.xml | 11 +++++++++++ java/vector/pom.xml | 4 ---- 8 files changed, 20 insertions(+), 16 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index ad34444ad8a..d818269e205 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -77,6 +77,9 @@ groups: c-glib: - test-*c-glib* + java: + - "*java*" + python: - test-*python* @@ -861,13 +864,6 @@ tasks: - arrow-memory-unsafe-{no_rc_snapshot_version}-tests.jar - arrow-memory-unsafe-{no_rc_snapshot_version}.jar - arrow-memory-unsafe-{no_rc_snapshot_version}.pom - - arrow-memory-ffm-{no_rc_snapshot_version}-cyclonedx.json - - arrow-memory-ffm-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-memory-ffm-{no_rc_snapshot_version}-javadoc.jar - - arrow-memory-ffm-{no_rc_snapshot_version}-sources.jar - - arrow-memory-ffm-{no_rc_snapshot_version}-tests.jar - - arrow-memory-ffm-{no_rc_snapshot_version}.jar - - arrow-memory-ffm-{no_rc_snapshot_version}.pom - arrow-memory-{no_rc_snapshot_version}-cyclonedx.json - arrow-memory-{no_rc_snapshot_version}-cyclonedx.xml - arrow-memory-{no_rc_snapshot_version}.pom diff --git a/docs/source/developers/java/building.rst b/docs/source/developers/java/building.rst index 8b2a504631f..2e094d0a240 100644 --- a/docs/source/developers/java/building.rst +++ b/docs/source/developers/java/building.rst @@ -32,9 +32,11 @@ Arrow Java uses the `Maven `_ build system. Building requires: -* JDK 8, 9, 10, 11, 17, or 18, but only JDK 8, 11 and 17 are tested in CI. +* JDK 8+ * Maven 3+ +Note: CI tests all supported JDK LTS versions, plus the latest non-LTS version. + Building ======== diff --git a/docs/source/java/overview.rst b/docs/source/java/overview.rst index 55b564bce4c..6dd6b8404cb 100644 --- a/docs/source/java/overview.rst +++ b/docs/source/java/overview.rst @@ -45,7 +45,7 @@ but some modules are JNI bindings to the C++ library. - Memory management implementation based on Netty. - Native * - arrow-memory-ffm - - Memory management implementation based on java.lang.foreign. + - (Experimental) Memory management implementation based on java.lang.foreign. Not released, can only be built from source. - Native * - arrow-vector - An off-heap reference implementation for Arrow columnar data format. diff --git a/java/bom/pom.xml b/java/bom/pom.xml index 1f6f854f600..d3bc644b29d 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -145,4 +145,5 @@ + diff --git a/java/memory/memory-ffm/pom.xml b/java/memory/memory-ffm/pom.xml index bb18a4144cb..036459ddd87 100644 --- a/java/memory/memory-ffm/pom.xml +++ b/java/memory/memory-ffm/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 14.0.0-SNAPSHOT + 15.0.0-SNAPSHOT 4.0.0 @@ -25,7 +25,6 @@ org.apache.arrow arrow-memory-core - ${project.version} diff --git a/java/memory/pom.xml b/java/memory/pom.xml index 802947c18a9..c10263b97f1 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -22,7 +22,6 @@ memory-core - memory-ffm memory-unsafe memory-netty diff --git a/java/pom.xml b/java/pom.xml index 2a9997b7012..a6c0b8d5958 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -1226,6 +1226,17 @@ + + + jdk21+ + + [21,] + + + memory/memory-ffm + + + diff --git a/java/vector/pom.xml b/java/vector/pom.xml index cc7fba70574..5e07f46cdb5 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -62,10 +62,6 @@ test - org.apache.arrow - arrow-memory-ffm - test - com.google.flatbuffers flatbuffers-java ${dep.fbs.version} From 1ca51fce02c147fb0561b477da8f13a09213f54e Mon Sep 17 00:00:00 2001 From: Dane Pitkin Date: Mon, 13 Nov 2023 15:04:01 -0500 Subject: [PATCH 3/6] Remove unecessary edits --- java/bom/pom.xml | 1 - java/vector/pom.xml | 13 ------------- 2 files changed, 14 deletions(-) diff --git a/java/bom/pom.xml b/java/bom/pom.xml index d3bc644b29d..1f6f854f600 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -145,5 +145,4 @@ - diff --git a/java/vector/pom.xml b/java/vector/pom.xml index 5e07f46cdb5..4c8bf1e594a 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -129,19 +129,6 @@ netty - - run-unsafe2 - test - - test - - - - org.apache.arrow:arrow-memory-ffm - - ffm - - From ef6665dcbb58afb7c3a3fb971ac73ab7346a34e6 Mon Sep 17 00:00:00 2001 From: Dane Pitkin Date: Mon, 13 Nov 2023 15:04:40 -0500 Subject: [PATCH 4/6] Clean up documentation wording --- docs/source/developers/java/building.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/developers/java/building.rst b/docs/source/developers/java/building.rst index 2e094d0a240..8145292769f 100644 --- a/docs/source/developers/java/building.rst +++ b/docs/source/developers/java/building.rst @@ -35,7 +35,7 @@ Building requires: * JDK 8+ * Maven 3+ -Note: CI tests all supported JDK LTS versions, plus the latest non-LTS version. +Note: CI will test all supported JDK LTS versions, plus the latest non-LTS version. Building ======== From 0cd79396c440f5d42ea53a5aa357e2d926952275 Mon Sep 17 00:00:00 2001 From: Dane Pitkin Date: Mon, 13 Nov 2023 15:17:14 -0500 Subject: [PATCH 5/6] Add runtime exception message for version mismatch --- .../memory/DefaultAllocationManagerOption.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java index 15be33beace..ffc9578e0ce 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java @@ -34,6 +34,10 @@ public class DefaultAllocationManagerOption { */ public static final String ALLOCATION_MANAGER_TYPE_PROPERTY_NAME = "arrow.allocation.manager.type"; + // Java 1.8, 9, 11, 17, 21 becomes 1, 9, 11, 17, and 21. + private static final int majorVersion = + Integer.parseInt(System.getProperty("java.specification.version").split("\\D+")[0]); + static final org.slf4j.Logger LOGGER = org.slf4j.LoggerFactory.getLogger(DefaultAllocationManagerOption.class); /** @@ -56,7 +60,7 @@ public enum AllocationManagerType { Unsafe, /** - * FFM based allocation manager. + * (Experimental) FFM based allocation manager. */ FFM, @@ -143,8 +147,13 @@ private static AllocationManager.Factory getFfmFactory() { try { return getFactory("org.apache.arrow.memory.FfmAllocationManager"); } catch (RuntimeException e) { - throw new RuntimeException("Please add arrow-memory-ffm to your classpath," + - " No DefaultAllocationManager found to instantiate an FfmAllocationManager", e); + if (majorVersion < 21) { + throw new RuntimeException("arrow-memory-ffm requires JDK 21+, current JDK version: " + + majorVersion); + } else { + throw new RuntimeException("Please add arrow-memory-ffm to your classpath," + + " No DefaultAllocationManager found to instantiate an FfmAllocationManager", e); + } } } } From 8c05046dc042b54a643447288575cad1e794ba3f Mon Sep 17 00:00:00 2001 From: Dane Pitkin Date: Tue, 14 Nov 2023 18:05:03 -0500 Subject: [PATCH 6/6] address comments, update maven profile --- docs/source/java/overview.rst | 2 +- .../DefaultAllocationManagerOption.java | 25 ++---- .../DefaultAllocationManagerFactory.java | 32 +------- .../memory/JavaForeignAllocationManager.java} | 6 +- .../{memory-ffm => memory-foreign}/pom.xml | 4 +- .../DefaultAllocationManagerFactory.java | 2 +- .../memory/JavaForeignAllocationManager.java | 77 +++++++++++++++++++ .../TestAllocationManagerJavaForeign.java} | 8 +- .../TestJavaForeignAllocationManager.java} | 21 ++--- java/memory/pom.xml | 1 + java/pom.xml | 63 +++++++++++++-- 11 files changed, 166 insertions(+), 75 deletions(-) rename java/memory/{memory-ffm/src/main/java/org/apache/arrow/memory/FfmAllocationManager.java => memory-core/src/test/java/org/apache/arrow/memory/JavaForeignAllocationManager.java} (89%) rename java/memory/{memory-ffm => memory-foreign}/pom.xml (94%) rename java/memory/{memory-ffm => memory-foreign}/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java (92%) create mode 100644 java/memory/memory-foreign/src/main/java/org/apache/arrow/memory/JavaForeignAllocationManager.java rename java/memory/{memory-ffm/src/test/java/org/apache/arrow/memory/TestAllocationManagerFfm.java => memory-foreign/src/test/java/org/apache/arrow/memory/TestAllocationManagerJavaForeign.java} (89%) rename java/memory/{memory-ffm/src/test/java/org/apache/arrow/memory/TestFfmAllocationManager.java => memory-foreign/src/test/java/org/apache/arrow/memory/TestJavaForeignAllocationManager.java} (71%) diff --git a/docs/source/java/overview.rst b/docs/source/java/overview.rst index 6dd6b8404cb..dad9272378f 100644 --- a/docs/source/java/overview.rst +++ b/docs/source/java/overview.rst @@ -44,7 +44,7 @@ but some modules are JNI bindings to the C++ library. * - arrow-memory-netty - Memory management implementation based on Netty. - Native - * - arrow-memory-ffm + * - arrow-memory-foreign - (Experimental) Memory management implementation based on java.lang.foreign. Not released, can only be built from source. - Native * - arrow-vector diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java index ffc9578e0ce..05d2068ad91 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java @@ -34,10 +34,6 @@ public class DefaultAllocationManagerOption { */ public static final String ALLOCATION_MANAGER_TYPE_PROPERTY_NAME = "arrow.allocation.manager.type"; - // Java 1.8, 9, 11, 17, 21 becomes 1, 9, 11, 17, and 21. - private static final int majorVersion = - Integer.parseInt(System.getProperty("java.specification.version").split("\\D+")[0]); - static final org.slf4j.Logger LOGGER = org.slf4j.LoggerFactory.getLogger(DefaultAllocationManagerOption.class); /** @@ -60,9 +56,9 @@ public enum AllocationManagerType { Unsafe, /** - * (Experimental) FFM based allocation manager. + * (Experimental) java.lang.foreign based allocation manager. */ - FFM, + Foreign, /** * Unknown type. @@ -102,8 +98,8 @@ static AllocationManager.Factory getDefaultAllocationManagerFactory() { case Unsafe: DEFAULT_ALLOCATION_MANAGER_FACTORY = getUnsafeFactory(); break; - case FFM: - DEFAULT_ALLOCATION_MANAGER_FACTORY = getFfmFactory(); + case Foreign: + DEFAULT_ALLOCATION_MANAGER_FACTORY = getForeignFactory(); break; case Unknown: LOGGER.info("allocation manager type not specified, using netty as the default type"); @@ -143,17 +139,12 @@ private static AllocationManager.Factory getNettyFactory() { } } - private static AllocationManager.Factory getFfmFactory() { + private static AllocationManager.Factory getForeignFactory() { try { - return getFactory("org.apache.arrow.memory.FfmAllocationManager"); + return getFactory("org.apache.arrow.memory.JavaForeignAllocationManager"); } catch (RuntimeException e) { - if (majorVersion < 21) { - throw new RuntimeException("arrow-memory-ffm requires JDK 21+, current JDK version: " + - majorVersion); - } else { - throw new RuntimeException("Please add arrow-memory-ffm to your classpath," + - " No DefaultAllocationManager found to instantiate an FfmAllocationManager", e); - } + throw new RuntimeException("Please add arrow-memory-foreign to your classpath," + + " No DefaultAllocationManager found to instantiate an JavaForeignAllocationManager", e); } } } diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java index bfe496532b1..e4b781c32e1 100644 --- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java @@ -17,47 +17,21 @@ package org.apache.arrow.memory; -import org.apache.arrow.memory.util.MemoryUtil; - /** * The default Allocation Manager Factory for a module. * - * This is only used by tests and contains only a simplistic allocator method. - * */ public class DefaultAllocationManagerFactory implements AllocationManager.Factory { - public static final AllocationManager.Factory FACTORY = new DefaultAllocationManagerFactory(); - private static final ArrowBuf EMPTY = new ArrowBuf(ReferenceManager.NO_OP, - null, - 0, - MemoryUtil.UNSAFE.allocateMemory(0)); + public static final AllocationManager.Factory FACTORY = JavaForeignAllocationManager.FACTORY; @Override public AllocationManager create(BufferAllocator accountingAllocator, long size) { - return new AllocationManager(accountingAllocator) { - private final long allocatedSize = size; - private final long address = MemoryUtil.UNSAFE.allocateMemory(size); - - @Override - public long getSize() { - return allocatedSize; - } - - @Override - protected long memoryAddress() { - return address; - } - - @Override - protected void release0() { - MemoryUtil.UNSAFE.freeMemory(address); - } - }; + return FACTORY.create(accountingAllocator, size); } @Override public ArrowBuf empty() { - return EMPTY; + return FACTORY.empty(); } } diff --git a/java/memory/memory-ffm/src/main/java/org/apache/arrow/memory/FfmAllocationManager.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/JavaForeignAllocationManager.java similarity index 89% rename from java/memory/memory-ffm/src/main/java/org/apache/arrow/memory/FfmAllocationManager.java rename to java/memory/memory-core/src/test/java/org/apache/arrow/memory/JavaForeignAllocationManager.java index 96b69bd5c24..ce41e7db1e1 100644 --- a/java/memory/memory-ffm/src/main/java/org/apache/arrow/memory/FfmAllocationManager.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/JavaForeignAllocationManager.java @@ -23,7 +23,7 @@ /** * Allocation manager based on java.lang.foreign API. */ -public final class FfmAllocationManager extends AllocationManager { +public final class JavaForeignAllocationManager extends AllocationManager { private static final ArrowBuf EMPTY = new ArrowBuf(ReferenceManager.NO_OP, null, @@ -34,7 +34,7 @@ public final class FfmAllocationManager extends AllocationManager { public static final AllocationManager.Factory FACTORY = new Factory() { @Override public AllocationManager create(BufferAllocator accountingAllocator, long size) { - return new FfmAllocationManager(accountingAllocator, size); + return new JavaForeignAllocationManager(accountingAllocator, size); } @Override @@ -51,7 +51,7 @@ public ArrowBuf empty() { private final long allocatedAddress; - FfmAllocationManager(BufferAllocator accountingAllocator, long requestedSize) { + JavaForeignAllocationManager(BufferAllocator accountingAllocator, long requestedSize) { super(accountingAllocator); arena = Arena.ofShared(); allocatedMemorySegment = arena.allocate(requestedSize, /*byteAlignment*/ 8); diff --git a/java/memory/memory-ffm/pom.xml b/java/memory/memory-foreign/pom.xml similarity index 94% rename from java/memory/memory-ffm/pom.xml rename to java/memory/memory-foreign/pom.xml index 036459ddd87..576ac437391 100644 --- a/java/memory/memory-ffm/pom.xml +++ b/java/memory/memory-foreign/pom.xml @@ -17,8 +17,8 @@ 4.0.0 - arrow-memory-ffm - Arrow Memory - FFM + arrow-memory-foreign + Arrow Memory - Foreign Allocator and utils for allocating memory in Arrow based on java.lang.foreign diff --git a/java/memory/memory-ffm/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java b/java/memory/memory-foreign/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java similarity index 92% rename from java/memory/memory-ffm/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java rename to java/memory/memory-foreign/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java index 0b96fed2f15..e4b781c32e1 100644 --- a/java/memory/memory-ffm/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java +++ b/java/memory/memory-foreign/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java @@ -23,7 +23,7 @@ */ public class DefaultAllocationManagerFactory implements AllocationManager.Factory { - public static final AllocationManager.Factory FACTORY = FfmAllocationManager.FACTORY; + public static final AllocationManager.Factory FACTORY = JavaForeignAllocationManager.FACTORY; @Override public AllocationManager create(BufferAllocator accountingAllocator, long size) { diff --git a/java/memory/memory-foreign/src/main/java/org/apache/arrow/memory/JavaForeignAllocationManager.java b/java/memory/memory-foreign/src/main/java/org/apache/arrow/memory/JavaForeignAllocationManager.java new file mode 100644 index 00000000000..ce41e7db1e1 --- /dev/null +++ b/java/memory/memory-foreign/src/main/java/org/apache/arrow/memory/JavaForeignAllocationManager.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; + +/** + * Allocation manager based on java.lang.foreign API. + */ +public final class JavaForeignAllocationManager extends AllocationManager { + + private static final ArrowBuf EMPTY = new ArrowBuf(ReferenceManager.NO_OP, + null, + 0, + MemorySegment.NULL.address() + ); + + public static final AllocationManager.Factory FACTORY = new Factory() { + @Override + public AllocationManager create(BufferAllocator accountingAllocator, long size) { + return new JavaForeignAllocationManager(accountingAllocator, size); + } + + @Override + public ArrowBuf empty() { + return EMPTY; + } + }; + + private final Arena arena; + + private final MemorySegment allocatedMemorySegment; + + private final long allocatedSize; + + private final long allocatedAddress; + + JavaForeignAllocationManager(BufferAllocator accountingAllocator, long requestedSize) { + super(accountingAllocator); + arena = Arena.ofShared(); + allocatedMemorySegment = arena.allocate(requestedSize, /*byteAlignment*/ 8); + allocatedAddress = allocatedMemorySegment.address(); + allocatedSize = requestedSize; + } + + @Override + public long getSize() { + return allocatedSize; + } + + @Override + protected long memoryAddress() { + return allocatedAddress; + } + + @Override + protected void release0() { + arena.close(); + } + +} diff --git a/java/memory/memory-ffm/src/test/java/org/apache/arrow/memory/TestAllocationManagerFfm.java b/java/memory/memory-foreign/src/test/java/org/apache/arrow/memory/TestAllocationManagerJavaForeign.java similarity index 89% rename from java/memory/memory-ffm/src/test/java/org/apache/arrow/memory/TestAllocationManagerFfm.java rename to java/memory/memory-foreign/src/test/java/org/apache/arrow/memory/TestAllocationManagerJavaForeign.java index 27a4ada2ba7..199f49efdb1 100644 --- a/java/memory/memory-ffm/src/test/java/org/apache/arrow/memory/TestAllocationManagerFfm.java +++ b/java/memory/memory-foreign/src/test/java/org/apache/arrow/memory/TestAllocationManagerJavaForeign.java @@ -24,18 +24,18 @@ /** * Test cases for {@link AllocationManager}. */ -public class TestAllocationManagerFfm { +public class TestAllocationManagerJavaForeign { @Test public void testAllocationManagerType() { - // test FFM allocation manager type + // test Java Foreign allocation manager type System.setProperty( - DefaultAllocationManagerOption.ALLOCATION_MANAGER_TYPE_PROPERTY_NAME, "FFM"); + DefaultAllocationManagerOption.ALLOCATION_MANAGER_TYPE_PROPERTY_NAME, "Foreign"); DefaultAllocationManagerOption.AllocationManagerType mgrType = DefaultAllocationManagerOption.getDefaultAllocationManagerType(); - assertEquals(DefaultAllocationManagerOption.AllocationManagerType.FFM, mgrType); + assertEquals(DefaultAllocationManagerOption.AllocationManagerType.Foreign, mgrType); } } diff --git a/java/memory/memory-ffm/src/test/java/org/apache/arrow/memory/TestFfmAllocationManager.java b/java/memory/memory-foreign/src/test/java/org/apache/arrow/memory/TestJavaForeignAllocationManager.java similarity index 71% rename from java/memory/memory-ffm/src/test/java/org/apache/arrow/memory/TestFfmAllocationManager.java rename to java/memory/memory-foreign/src/test/java/org/apache/arrow/memory/TestJavaForeignAllocationManager.java index df616e4e6aa..f50e6a44f25 100644 --- a/java/memory/memory-ffm/src/test/java/org/apache/arrow/memory/TestFfmAllocationManager.java +++ b/java/memory/memory-foreign/src/test/java/org/apache/arrow/memory/TestJavaForeignAllocationManager.java @@ -23,12 +23,13 @@ import org.junit.Test; /** - * Test cases for {@link FfmAllocationManager}. + * Test cases for {@link JavaForeignAllocationManager}. */ -public class TestFfmAllocationManager { +public class TestJavaForeignAllocationManager { - private BaseAllocator createFfmAllocator() { - return new RootAllocator(BaseAllocator.configBuilder().allocationManagerFactory(FfmAllocationManager.FACTORY) + private BaseAllocator createJavaForeignAllocator() { + return new RootAllocator(BaseAllocator.configBuilder() + .allocationManagerFactory(JavaForeignAllocationManager.FACTORY) .build()); } @@ -46,22 +47,22 @@ private void readWriteArrowBuf(ArrowBuf buffer) { } /** - * Test the memory allocation for {@link FfmAllocationManager}. + * Test the memory allocation for {@link JavaForeignAllocationManager}. */ @Test public void testBufferAllocation() { final long bufSize = 4096L; - try (BaseAllocator allocator = createFfmAllocator(); + try (BaseAllocator allocator = createJavaForeignAllocator(); ArrowBuf buffer = allocator.buffer(bufSize)) { assertTrue(buffer.getReferenceManager() instanceof BufferLedger); BufferLedger bufferLedger = (BufferLedger) buffer.getReferenceManager(); - // make sure we are using FFM allocation manager + // make sure we are using the Java Foreign allocation manager AllocationManager allocMgr = bufferLedger.getAllocationManager(); - assertTrue(allocMgr instanceof FfmAllocationManager); - FfmAllocationManager ffmMgr = (FfmAllocationManager) allocMgr; + assertTrue(allocMgr instanceof JavaForeignAllocationManager); + JavaForeignAllocationManager mgr = (JavaForeignAllocationManager) allocMgr; - assertEquals(bufSize, ffmMgr.getSize()); + assertEquals(bufSize, mgr.getSize()); readWriteArrowBuf(buffer); } } diff --git a/java/memory/pom.xml b/java/memory/pom.xml index c10263b97f1..c4e544e76e0 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -22,6 +22,7 @@ memory-core + memory-foreign memory-unsafe memory-netty diff --git a/java/pom.xml b/java/pom.xml index a6c0b8d5958..0ecc7411f3a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -354,6 +354,10 @@ javax.annotation:javax.annotation-api:* org.apache.hadoop:hadoop-client-api + + com.google.guava:* + com.fasterxml.jackson.core:* + @@ -361,7 +365,7 @@ org.cyclonedx cyclonedx-maven-plugin - 2.7.6 + 2.7.10 package @@ -378,7 +382,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.0.1 + 3.6.1 org.apache.rat @@ -1229,12 +1233,55 @@ jdk21+ - - [21,] - - - memory/memory-ffm - + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 21 + 21 + UTF-8 + + -XDcompilePolicy=simple + -Xplugin:ErrorProne -XepExcludedPaths:.*/(target/generated-sources)/.* + -J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED + --enable-preview + + + + com.google.errorprone + error_prone_core + ${error_prone_core.version} + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-opens=java.base/java.nio=ALL-UNNAMED + --enable-preview + + Foreign + + + + +