From 9b12e89d772ea65cfd07f87184a4daa3e4da9c0d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 3 Jun 2020 12:46:01 +0200 Subject: [PATCH] ARROW-9300: [Java] Separate Netty Memory to its own module This finishes the netty split in arrow-memory and creates 3 new modules * memory-core: core memory implementation * memory-netty: netty allocation manager * memory-unsafe: unsafe allocation manager The bulk of the changes here are moving files and adding the correct dependencies to poms --- java/adapter/avro/pom.xml | 12 +- java/adapter/jdbc/pom.xml | 12 +- java/adapter/orc/pom.xml | 8 +- java/algorithm/pom.xml | 8 +- java/flight/flight-core/pom.xml | 8 +- java/flight/flight-grpc/pom.xml | 12 +- java/gandiva/pom.xml | 8 +- java/memory/memory-core/pom.xml | 60 ++++++++++ .../org/apache/arrow/memory/Accountant.java | 0 .../arrow/memory/AllocationListener.java | 0 .../arrow/memory/AllocationManager.java | 0 .../arrow/memory/AllocationOutcome.java | 0 .../memory/AllocationOutcomeDetails.java | 0 .../arrow/memory/AllocationReservation.java | 0 .../memory/AllocatorClosedException.java | 0 .../org/apache/arrow/memory/ArrowBuf.java | 0 .../apache/arrow/memory/BaseAllocator.java | 0 .../apache/arrow/memory/BoundsChecking.java | 0 .../apache/arrow/memory/BufferAllocator.java | 0 .../org/apache/arrow/memory/BufferLedger.java | 0 .../apache/arrow/memory/BufferManager.java | 0 .../apache/arrow/memory/CheckAllocator.java | 3 +- .../apache/arrow/memory/ChildAllocator.java | 0 .../DefaultAllocationManagerOption.java | 14 ++- .../arrow/memory/LowCostIdentityHashMap.java | 0 .../arrow/memory/OutOfMemoryException.java | 0 .../arrow/memory/OwnershipTransferNOOP.java | 0 .../arrow/memory/OwnershipTransferResult.java | 0 .../java/org/apache/arrow/memory/README.md | 0 .../apache/arrow/memory/ReferenceManager.java | 0 .../apache/arrow/memory/RootAllocator.java | 0 .../arrow/memory/ValueWithKeyIncluded.java | 0 .../org/apache/arrow/memory/package-info.java | 0 .../rounding/DefaultRoundingPolicy.java | 6 +- .../arrow/memory/rounding/RoundingPolicy.java | 0 .../rounding/SegmentRoundingPolicy.java | 0 .../arrow/memory/util/ArrowBufPointer.java | 0 .../arrow/memory/util/AssertionUtil.java | 0 .../arrow/memory/util/AutoCloseableLock.java | 0 .../memory/util/ByteFunctionHelpers.java | 0 .../apache/arrow/memory/util/CommonUtil.java | 0 .../arrow/memory/util/HistoricalLog.java | 0 .../arrow/memory/util/LargeMemoryUtil.java | 0 .../apache/arrow/memory/util/MemoryUtil.java | 0 .../apache/arrow/memory/util/StackTrace.java | 0 .../memory/util/hash/ArrowBufHasher.java | 0 .../arrow/memory/util/hash/MurmurHasher.java | 0 .../arrow/memory/util/hash/SimpleHasher.java | 0 .../org/apache/arrow/util/Preconditions.java | 0 .../apache/arrow/util/VisibleForTesting.java | 0 .../DefaultAllocationManagerFactory.java | 63 +++++++++++ .../apache/arrow/memory/TestAccountant.java | 0 .../arrow/memory/TestAllocationManager.java | 17 +-- .../org/apache/arrow/memory/TestArrowBuf.java | 0 .../arrow/memory/TestBoundaryChecking.java | 0 .../memory/TestLowCostIdentityHashMap.java | 0 .../memory/util/TestArrowBufPointer.java | 0 .../memory/util/TestByteFunctionHelpers.java | 0 .../memory/util/hash/TestArrowBufHasher.java | 0 java/memory/memory-netty/pom.xml | 103 ++++++++++++++++++ .../io/netty/buffer/ExpandableByteBuf.java | 0 .../java/io/netty/buffer/LargeBuffer.java | 0 .../netty/buffer/MutableWrappedByteBuf.java | 0 .../java/io/netty/buffer/NettyArrowBuf.java | 0 .../netty/buffer/PooledByteBufAllocatorL.java | 0 .../buffer/UnsafeDirectLittleEndian.java | 0 .../arrow/memory/ArrowByteBufAllocator.java | 0 .../DefaultAllocationManagerFactory.java | 10 +- .../arrow/memory/NettyAllocationManager.java | 0 .../io/netty/buffer/TestNettyArrowBuf.java | 0 .../arrow/memory/ITTestLargeArrowBuf.java | 0 .../memory/TestAllocationManagerNetty.java | 39 +++++++ .../arrow/memory/TestBaseAllocator.java | 0 .../arrow/memory/TestEmptyArrowBuf.java | 0 .../apache/arrow/memory/TestEndianness.java | 0 .../memory/TestNettyAllocationManager.java | 0 .../src/test/resources/logback.xml | 0 java/memory/memory-unsafe/pom.xml | 54 +++++++++ .../DefaultAllocationManagerFactory.java | 37 +++++++ .../arrow/memory/UnsafeAllocationManager.java | 0 .../memory/TestAllocationManagerUnsafe.java | 41 +++++++ .../memory/TestUnsafeAllocationManager.java | 0 java/memory/pom.xml | 65 +---------- java/performance/pom.xml | 8 +- java/tools/pom.xml | 9 +- java/vector/pom.xml | 7 +- .../src/main/codegen/includes/vv_imports.ftl | 2 - 87 files changed, 500 insertions(+), 106 deletions(-) create mode 100644 java/memory/memory-core/pom.xml rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/Accountant.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/AllocationListener.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/AllocationManager.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/AllocationOutcome.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/AllocationOutcomeDetails.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/AllocationReservation.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/ArrowBuf.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/BaseAllocator.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/BoundsChecking.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/BufferAllocator.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/BufferLedger.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/BufferManager.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/CheckAllocator.java (95%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/ChildAllocator.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java (86%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/OutOfMemoryException.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/OwnershipTransferNOOP.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/OwnershipTransferResult.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/README.md (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/ReferenceManager.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/RootAllocator.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/ValueWithKeyIncluded.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/package-info.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java (94%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/rounding/RoundingPolicy.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/rounding/SegmentRoundingPolicy.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/util/ArrowBufPointer.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/util/AssertionUtil.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/util/AutoCloseableLock.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/util/CommonUtil.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/util/LargeMemoryUtil.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/util/StackTrace.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/util/hash/ArrowBufHasher.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/util/hash/MurmurHasher.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/memory/util/hash/SimpleHasher.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/util/Preconditions.java (100%) rename java/memory/{ => memory-core}/src/main/java/org/apache/arrow/util/VisibleForTesting.java (100%) create mode 100644 java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java rename java/memory/{ => memory-core}/src/test/java/org/apache/arrow/memory/TestAccountant.java (100%) rename java/memory/{ => memory-core}/src/test/java/org/apache/arrow/memory/TestAllocationManager.java (68%) rename java/memory/{ => memory-core}/src/test/java/org/apache/arrow/memory/TestArrowBuf.java (100%) rename java/memory/{ => memory-core}/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java (100%) rename java/memory/{ => memory-core}/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java (100%) rename java/memory/{ => memory-core}/src/test/java/org/apache/arrow/memory/util/TestArrowBufPointer.java (100%) rename java/memory/{ => memory-core}/src/test/java/org/apache/arrow/memory/util/TestByteFunctionHelpers.java (100%) rename java/memory/{ => memory-core}/src/test/java/org/apache/arrow/memory/util/hash/TestArrowBufHasher.java (100%) create mode 100644 java/memory/memory-netty/pom.xml rename java/memory/{ => memory-netty}/src/main/java/io/netty/buffer/ExpandableByteBuf.java (100%) rename java/memory/{ => memory-netty}/src/main/java/io/netty/buffer/LargeBuffer.java (100%) rename java/memory/{ => memory-netty}/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java (100%) rename java/memory/{ => memory-netty}/src/main/java/io/netty/buffer/NettyArrowBuf.java (100%) rename java/memory/{ => memory-netty}/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java (100%) rename java/memory/{ => memory-netty}/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java (100%) rename java/memory/{ => memory-netty}/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java (100%) rename java/memory/{ => memory-netty}/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java (75%) rename java/memory/{ => memory-netty}/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java (100%) rename java/memory/{ => memory-netty}/src/test/java/io/netty/buffer/TestNettyArrowBuf.java (100%) rename java/memory/{ => memory-netty}/src/test/java/org/apache/arrow/memory/ITTestLargeArrowBuf.java (100%) create mode 100644 java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestAllocationManagerNetty.java rename java/memory/{ => memory-netty}/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java (100%) rename java/memory/{ => memory-netty}/src/test/java/org/apache/arrow/memory/TestEmptyArrowBuf.java (100%) rename java/memory/{ => memory-netty}/src/test/java/org/apache/arrow/memory/TestEndianness.java (100%) rename java/memory/{ => memory-netty}/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java (100%) rename java/memory/{ => memory-netty}/src/test/resources/logback.xml (100%) create mode 100644 java/memory/memory-unsafe/pom.xml create mode 100644 java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java rename java/memory/{ => memory-unsafe}/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java (100%) create mode 100644 java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/TestAllocationManagerUnsafe.java rename java/memory/{ => memory-unsafe}/src/test/java/org/apache/arrow/memory/TestUnsafeAllocationManager.java (100%) diff --git a/java/adapter/avro/pom.xml b/java/adapter/avro/pom.xml index d9c618250c4..5274ccae2db 100644 --- a/java/adapter/avro/pom.xml +++ b/java/adapter/avro/pom.xml @@ -27,13 +27,21 @@ - + org.apache.arrow - arrow-memory + arrow-memory-core ${project.version} + + + org.apache.arrow + arrow-memory-netty + ${project.version} + runtime + + org.apache.arrow diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index eec274f773f..7a749af0ebb 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -27,13 +27,21 @@ - + org.apache.arrow - arrow-memory + arrow-memory-core ${project.version} + + + org.apache.arrow + arrow-memory-netty + ${project.version} + runtime + + org.apache.arrow diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml index b753cf6f897..a21469fe643 100644 --- a/java/adapter/orc/pom.xml +++ b/java/adapter/orc/pom.xml @@ -15,10 +15,16 @@ org.apache.arrow - arrow-memory + arrow-memory-core ${project.version} compile + + org.apache.arrow + arrow-memory-netty + ${project.version} + runtime + org.apache.arrow arrow-vector diff --git a/java/algorithm/pom.xml b/java/algorithm/pom.xml index 774fdd52815..b99e9ad78a9 100644 --- a/java/algorithm/pom.xml +++ b/java/algorithm/pom.xml @@ -35,9 +35,15 @@ org.apache.arrow - arrow-memory + arrow-memory-core ${project.version} + + org.apache.arrow + arrow-memory-netty + ${project.version} + test + io.netty netty-common diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index a2aac999d26..a7882197c78 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -38,9 +38,15 @@ org.apache.arrow - arrow-memory + arrow-memory-core ${project.version} + + org.apache.arrow + arrow-memory-netty + ${project.version} + runtime + io.grpc grpc-netty diff --git a/java/flight/flight-grpc/pom.xml b/java/flight/flight-grpc/pom.xml index 62256777809..acb1fcb344b 100644 --- a/java/flight/flight-grpc/pom.xml +++ b/java/flight/flight-grpc/pom.xml @@ -59,11 +59,17 @@ grpc-stub ${dep.grpc.version} - + + org.apache.arrow + arrow-memory-core + ${project.version} + compile + + org.apache.arrow - arrow-memory + arrow-memory-netty ${project.version} - compile + runtime io.grpc diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml index 5d3fcd0d09c..4f7f793d235 100644 --- a/java/gandiva/pom.xml +++ b/java/gandiva/pom.xml @@ -33,9 +33,15 @@ org.apache.arrow - arrow-memory + arrow-memory-core ${project.version} + + org.apache.arrow + arrow-memory-netty + ${project.version} + test + org.apache.arrow arrow-vector diff --git a/java/memory/memory-core/pom.xml b/java/memory/memory-core/pom.xml new file mode 100644 index 00000000000..8641df444cb --- /dev/null +++ b/java/memory/memory-core/pom.xml @@ -0,0 +1,60 @@ + + + + + arrow-memory + org.apache.arrow + 1.0.0-SNAPSHOT + + 4.0.0 + + arrow-memory-core + + Arrow Memory - Core + Core off-heap memory management libraries for Arrow ValueVectors. + + + + com.google.code.findbugs + jsr305 + + + org.slf4j + slf4j-api + + + org.immutables + value + + + + + + + maven-surefire-plugin + 3.0.0-M3 + + true + true + ${forkCount} + true + + ${project.build.directory} + UTC + + + + + + diff --git a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/Accountant.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/Accountant.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationListener.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/AllocationListener.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationListener.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationManager.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationManager.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationOutcome.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationOutcome.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/AllocationOutcome.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationOutcome.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationOutcomeDetails.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationOutcomeDetails.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/AllocationOutcomeDetails.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationOutcomeDetails.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationReservation.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationReservation.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/AllocationReservation.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationReservation.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocatorClosedException.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ArrowBuf.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/ArrowBuf.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/ArrowBuf.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BoundsChecking.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BoundsChecking.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/BoundsChecking.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/BoundsChecking.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferAllocator.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferLedger.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/BufferLedger.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferManager.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/BufferManager.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferManager.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/CheckAllocator.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/CheckAllocator.java similarity index 95% rename from java/memory/src/main/java/org/apache/arrow/memory/CheckAllocator.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/CheckAllocator.java index 8421057f32a..79b825aa2e8 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/CheckAllocator.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/CheckAllocator.java @@ -78,7 +78,8 @@ private static URL assertOnlyOne(Set urls) { logger.warn("More than one DefaultAllocationManager on classpath. Choosing first found"); } if (urls.isEmpty()) { - throw new RuntimeException("No DefaultAllocationManager found on classpath. Can't allocate Arrow buffers."); + throw new RuntimeException("No DefaultAllocationManager found on classpath. Can't allocate Arrow buffers." + + " Please consider adding arrow-memory-netty or arrow-memory-unsafe as a dependency."); } return urls.iterator().next(); } diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ChildAllocator.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/ChildAllocator.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/ChildAllocator.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java similarity index 86% rename from java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java index 7f4e54f0bd1..15120c252fc 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java @@ -114,10 +114,20 @@ private static AllocationManager.Factory getFactory(String clazzName) { } private static AllocationManager.Factory getUnsafeFactory() { - return getFactory("org.apache.arrow.memory.UnsafeAllocationManager"); + try { + return getFactory("org.apache.arrow.memory.UnsafeAllocationManager"); + } catch (RuntimeException e) { + throw new RuntimeException("Please add arrow-memory-unsafe to your classpath," + + " No DefaultAllocationManager found to instantiate an UnsafeAllocationManager", e); + } } private static AllocationManager.Factory getNettyFactory() { - return getFactory("org.apache.arrow.memory.NettyAllocationManager"); + try { + return getFactory("org.apache.arrow.memory.NettyAllocationManager"); + } catch (RuntimeException e) { + throw new RuntimeException("Please add arrow-memory-netty to your classpath," + + " No DefaultAllocationManager found to instantiate an NettyAllocationManager", e); + } } } diff --git a/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/OutOfMemoryException.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/OutOfMemoryException.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/OutOfMemoryException.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/OutOfMemoryException.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/OwnershipTransferNOOP.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/OwnershipTransferNOOP.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/OwnershipTransferNOOP.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/OwnershipTransferNOOP.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/OwnershipTransferResult.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/OwnershipTransferResult.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/OwnershipTransferResult.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/OwnershipTransferResult.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/README.md b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/README.md similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/README.md rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/README.md diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ReferenceManager.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ReferenceManager.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/ReferenceManager.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/ReferenceManager.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/RootAllocator.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/RootAllocator.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ValueWithKeyIncluded.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ValueWithKeyIncluded.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/ValueWithKeyIncluded.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/ValueWithKeyIncluded.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/package-info.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/package-info.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/package-info.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/package-info.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java similarity index 94% rename from java/memory/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java index 1b4b2d87be2..7ba231b0c20 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java @@ -21,8 +21,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.netty.util.internal.SystemPropertyUtil; - /** * The default rounding policy. That is, if the requested size is within the chunk size, * the rounded size will be the next power of two. Otherwise, the rounded size @@ -45,7 +43,7 @@ public class DefaultRoundingPolicy implements RoundingPolicy { static { - int defaultPageSize = SystemPropertyUtil.getInt("org.apache.memory.allocator.pageSize", 8192); + int defaultPageSize = Integer.getInteger("org.apache.memory.allocator.pageSize", 8192); Throwable pageSizeFallbackCause = null; try { validateAndCalculatePageShifts(defaultPageSize); @@ -54,7 +52,7 @@ public class DefaultRoundingPolicy implements RoundingPolicy { defaultPageSize = 8192; } - int defaultMaxOrder = SystemPropertyUtil.getInt("org.apache.memory.allocator.maxOrder", 11); + int defaultMaxOrder = Integer.getInteger("org.apache.memory.allocator.maxOrder", 11); Throwable maxOrderFallbackCause = null; try { validateAndCalculateChunkSize(defaultPageSize, defaultMaxOrder); diff --git a/java/memory/src/main/java/org/apache/arrow/memory/rounding/RoundingPolicy.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/RoundingPolicy.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/rounding/RoundingPolicy.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/RoundingPolicy.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/rounding/SegmentRoundingPolicy.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/SegmentRoundingPolicy.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/rounding/SegmentRoundingPolicy.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/SegmentRoundingPolicy.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/ArrowBufPointer.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ArrowBufPointer.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/util/ArrowBufPointer.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ArrowBufPointer.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/AssertionUtil.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/AssertionUtil.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/util/AssertionUtil.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/AssertionUtil.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/AutoCloseableLock.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/AutoCloseableLock.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/util/AutoCloseableLock.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/AutoCloseableLock.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/CommonUtil.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/CommonUtil.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/util/CommonUtil.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/CommonUtil.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/LargeMemoryUtil.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/LargeMemoryUtil.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/util/LargeMemoryUtil.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/LargeMemoryUtil.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/StackTrace.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/StackTrace.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/util/StackTrace.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/StackTrace.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/hash/ArrowBufHasher.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/ArrowBufHasher.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/util/hash/ArrowBufHasher.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/ArrowBufHasher.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/hash/MurmurHasher.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/MurmurHasher.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/util/hash/MurmurHasher.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/MurmurHasher.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/util/hash/SimpleHasher.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/SimpleHasher.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/util/hash/SimpleHasher.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/SimpleHasher.java diff --git a/java/memory/src/main/java/org/apache/arrow/util/Preconditions.java b/java/memory/memory-core/src/main/java/org/apache/arrow/util/Preconditions.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/util/Preconditions.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/util/Preconditions.java diff --git a/java/memory/src/main/java/org/apache/arrow/util/VisibleForTesting.java b/java/memory/memory-core/src/main/java/org/apache/arrow/util/VisibleForTesting.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/util/VisibleForTesting.java rename to java/memory/memory-core/src/main/java/org/apache/arrow/util/VisibleForTesting.java diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java new file mode 100644 index 00000000000..e4553104715 --- /dev/null +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import org.apache.arrow.memory.util.MemoryUtil; + +/** + * The default Allocation Manager Factory for a module. + * + * This is only used by tests and contains only a simplistic allocator method. + * + */ +public class DefaultAllocationManagerFactory implements AllocationManager.Factory { + + public static final AllocationManager.Factory FACTORY = new DefaultAllocationManagerFactory(); + private static final ArrowBuf EMPTY = new ArrowBuf(ReferenceManager.NO_OP, + null, + 0, + MemoryUtil.UNSAFE.allocateMemory(0)); + + @Override + public AllocationManager create(BaseAllocator accountingAllocator, long size) { + return new AllocationManager(accountingAllocator) { + private final long allocatedSize = size; + private final long address = MemoryUtil.UNSAFE.allocateMemory(size); + + @Override + public long getSize() { + return allocatedSize; + } + + @Override + protected long memoryAddress() { + return address; + } + + @Override + protected void release0() { + MemoryUtil.UNSAFE.freeMemory(address); + } + }; + } + + @Override + public ArrowBuf empty() { + return EMPTY; + } +} diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestAccountant.java similarity index 100% rename from java/memory/src/test/java/org/apache/arrow/memory/TestAccountant.java rename to java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestAccountant.java diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestAllocationManager.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestAllocationManager.java similarity index 68% rename from java/memory/src/test/java/org/apache/arrow/memory/TestAllocationManager.java rename to java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestAllocationManager.java index 3f68a6f3737..df28424b3a9 100644 --- a/java/memory/src/test/java/org/apache/arrow/memory/TestAllocationManager.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestAllocationManager.java @@ -28,24 +28,11 @@ public class TestAllocationManager { @Test public void testAllocationManagerType() { - // test netty allocation manager type - System.setProperty( - DefaultAllocationManagerOption.ALLOCATION_MANAGER_TYPE_PROPERTY_NAME, "Netty"); - DefaultAllocationManagerOption.AllocationManagerType mgrType = - DefaultAllocationManagerOption.getDefaultAllocationManagerType(); - - assertEquals(DefaultAllocationManagerOption.AllocationManagerType.Netty, mgrType); - - // test unsafe allocation manager type - System.setProperty( - DefaultAllocationManagerOption.ALLOCATION_MANAGER_TYPE_PROPERTY_NAME, "Unsafe"); - mgrType = DefaultAllocationManagerOption.getDefaultAllocationManagerType(); - - assertEquals(DefaultAllocationManagerOption.AllocationManagerType.Unsafe, mgrType); // test unknown allocation manager type System.clearProperty(DefaultAllocationManagerOption.ALLOCATION_MANAGER_TYPE_PROPERTY_NAME); - mgrType = DefaultAllocationManagerOption.getDefaultAllocationManagerType(); + DefaultAllocationManagerOption.AllocationManagerType mgrType = + DefaultAllocationManagerOption.getDefaultAllocationManagerType(); assertEquals(DefaultAllocationManagerOption.AllocationManagerType.Unknown, mgrType); } diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestArrowBuf.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestArrowBuf.java similarity index 100% rename from java/memory/src/test/java/org/apache/arrow/memory/TestArrowBuf.java rename to java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestArrowBuf.java diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java similarity index 100% rename from java/memory/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java rename to java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java similarity index 100% rename from java/memory/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java rename to java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java diff --git a/java/memory/src/test/java/org/apache/arrow/memory/util/TestArrowBufPointer.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestArrowBufPointer.java similarity index 100% rename from java/memory/src/test/java/org/apache/arrow/memory/util/TestArrowBufPointer.java rename to java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestArrowBufPointer.java diff --git a/java/memory/src/test/java/org/apache/arrow/memory/util/TestByteFunctionHelpers.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestByteFunctionHelpers.java similarity index 100% rename from java/memory/src/test/java/org/apache/arrow/memory/util/TestByteFunctionHelpers.java rename to java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestByteFunctionHelpers.java diff --git a/java/memory/src/test/java/org/apache/arrow/memory/util/hash/TestArrowBufHasher.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/hash/TestArrowBufHasher.java similarity index 100% rename from java/memory/src/test/java/org/apache/arrow/memory/util/hash/TestArrowBufHasher.java rename to java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/hash/TestArrowBufHasher.java diff --git a/java/memory/memory-netty/pom.xml b/java/memory/memory-netty/pom.xml new file mode 100644 index 00000000000..c4ee45f78ff --- /dev/null +++ b/java/memory/memory-netty/pom.xml @@ -0,0 +1,103 @@ + + + + + arrow-memory + org.apache.arrow + 1.0.0-SNAPSHOT + + 4.0.0 + + arrow-memory-netty + Arrow Memory - Netty + Netty allocator and utils for allocating memory in Arrow + + + + org.apache.arrow + arrow-memory-core + ${project.version} + + + io.netty + netty-buffer + + + io.netty + netty-common + + + org.slf4j + slf4j-api + + + org.immutables + value + + + + + + + maven-surefire-plugin + 3.0.0-M3 + + true + true + ${forkCount} + true + + ${project.build.directory} + true + 1048576 + UTC + + + + + + + + + + integration-tests + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + ${project.build.directory} + true + UTC + + + + + + + integration-test + verify + + + + + + + + + diff --git a/java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java b/java/memory/memory-netty/src/main/java/io/netty/buffer/ExpandableByteBuf.java similarity index 100% rename from java/memory/src/main/java/io/netty/buffer/ExpandableByteBuf.java rename to java/memory/memory-netty/src/main/java/io/netty/buffer/ExpandableByteBuf.java diff --git a/java/memory/src/main/java/io/netty/buffer/LargeBuffer.java b/java/memory/memory-netty/src/main/java/io/netty/buffer/LargeBuffer.java similarity index 100% rename from java/memory/src/main/java/io/netty/buffer/LargeBuffer.java rename to java/memory/memory-netty/src/main/java/io/netty/buffer/LargeBuffer.java diff --git a/java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java b/java/memory/memory-netty/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java similarity index 100% rename from java/memory/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java rename to java/memory/memory-netty/src/main/java/io/netty/buffer/MutableWrappedByteBuf.java diff --git a/java/memory/src/main/java/io/netty/buffer/NettyArrowBuf.java b/java/memory/memory-netty/src/main/java/io/netty/buffer/NettyArrowBuf.java similarity index 100% rename from java/memory/src/main/java/io/netty/buffer/NettyArrowBuf.java rename to java/memory/memory-netty/src/main/java/io/netty/buffer/NettyArrowBuf.java diff --git a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/java/memory/memory-netty/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java similarity index 100% rename from java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java rename to java/memory/memory-netty/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java diff --git a/java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/java/memory/memory-netty/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java similarity index 100% rename from java/memory/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java rename to java/memory/memory-netty/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java b/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java rename to java/memory/memory-netty/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java diff --git a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java b/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java similarity index 75% rename from java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java rename to java/memory/memory-netty/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java index 052fa24bed6..15651a38e4a 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java +++ b/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java @@ -20,21 +20,19 @@ /** * The default Allocation Manager Factory for a module. * - * This will be split out to the arrow-memory-netty/arrow-memory-unsafe modules - * as part of ARROW-8230. This is currently a placeholder which defaults to Netty. - * */ public class DefaultAllocationManagerFactory implements AllocationManager.Factory { - public static final AllocationManager.Factory FACTORY = new DefaultAllocationManagerFactory(); + public static final AllocationManager.Factory FACTORY = NettyAllocationManager.FACTORY; @Override public AllocationManager create(BaseAllocator accountingAllocator, long size) { - return new NettyAllocationManager(accountingAllocator, size); + return FACTORY.create(accountingAllocator, size); } @Override public ArrowBuf empty() { - return NettyAllocationManager.EMPTY_BUFFER; + return FACTORY.empty(); } + } diff --git a/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java b/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java rename to java/memory/memory-netty/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java diff --git a/java/memory/src/test/java/io/netty/buffer/TestNettyArrowBuf.java b/java/memory/memory-netty/src/test/java/io/netty/buffer/TestNettyArrowBuf.java similarity index 100% rename from java/memory/src/test/java/io/netty/buffer/TestNettyArrowBuf.java rename to java/memory/memory-netty/src/test/java/io/netty/buffer/TestNettyArrowBuf.java diff --git a/java/memory/src/test/java/org/apache/arrow/memory/ITTestLargeArrowBuf.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/ITTestLargeArrowBuf.java similarity index 100% rename from java/memory/src/test/java/org/apache/arrow/memory/ITTestLargeArrowBuf.java rename to java/memory/memory-netty/src/test/java/org/apache/arrow/memory/ITTestLargeArrowBuf.java diff --git a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestAllocationManagerNetty.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestAllocationManagerNetty.java new file mode 100644 index 00000000000..2dbd56480b8 --- /dev/null +++ b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestAllocationManagerNetty.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +/** + * Test cases for {@link AllocationManager}. + */ +public class TestAllocationManagerNetty { + + @Test + public void testAllocationManagerType() { + // test netty allocation manager type + System.setProperty( + DefaultAllocationManagerOption.ALLOCATION_MANAGER_TYPE_PROPERTY_NAME, "Netty"); + DefaultAllocationManagerOption.AllocationManagerType mgrType = + DefaultAllocationManagerOption.getDefaultAllocationManagerType(); + + assertEquals(DefaultAllocationManagerOption.AllocationManagerType.Netty, mgrType); + } +} diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java similarity index 100% rename from java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java rename to java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestEmptyArrowBuf.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEmptyArrowBuf.java similarity index 100% rename from java/memory/src/test/java/org/apache/arrow/memory/TestEmptyArrowBuf.java rename to java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEmptyArrowBuf.java diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestEndianness.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEndianness.java similarity index 100% rename from java/memory/src/test/java/org/apache/arrow/memory/TestEndianness.java rename to java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestEndianness.java diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java similarity index 100% rename from java/memory/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java rename to java/memory/memory-netty/src/test/java/org/apache/arrow/memory/TestNettyAllocationManager.java diff --git a/java/memory/src/test/resources/logback.xml b/java/memory/memory-netty/src/test/resources/logback.xml similarity index 100% rename from java/memory/src/test/resources/logback.xml rename to java/memory/memory-netty/src/test/resources/logback.xml diff --git a/java/memory/memory-unsafe/pom.xml b/java/memory/memory-unsafe/pom.xml new file mode 100644 index 00000000000..85dbaf2b8c1 --- /dev/null +++ b/java/memory/memory-unsafe/pom.xml @@ -0,0 +1,54 @@ + + + + + arrow-memory + org.apache.arrow + 1.0.0-SNAPSHOT + + 4.0.0 + + arrow-memory-unsafe + Arrow Memory - Unsafe + Allocator and utils for allocating memory in Arrow based on sun.misc.Unsafe + + + + + org.apache.arrow + arrow-memory-core + ${project.version} + + + + + + + maven-surefire-plugin + 3.0.0-M3 + + true + true + ${forkCount} + true + + ${project.build.directory} + UTC + + + + + + + diff --git a/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java b/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java new file mode 100644 index 00000000000..3963c1875d0 --- /dev/null +++ b/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +/** + * The default Allocation Manager Factory for a module. + * + */ +public class DefaultAllocationManagerFactory implements AllocationManager.Factory { + + public static final AllocationManager.Factory FACTORY = UnsafeAllocationManager.FACTORY; + + @Override + public AllocationManager create(BaseAllocator accountingAllocator, long size) { + return FACTORY.create(accountingAllocator, size); + } + + @Override + public ArrowBuf empty() { + return UnsafeAllocationManager.FACTORY.empty(); + } +} diff --git a/java/memory/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java b/java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java similarity index 100% rename from java/memory/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java rename to java/memory/memory-unsafe/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java diff --git a/java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/TestAllocationManagerUnsafe.java b/java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/TestAllocationManagerUnsafe.java new file mode 100644 index 00000000000..33abe92e50f --- /dev/null +++ b/java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/TestAllocationManagerUnsafe.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.memory; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +/** + * Test cases for {@link AllocationManager}. + */ +public class TestAllocationManagerUnsafe { + + @Test + public void testAllocationManagerType() { + + // test unsafe allocation manager type + System.setProperty( + DefaultAllocationManagerOption.ALLOCATION_MANAGER_TYPE_PROPERTY_NAME, "Unsafe"); + DefaultAllocationManagerOption.AllocationManagerType mgrType = + DefaultAllocationManagerOption.getDefaultAllocationManagerType(); + + assertEquals(DefaultAllocationManagerOption.AllocationManagerType.Unsafe, mgrType); + + } +} diff --git a/java/memory/src/test/java/org/apache/arrow/memory/TestUnsafeAllocationManager.java b/java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/TestUnsafeAllocationManager.java similarity index 100% rename from java/memory/src/test/java/org/apache/arrow/memory/TestUnsafeAllocationManager.java rename to java/memory/memory-unsafe/src/test/java/org/apache/arrow/memory/TestUnsafeAllocationManager.java diff --git a/java/memory/pom.xml b/java/memory/pom.xml index 06baa33eb54..10e982abb6e 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -18,65 +18,12 @@ arrow-memory Arrow Memory - Core off-heap memory management libraries for Arrow ValueVectors. + pom - - - com.google.code.findbugs - jsr305 - - - io.netty - netty-buffer - - - io.netty - netty-common - - - org.slf4j - slf4j-api - - - org.immutables - value - - + + memory-core + memory-unsafe + memory-netty + - - - - - - - integration-tests - - - - org.apache.maven.plugins - maven-failsafe-plugin - - - ${project.build.directory} - true - false - UTC - - - - - - - integration-test - verify - - - - - - - - diff --git a/java/performance/pom.xml b/java/performance/pom.xml index b2cdbdd01df..d42ed18d5bc 100644 --- a/java/performance/pom.xml +++ b/java/performance/pom.xml @@ -53,9 +53,15 @@ org.apache.arrow - arrow-memory + arrow-memory-core ${project.version} + + org.apache.arrow + arrow-memory-netty + ${project.version} + test + org.apache.avro avro diff --git a/java/tools/pom.xml b/java/tools/pom.xml index f5248b8fb36..67faef5c7c0 100644 --- a/java/tools/pom.xml +++ b/java/tools/pom.xml @@ -23,7 +23,7 @@ org.apache.arrow - arrow-memory + arrow-memory-core ${project.version} @@ -59,7 +59,12 @@ org.slf4j slf4j-api - + + org.apache.arrow + arrow-memory-netty + ${project.version} + runtime + org.apache.arrow arrow-vector diff --git a/java/vector/pom.xml b/java/vector/pom.xml index 3056ce89e37..04fc4bd1b5d 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -30,7 +30,7 @@ org.apache.arrow - arrow-memory + arrow-memory-core ${project.version} @@ -51,8 +51,9 @@ 1.10 - io.netty - netty-buffer + org.apache.arrow + arrow-memory-netty + ${project.version} test diff --git a/java/vector/src/main/codegen/includes/vv_imports.ftl b/java/vector/src/main/codegen/includes/vv_imports.ftl index de437b4b081..2acd4361553 100644 --- a/java/vector/src/main/codegen/includes/vv_imports.ftl +++ b/java/vector/src/main/codegen/includes/vv_imports.ftl @@ -20,8 +20,6 @@ import static org.apache.arrow.util.Preconditions.checkState; import com.google.flatbuffers.FlatBufferBuilder; -import io.netty.buffer.*; - import org.apache.arrow.memory.*; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.types.Types;