From 5ba874f212b2cde7a634224d112b79b5ebb4e616 Mon Sep 17 00:00:00 2001 From: liyafan82 Date: Tue, 25 Feb 2020 21:41:46 +0800 Subject: [PATCH 1/2] [ARROW-7935][Java] Remove Netty dependency for BufferAllocator and ReferenceManager --- .../org/apache/arrow/memory/ArrowByteBufAllocator.java | 3 +++ .../java/org/apache/arrow/memory/BaseAllocator.java | 10 ---------- .../java/org/apache/arrow/memory/BufferAllocator.java | 3 +++ .../java/org/apache/arrow/memory/BufferLedger.java | 3 ++- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java index b6296cfc723..96fe7eb1c51 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/ArrowByteBufAllocator.java @@ -30,7 +30,10 @@ * the signature and the fact that this Allocator returns ExpandableByteBufs which enable * otherwise non-expandable * ArrowBufs to be expandable. + * + * @deprecated This class may be removed in a future release. */ +@Deprecated public class ArrowByteBufAllocator extends AbstractByteBufAllocator { private static final int DEFAULT_BUFFER_SIZE = 4096; diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java index 230a6580f0e..379a753a3e2 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java @@ -33,7 +33,6 @@ import org.immutables.value.Value; import io.netty.buffer.ArrowBuf; -import io.netty.util.internal.OutOfDirectMemoryError; /** * A base-class that implements all functionality of {@linkplain BufferAllocator}s. @@ -319,15 +318,6 @@ public ArrowBuf buffer(final long initialRequestSize, BufferManager manager) { listener.onAllocation(actualRequestSize); return buffer; } catch (OutOfMemoryError e) { - /* - * OutOfDirectMemoryError is thrown by Netty when we exceed the direct memory limit defined by - * -XX:MaxDirectMemorySize. OutOfMemoryError with "Direct buffer memory" message is thrown by - * java.nio.Bits when we exceed the direct memory limit. This should never be hit in practice - * as Netty is expected to throw an OutOfDirectMemoryError first. - */ - if (e instanceof OutOfDirectMemoryError || "Direct buffer memory".equals(e.getMessage())) { - throw new OutOfMemoryException(e); - } throw e; } finally { if (!success) { diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java index 2b5d0816499..d1f765cc3fe 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferAllocator.java @@ -56,7 +56,10 @@ public interface BufferAllocator extends AutoCloseable { * Returns the allocator this allocator falls back to when it needs more memory. * * @return the underlying allocator used by this allocator + * + * @deprecated This method may be removed in a future release. */ + @Deprecated ByteBufAllocator getAsByteBufAllocator(); /** diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BufferLedger.java b/java/memory/src/main/java/org/apache/arrow/memory/BufferLedger.java index a7ef51d4524..d1091f1f471 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/BufferLedger.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/BufferLedger.java @@ -526,7 +526,7 @@ void print(StringBuilder sb, int indent, BaseAllocator.Verbosity verbosity) { /** * Returns the underlying {@link UnsafeDirectLittleEndian} instance used by this BufferLedger. * - * @deprecated Use #unwrap(UnsafeDirectLittleEndian.class) instead. + * @deprecated This method may be removed in a future release. */ @Deprecated public UnsafeDirectLittleEndian getUnderlying() { @@ -554,6 +554,7 @@ public T unwrap(Class clazz) { return clazz.cast(allocationManager); } + // TODO: remove this in a future release. if (clazz == UnsafeDirectLittleEndian.class) { Preconditions.checkState(allocationManager instanceof NettyAllocationManager, "Underlying memory was not allocated by Netty"); From 58e574949e0e85129a7f32d45045c107272d49b3 Mon Sep 17 00:00:00 2001 From: liyafan82 Date: Wed, 4 Mar 2020 21:57:23 +0800 Subject: [PATCH 2/2] [ARROW-7935][Java] Move Netty related exception handling to PooledByteBufAllocatorL --- .../io/netty/buffer/PooledByteBufAllocatorL.java | 13 +++++++++++-- .../java/org/apache/arrow/memory/BaseAllocator.java | 1 - 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java index 861f9ae41b6..d0a5a9945ce 100644 --- a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java +++ b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java @@ -26,6 +26,7 @@ import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.memory.util.LargeMemoryUtil; +import io.netty.util.internal.OutOfDirectMemoryError; import io.netty.util.internal.StringUtil; /** @@ -56,9 +57,17 @@ public UnsafeDirectLittleEndian allocate(long size) { try { return allocator.directBuffer(LargeMemoryUtil.checkedCastToInt(size), Integer.MAX_VALUE); } catch (OutOfMemoryError e) { - throw new OutOfMemoryException("Failure allocating buffer.", e); + /* + * OutOfDirectMemoryError is thrown by Netty when we exceed the direct memory limit defined by + * -XX:MaxDirectMemorySize. OutOfMemoryError with "Direct buffer memory" message is thrown by + * java.nio.Bits when we exceed the direct memory limit. This should never be hit in practice + * as Netty is expected to throw an OutOfDirectMemoryError first. + */ + if (e instanceof OutOfDirectMemoryError || "Direct buffer memory".equals(e.getMessage())) { + throw new OutOfMemoryException("Failure allocating buffer.", e); + } + throw e; } - } public int getChunkSize() { diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java index 379a753a3e2..84a8f63e591 100644 --- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java +++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java @@ -324,7 +324,6 @@ public ArrowBuf buffer(final long initialRequestSize, BufferManager manager) { releaseBytes(actualRequestSize); } } - } /**