From b56393f96a1b31bf765a9d996aa954ceb11af7f9 Mon Sep 17 00:00:00 2001 From: Yurui Zhou Date: Tue, 23 Apr 2019 15:46:25 +0800 Subject: [PATCH 1/4] add unsafe read/write access method to ArrowBuf --- java/adapter/orc/pom.xml | 46 ++ .../apache/arrow/adapter/orc/OrcReader.java | 69 +++ .../main/java/io/netty/buffer/ArrowBuf.java | 542 +++++++++++++++++- java/pom.xml | 1 + 4 files changed, 630 insertions(+), 28 deletions(-) create mode 100644 java/adapter/orc/pom.xml create mode 100644 java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReader.java diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml new file mode 100644 index 00000000000..c58747044aa --- /dev/null +++ b/java/adapter/orc/pom.xml @@ -0,0 +1,46 @@ + + + + arrow-java-root + org.apache.arrow + 0.13.0-SNAPSHOT + ../../pom.xml + + 4.0.0 + + arrow-orc + Arrow Orc Adapter + http://maven.apache.org + + + + + org.apache.arrow + arrow-memory + ${project.version} + + + + + org.apache.arrow + arrow-vector + ${project.version} + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + UTC + + + + + + \ No newline at end of file diff --git a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReader.java b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReader.java new file mode 100644 index 00000000000..1977cd4437a --- /dev/null +++ b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReader.java @@ -0,0 +1,69 @@ +package org.apache.arrow.adapter.orc; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch; +import org.apache.arrow.vector.types.pojo.Schema; + +import java.io.IOException; + +public class OrcReader extends ArrowReader { + + protected OrcReader(BufferAllocator allocator) { + super(allocator); + } + + /** + * Load the next ArrowRecordBatch to the vector schema root if available. + * + * @return true if a batch was read, false on EOS + * @throws IOException on error + */ + @Override + public boolean loadNextBatch() throws IOException { + return false; + } + + /** + * Return the number of bytes read from the ReadChannel. + * + * @return number of bytes read + */ + @Override + public long bytesRead() { + return 0; + } + + /** + * Close the underlying read source. + * + * @throws IOException on error + */ + @Override + protected void closeReadSource() throws IOException { + + } + + /** + * Read the Schema from the source, will be invoked at the beginning the initialization. + * + * @return the read Schema + * @throws IOException on error + */ + @Override + protected Schema readSchema() throws IOException { + return null; + } + + /** + * Read a dictionary batch from the source, will be invoked after the schema has been read and + * called N times, where N is the number of dictionaries indicated by the schema Fields. + * + * @return the read ArrowDictionaryBatch + * @throws IOException on error + */ + @Override + protected ArrowDictionaryBatch readDictionary() throws IOException { + return null; + } +} diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java index bb6a940847f..8171fed4b9a 100644 --- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java +++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java @@ -531,206 +531,659 @@ public ByteBuf touch(Object hint) { return this; } + /** + * Get long value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 8 byte long value + */ @Override public long getLong(int index) { chk(index, 8); + return getLongUnsafe(index); + } + + /** + * Get long value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 8 byte long value + */ + public long getLongUnsafe(int index) { final long v = PlatformDependent.getLong(addr(index)); return v; } + /** + * Get float value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 4 byte float value + */ @Override public float getFloat(int index) { - return Float.intBitsToFloat(getInt(index)); + chk(index, 4); + return getFloatUnsafe(index); } /** - * Gets a 64-bit long integer at the specified absolute {@code index} in - * this buffer in Big Endian Byte Order. + * Get float value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 4 byte float value + */ + public float getFloatUnsafe(int index) { + return Float.intBitsToFloat(getIntUnsafe(index)); + } + + /** + * Get long value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 8 byte long value */ @Override public long getLongLE(int index) { chk(index, 8); + return getLongLEUnsafe(index); + } + + /** + * Get long value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 8 byte long value + */ + public long getLongLEUnsafe(int index) { final long v = PlatformDependent.getLong(addr(index)); return Long.reverseBytes(v); } + /** + * Get double value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 8 byte double value + */ @Override public double getDouble(int index) { - return Double.longBitsToDouble(getLong(index)); + chk(index, 8); + return getDoubleUnsafe(index); + } + + /** + * Get double value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 8 byte double value + */ + public double getDoubleUnsafe(int index) { + return Double.longBitsToDouble(getLongUnsafe(index)); } + /** + * Get char value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 2 byte char value + */ @Override public char getChar(int index) { - return (char) getShort(index); + chk(index, 2); + return getCharUnsafe(index); } + /** + * Get char value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 2 byte char value + */ + public char getCharUnsafe(int index) { + return (char) getShortUnsafe(index); + } + + /** + * Get unsigned int value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 4 byte unsigned int value + */ @Override public long getUnsignedInt(int index) { - return getInt(index) & 0xFFFFFFFFL; + chk(index, 4); + return getUnsignedIntUnsafe(index); + } + + /** + * Get unsigned int value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 4 byte unsigned int value + */ + public long getUnsignedIntUnsafe(int index) { + return getIntUnsafe(index) & 0xFFFFFFFFL; } + /** + * Get int value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 4 byte int value + */ @Override public int getInt(int index) { chk(index, 4); + return getIntUnsafe(index); + } + + /** + * Get int value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 4 byte int value + */ + public int getIntUnsafe(int index) { final int v = PlatformDependent.getInt(addr(index)); return v; } /** - * Gets a 32-bit integer at the specified absolute {@code index} in - * this buffer in Big Endian Byte Order. + * Get int value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 4 byte int value */ @Override public int getIntLE(int index) { chk(index, 4); + return getIntLEUnsafe(index); + } + + /** + * Get int value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 4 byte int value + */ + public int getIntLEUnsafe(int index) { final int v = PlatformDependent.getInt(addr(index)); return Integer.reverseBytes(v); } + /** + * Get unsigned short value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 2 byte unsigned short value + */ @Override public int getUnsignedShort(int index) { - return getShort(index) & 0xFFFF; + chk(index, 2); + return getUnsignedShortUnsafe(index); } + /** + * Get unsigned short value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 2 byte unsigned short value + */ + public int getUnsignedShortUnsafe(int index) { + return getShortUnsafe(index) & 0xFFFF; + } + + /** + * Get short value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 2 byte short value + */ @Override public short getShort(int index) { chk(index, 2); + return getShortUnsafe(index); + } + + /** + * Get short value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 2 byte short value + */ + public short getShortUnsafe(int index) { final short v = PlatformDependent.getShort(addr(index)); return v; } /** - * Gets a 16-bit short integer at the specified absolute {@code index} in - * this buffer in Big Endian Byte Order. + * Get short value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 2 byte short value */ @Override public short getShortLE(int index) { + chk(index, 2); + return getShortLEUnsafe(index); + } + + /** + * Get short value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 2 byte short value + */ + public short getShortLEUnsafe(int index) { final short v = PlatformDependent.getShort(addr(index)); return Short.reverseBytes(v); } /** - * Gets an unsigned 24-bit medium integer at the specified absolute - * {@code index} in this buffer. + * Get medium value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 3 byte medium value */ @Override public int getUnsignedMedium(int index) { chk(index, 3); + return getUnsignedMediumUnsafe(index); + } + + /** + * Get medium value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 3 byte medium value + */ + public int getUnsignedMediumUnsafe(int index) { final long addr = addr(index); return (PlatformDependent.getByte(addr) & 0xff) << 16 | - (PlatformDependent.getShort(addr + 1) & 0xffff); + (PlatformDependent.getShort(addr + 1) & 0xffff); } /** - * Gets an unsigned 24-bit medium integer at the specified absolute {@code index} in - * this buffer in Big Endian Byte Order. + * Get medium value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 3 byte medium value */ @Override public int getUnsignedMediumLE(int index) { + chk(index, 3); + return getUnsignedMediumLEUnsafe(index); + } + + /** + * Get medium value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @return 3 byte medium value + */ + public int getUnsignedMediumLEUnsafe(int index) { chk(index, 3); final long addr = addr(index); return (PlatformDependent.getByte(addr) & 0xff) | - (Short.reverseBytes(PlatformDependent.getShort(addr + 1)) & 0xffff) << 8; + (Short.reverseBytes(PlatformDependent.getShort(addr + 1)) & 0xffff) << 8; } + /** + * Set short value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 2 byte short value + * @return this ArrowBuf + */ @Override public ArrowBuf setShort(int index, int value) { chk(index, 2); + return setShortUnsafe(index, value); + } + + /** + * Set short value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 2 byte short value + * @return this ArrowBuf + */ + public ArrowBuf setShortUnsafe(int index, int value) { PlatformDependent.putShort(addr(index), (short) value); return this; } /** - * Sets the specified 16-bit short integer at the specified absolute {@code index} - * in this buffer with Big Endian byte order. + * Set short value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 2 byte short value + * @return this ArrowBuf */ @Override public ByteBuf setShortLE(int index, int value) { chk(index, 2); + return setShortLEUnsafe(index, value); + } + + /** + * Set short value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 2 byte short value + * @return this ArrowBuf + */ + public ByteBuf setShortLEUnsafe(int index, int value) { PlatformDependent.putShort(addr(index), Short.reverseBytes((short) value)); return this; } /** - * Sets the specified 24-bit medium integer at the specified absolute - * {@code index} in this buffer. + * Set medium value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 3 byte medium value + * @return this ArrowBuf */ @Override public ByteBuf setMedium(int index, int value) { chk(index, 3); + return setMediumUnsafe(index, value); + } + + /** + * Set medium value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 3 byte medium value + * @return this ArrowBuf + */ + public ByteBuf setMediumUnsafe(int index, int value) { final long addr = addr(index); PlatformDependent.putByte(addr, (byte) (value >>> 16)); PlatformDependent.putShort(addr + 1, (short) value); return this; } - /** - * Sets the specified 24-bit medium integer at the specified absolute {@code index} - * in this buffer with Big Endian byte order. + * Set medium value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 3 byte medium value + * @return this ArrowBuf */ @Override public ByteBuf setMediumLE(int index, int value) { chk(index, 3); + return setMediumLEUnsafe(index, value); + } + + /** + * Set medium value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 3 byte medium value + * @return this ArrowBuf + */ + public ByteBuf setMediumLEUnsafe(int index, int value) { final long addr = addr(index); PlatformDependent.putByte(addr, (byte) value); PlatformDependent.putShort(addr + 1, Short.reverseBytes((short) (value >>> 8))); return this; } + /** + * Set int value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 4 byte int value + * @return this ArrowBuf + */ @Override public ArrowBuf setInt(int index, int value) { chk(index, 4); + return setIntUnsafe(index, value); + } + + /** + * Set int value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 4 byte int value + * @return this ArrowBuf + */ + public ArrowBuf setIntUnsafe(int index, int value) { PlatformDependent.putInt(addr(index), value); return this; } /** - * Sets the specified 32-bit integer at the specified absolute {@code index} - * in this buffer with Big Endian byte order. + * Set int value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 4 byte int value + * @return this ArrowBuf */ @Override public ByteBuf setIntLE(int index, int value) { chk(index, 4); + return setIntLEUnsafe(index, value); + } + + /** + * Set int value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 4 byte int value + * @return this ArrowBuf + */ + public ByteBuf setIntLEUnsafe(int index, int value) { PlatformDependent.putInt(addr(index), Integer.reverseBytes(value)); return this; } + /** + * Set long value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 8 byte long value + * @return this ArrowBuf + */ @Override public ArrowBuf setLong(int index, long value) { chk(index, 8); + return setLongUnsafe(index, value); + } + + /** + * Set long value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 8 byte long value + * @return this ArrowBuf + */ + public ArrowBuf setLongUnsafe(int index, long value) { PlatformDependent.putLong(addr(index), value); return this; } /** - * Sets the specified 64-bit long integer at the specified absolute {@code index} - * in this buffer with Big Endian byte order. + * Set long value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 8 byte long value + * @return this ArrowBuf */ @Override public ByteBuf setLongLE(int index, long value) { chk(index, 8); + return setLongLEUnsafe(index, value); + } + + /** + * Set long value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * in Big Endian Byte Order without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 8 byte long value + * @return this ArrowBuf + */ + public ByteBuf setLongLEUnsafe(int index, long value) { PlatformDependent.putLong(addr(index), Long.reverseBytes(value)); return this; } + /** + * Set char value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 2 byte char value + * @return this ArrowBuf + */ @Override public ArrowBuf setChar(int index, int value) { chk(index, 2); + return setCharUnsafe(index, value); + } + + /** + * Set char value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 2 byte char value + * @return this ArrowBuf + */ + public ArrowBuf setCharUnsafe(int index, int value) { PlatformDependent.putShort(addr(index), (short) value); return this; } + /** + * Set float value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 4 byte float value + * @return this ArrowBuf + */ @Override public ArrowBuf setFloat(int index, float value) { chk(index, 4); + return setFloatUnsafe(index, value); + } + + /** + * Set float value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 4 byte float value + * @return this ArrowBuf + */ + public ArrowBuf setFloatUnsafe(int index, float value) { PlatformDependent.putInt(addr(index), Float.floatToRawIntBits(value)); return this; } + /** + * Set double value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 8 byte double value + * @return this ArrowBuf + */ @Override public ArrowBuf setDouble(int index, double value) { chk(index, 8); + return setDoubleUnsafe(index, value); + } + + /** + * Set double value stored at designated index in the + * underlying memory chunk this ArrowBuf has access to + * without boundary check. + * @param index index (0 based relative to this ArrowBuf) + * where the value will be read from + * @param value 8 byte double value + * @return this ArrowBuf + */ + public ArrowBuf setDoubleUnsafe(int index, double value) { PlatformDependent.putLong(addr(index), Double.doubleToRawLongBits(value)); return this; } @@ -738,6 +1191,11 @@ public ArrowBuf setDouble(int index, double value) { @Override public ArrowBuf writeShort(int value) { ensure(2); + return writeShortUnsafe(value); + } + + + public ArrowBuf writeShortUnsafe(int value) { PlatformDependent.putShort(addr(writerIndex), (short) value); writerIndex += 2; return this; @@ -746,6 +1204,10 @@ public ArrowBuf writeShort(int value) { @Override public ArrowBuf writeInt(int value) { ensure(4); + return writeIntUnsafe(value); + } + + public ArrowBuf writeIntUnsafe(int value) { PlatformDependent.putInt(addr(writerIndex), value); writerIndex += 4; return this; @@ -754,6 +1216,10 @@ public ArrowBuf writeInt(int value) { @Override public ArrowBuf writeLong(long value) { ensure(8); + return writeLongUnsafe(value); + } + + public ArrowBuf writeLongUnsafe(long value) { PlatformDependent.putLong(addr(writerIndex), value); writerIndex += 8; return this; @@ -762,6 +1228,10 @@ public ArrowBuf writeLong(long value) { @Override public ArrowBuf writeChar(int value) { ensure(2); + return writeCharUnsafe(value); + } + + public ArrowBuf writeCharUnsafe(int value) { PlatformDependent.putShort(addr(writerIndex), (short) value); writerIndex += 2; return this; @@ -770,6 +1240,10 @@ public ArrowBuf writeChar(int value) { @Override public ArrowBuf writeFloat(float value) { ensure(4); + return writeFloatUnsafe(value); + } + + public ArrowBuf writeFloatUnsafe(float value) { PlatformDependent.putInt(addr(writerIndex), Float.floatToRawIntBits(value)); writerIndex += 4; return this; @@ -778,6 +1252,10 @@ public ArrowBuf writeFloat(float value) { @Override public ArrowBuf writeDouble(double value) { ensure(8); + return writeDoubleUnsafe(value); + } + + public ArrowBuf writeDoubleUnsafe(double value) { PlatformDependent.putLong(addr(writerIndex), Double.doubleToRawLongBits(value)); writerIndex += 8; return this; @@ -798,12 +1276,20 @@ public ArrowBuf getBytes(int index, ByteBuffer dst) { @Override public ArrowBuf setByte(int index, int value) { chk(index, 1); + return setByteUnsafe(index, value); + } + + public ArrowBuf setByteUnsafe(int index, int value) { PlatformDependent.putByte(addr(index), (byte) value); return this; } public void setByte(int index, byte b) { chk(index, 1); + setByteUnsafe(index, b); + } + + public void setByteUnsafe(int index, byte b) { PlatformDependent.putByte(addr(index), b); } diff --git a/java/pom.xml b/java/pom.xml index 17f91bd649a..6c836a0c478 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -652,6 +652,7 @@ adapter/jdbc plasma flight + adapter/orc From 87f22939ea99392412d33b32e0dc7f22e57f11eb Mon Sep 17 00:00:00 2001 From: Yurui Zhou Date: Tue, 23 Apr 2019 15:51:02 +0800 Subject: [PATCH 2/4] remove unrelated changes This reverts commit b56393f96a1b31bf765a9d996aa954ceb11af7f9. --- java/adapter/orc/pom.xml | 46 ------------- .../apache/arrow/adapter/orc/OrcReader.java | 69 ------------------- java/pom.xml | 1 - 3 files changed, 116 deletions(-) delete mode 100644 java/adapter/orc/pom.xml delete mode 100644 java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReader.java diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml deleted file mode 100644 index c58747044aa..00000000000 --- a/java/adapter/orc/pom.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - arrow-java-root - org.apache.arrow - 0.13.0-SNAPSHOT - ../../pom.xml - - 4.0.0 - - arrow-orc - Arrow Orc Adapter - http://maven.apache.org - - - - - org.apache.arrow - arrow-memory - ${project.version} - - - - - org.apache.arrow - arrow-vector - ${project.version} - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - UTC - - - - - - \ No newline at end of file diff --git a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReader.java b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReader.java deleted file mode 100644 index 1977cd4437a..00000000000 --- a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReader.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.apache.arrow.adapter.orc; - -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.ipc.ArrowReader; -import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch; -import org.apache.arrow.vector.types.pojo.Schema; - -import java.io.IOException; - -public class OrcReader extends ArrowReader { - - protected OrcReader(BufferAllocator allocator) { - super(allocator); - } - - /** - * Load the next ArrowRecordBatch to the vector schema root if available. - * - * @return true if a batch was read, false on EOS - * @throws IOException on error - */ - @Override - public boolean loadNextBatch() throws IOException { - return false; - } - - /** - * Return the number of bytes read from the ReadChannel. - * - * @return number of bytes read - */ - @Override - public long bytesRead() { - return 0; - } - - /** - * Close the underlying read source. - * - * @throws IOException on error - */ - @Override - protected void closeReadSource() throws IOException { - - } - - /** - * Read the Schema from the source, will be invoked at the beginning the initialization. - * - * @return the read Schema - * @throws IOException on error - */ - @Override - protected Schema readSchema() throws IOException { - return null; - } - - /** - * Read a dictionary batch from the source, will be invoked after the schema has been read and - * called N times, where N is the number of dictionaries indicated by the schema Fields. - * - * @return the read ArrowDictionaryBatch - * @throws IOException on error - */ - @Override - protected ArrowDictionaryBatch readDictionary() throws IOException { - return null; - } -} diff --git a/java/pom.xml b/java/pom.xml index 6c836a0c478..17f91bd649a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -652,7 +652,6 @@ adapter/jdbc plasma flight - adapter/orc From e67f248e1b315a0e3267cde3fa47c10264cc1c57 Mon Sep 17 00:00:00 2001 From: Yurui Zhou Date: Tue, 23 Apr 2019 15:59:35 +0800 Subject: [PATCH 3/4] remove unnecessary boudary check --- java/memory/src/main/java/io/netty/buffer/ArrowBuf.java | 1 - 1 file changed, 1 deletion(-) diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java index 8171fed4b9a..5a656d6eb28 100644 --- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java +++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java @@ -865,7 +865,6 @@ public int getUnsignedMediumLE(int index) { * @return 3 byte medium value */ public int getUnsignedMediumLEUnsafe(int index) { - chk(index, 3); final long addr = addr(index); return (PlatformDependent.getByte(addr) & 0xff) | (Short.reverseBytes(PlatformDependent.getShort(addr + 1)) & 0xffff) << 8; From 17bfce36f25c7b03a2ddaa75e44e8116c3fea003 Mon Sep 17 00:00:00 2001 From: Yurui Zhou Date: Tue, 23 Apr 2019 17:40:51 +0800 Subject: [PATCH 4/4] fix travis build error --- java/memory/src/main/java/io/netty/buffer/ArrowBuf.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java index 5a656d6eb28..5680900ad4a 100644 --- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java +++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java @@ -585,7 +585,7 @@ public float getFloatUnsafe(int index) { /** * Get long value stored at designated index in the * underlying memory chunk this ArrowBuf has access to - * in Big Endian Byte Order + * in Big Endian Byte Order. * @param index index (0 based relative to this ArrowBuf) * where the value will be read from * @return 8 byte long value