diff --git a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkPinotDataBuffer.java b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkPinotDataBuffer.java index bc05f64b6d41..9d01e92a1ea5 100644 --- a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkPinotDataBuffer.java +++ b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkPinotDataBuffer.java @@ -23,7 +23,6 @@ import java.util.Random; import java.util.concurrent.TimeUnit; import org.apache.pinot.segment.spi.memory.ByteBufferPinotBufferFactory; -import org.apache.pinot.segment.spi.memory.LArrayPinotBufferFactory; import org.apache.pinot.segment.spi.memory.PinotDataBuffer; import org.apache.pinot.segment.spi.memory.SmallWithFallbackPinotBufferFactory; import org.apache.pinot.segment.spi.memory.unsafe.UnsafePinotBufferFactory; @@ -61,7 +60,7 @@ public class BenchmarkPinotDataBuffer { @Param({"1", "32", "1024"}) private int _valueLength; - @Param({"bytebuffer", "larray", "unsafe", "wrapper+unsafe"}) + @Param({"bytebuffer", "unsafe", "wrapper+unsafe"}) private String _bufferLibrary; private byte[] _bytes; private PinotDataBuffer _buffer; @@ -96,16 +95,9 @@ public void setupBufferLibrary() { case "bytebuffer": PinotDataBuffer.useFactory(new ByteBufferPinotBufferFactory()); break; - case "larray": - PinotDataBuffer.useFactory(new LArrayPinotBufferFactory()); - break; case "unsafe": PinotDataBuffer.useFactory(new UnsafePinotBufferFactory()); break; - case "wrapper+larray": - PinotDataBuffer.useFactory(new SmallWithFallbackPinotBufferFactory( - new ByteBufferPinotBufferFactory(), new LArrayPinotBufferFactory())); - break; case "wrapper+unsafe": PinotDataBuffer.useFactory(new SmallWithFallbackPinotBufferFactory( new ByteBufferPinotBufferFactory(), new UnsafePinotBufferFactory())); diff --git a/pinot-segment-spi/pom.xml b/pinot-segment-spi/pom.xml index 273061e4d572..1c0dc9e1760c 100644 --- a/pinot-segment-spi/pom.xml +++ b/pinot-segment-spi/pom.xml @@ -56,10 +56,6 @@ org.locationtech.jts jts-core - - org.xerial.larray - larray-mmap - net.openhft chronicle-core diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/BasePinotLBuffer.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/BasePinotLBuffer.java deleted file mode 100644 index 9ba50911d226..000000000000 --- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/BasePinotLBuffer.java +++ /dev/null @@ -1,148 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pinot.segment.spi.memory; - -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.channels.FileChannel; -import javax.annotation.concurrent.ThreadSafe; -import xerial.larray.buffer.LBuffer; -import xerial.larray.buffer.LBufferAPI; -import xerial.larray.mmap.MMapBuffer; - - -@ThreadSafe -public abstract class BasePinotLBuffer extends PinotDataBuffer { - protected final LBufferAPI _buffer; - private final boolean _flushable; - - protected BasePinotLBuffer(LBufferAPI buffer, boolean closeable, boolean flushable) { - super(closeable); - _buffer = buffer; - _flushable = flushable; - } - - @Override - public byte getByte(int offset) { - return _buffer.getByte(offset); - } - - @Override - public byte getByte(long offset) { - return _buffer.getByte(offset); - } - - @Override - public void putByte(int offset, byte value) { - _buffer.putByte(offset, value); - } - - @Override - public void putByte(long offset, byte value) { - _buffer.putByte(offset, value); - } - - @Override - public void copyTo(long offset, DataBuffer buffer, long destOffset, long size) { - if (buffer instanceof BasePinotLBuffer) { - _buffer.copyTo(offset, ((BasePinotLBuffer) buffer)._buffer, destOffset, size); - } else { - super.copyTo(offset, buffer, destOffset, size); - } - } - - @Override - public void readFrom(long offset, byte[] buffer, int srcOffset, int size) { - if (size <= BULK_BYTES_PROCESSING_THRESHOLD) { - int end = srcOffset + size; - for (int i = srcOffset; i < end; i++) { - _buffer.putByte(offset++, buffer[i]); - } - } else { - _buffer.toDirectByteBuffer(offset, size).put(buffer, srcOffset, size); - } - } - - @Override - public void readFrom(long offset, ByteBuffer buffer) { - _buffer.toDirectByteBuffer(offset, buffer.remaining()).put(buffer); - } - - @Override - public void readFrom(long offset, File file, long srcOffset, long size) - throws IOException { - try (FileChannel fileChannel = new RandomAccessFile(file, "r").getChannel()) { - if (size <= Integer.MAX_VALUE) { - fileChannel.read(_buffer.toDirectByteBuffer(offset, (int) size), srcOffset); - } else { - while (size > Integer.MAX_VALUE) { - fileChannel.read(_buffer.toDirectByteBuffer(offset, Integer.MAX_VALUE), srcOffset); - offset += Integer.MAX_VALUE; - srcOffset += Integer.MAX_VALUE; - size -= Integer.MAX_VALUE; - } - fileChannel.read(_buffer.toDirectByteBuffer(offset, (int) size), srcOffset); - } - } - } - - @Override - public long size() { - if (_buffer instanceof MMapBuffer) { - // Workaround to handle cases where offset is not page-aligned - return _buffer.m.address() + _buffer.m.size() - _buffer.address(); - } else { - return _buffer.size(); - } - } - - @Override - public PinotDataBuffer view(long start, long end, ByteOrder byteOrder) { - if (byteOrder == NATIVE_ORDER) { - return new PinotNativeOrderLBuffer(_buffer.view(start, end), false, false); - } else { - return new PinotNonNativeOrderLBuffer(_buffer.view(start, end), false, false); - } - } - - @Override - public ByteBuffer toDirectByteBuffer(long offset, int size, ByteOrder byteOrder) { - return _buffer.toDirectByteBuffer(offset, size).order(byteOrder); - } - - @Override - public void flush() { - if (_flushable) { - ((MMapBuffer) _buffer).flush(); - } - } - - @Override - public void release() - throws IOException { - if (_buffer instanceof LBuffer) { - _buffer.release(); - } else if (_buffer instanceof MMapBuffer) { - ((MMapBuffer) _buffer).close(); - } - } -} diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/LArrayPinotBufferFactory.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/LArrayPinotBufferFactory.java deleted file mode 100644 index d547c88acc87..000000000000 --- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/LArrayPinotBufferFactory.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pinot.segment.spi.memory; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteOrder; - - -public class LArrayPinotBufferFactory implements PinotBufferFactory { - @Override - public PinotDataBuffer allocateDirect(long size, ByteOrder byteOrder) { - if (byteOrder == ByteOrder.nativeOrder()) { - return PinotNativeOrderLBuffer.allocateDirect(size); - } else { - return PinotNonNativeOrderLBuffer.allocateDirect(size); - } - } - - @Override - public PinotDataBuffer mapFile(File file, boolean readOnly, long offset, long size, ByteOrder byteOrder) - throws IOException { - if (byteOrder == ByteOrder.nativeOrder()) { - return PinotNativeOrderLBuffer.mapFile(file, readOnly, offset, size); - } else { - return PinotNonNativeOrderLBuffer.mapFile(file, readOnly, offset, size); - } - } -} diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotNativeOrderLBuffer.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotNativeOrderLBuffer.java deleted file mode 100644 index 5747a3d32677..000000000000 --- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotNativeOrderLBuffer.java +++ /dev/null @@ -1,183 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pinot.segment.spi.memory; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteOrder; -import javax.annotation.concurrent.ThreadSafe; -import xerial.larray.buffer.LBuffer; -import xerial.larray.buffer.LBufferAPI; -import xerial.larray.mmap.MMapBuffer; -import xerial.larray.mmap.MMapMode; - - -@ThreadSafe -public class PinotNativeOrderLBuffer extends BasePinotLBuffer { - - static PinotNativeOrderLBuffer allocateDirect(long size) { - LBufferAPI buffer = new LBuffer(size); - return new PinotNativeOrderLBuffer(buffer, true, false); - } - - static PinotNativeOrderLBuffer loadFile(File file, long offset, long size) - throws IOException { - PinotNativeOrderLBuffer buffer = allocateDirect(size); - buffer.readFrom(0, file, offset, size); - return buffer; - } - - public static PinotNativeOrderLBuffer mapFile(File file, boolean readOnly, long offset, long size) - throws IOException { - if (readOnly) { - return new PinotNativeOrderLBuffer(new MMapBuffer(file, offset, size, MMapMode.READ_ONLY), true, false); - } else { - return new PinotNativeOrderLBuffer(new MMapBuffer(file, offset, size, MMapMode.READ_WRITE), true, true); - } - } - - PinotNativeOrderLBuffer(LBufferAPI buffer, boolean closeable, boolean flushable) { - super(buffer, closeable, flushable); - } - - @Override - public char getChar(int offset) { - return _buffer.getChar(offset); - } - - @Override - public char getChar(long offset) { - return _buffer.getChar(offset); - } - - @Override - public void putChar(int offset, char value) { - _buffer.putChar(offset, value); - } - - @Override - public void putChar(long offset, char value) { - _buffer.putChar(offset, value); - } - - @Override - public short getShort(int offset) { - return _buffer.getShort(offset); - } - - @Override - public short getShort(long offset) { - return _buffer.getShort(offset); - } - - @Override - public void putShort(int offset, short value) { - _buffer.putShort(offset, value); - } - - @Override - public void putShort(long offset, short value) { - _buffer.putShort(offset, value); - } - - @Override - public int getInt(int offset) { - return _buffer.getInt(offset); - } - - @Override - public int getInt(long offset) { - return _buffer.getInt(offset); - } - - @Override - public void putInt(int offset, int value) { - _buffer.putInt(offset, value); - } - - @Override - public void putInt(long offset, int value) { - _buffer.putInt(offset, value); - } - - @Override - public long getLong(int offset) { - return _buffer.getLong(offset); - } - - @Override - public long getLong(long offset) { - return _buffer.getLong(offset); - } - - @Override - public void putLong(int offset, long value) { - _buffer.putLong(offset, value); - } - - @Override - public void putLong(long offset, long value) { - _buffer.putLong(offset, value); - } - - @Override - public float getFloat(int offset) { - return _buffer.getFloat(offset); - } - - @Override - public float getFloat(long offset) { - return _buffer.getFloat(offset); - } - - @Override - public void putFloat(int offset, float value) { - _buffer.putFloat(offset, value); - } - - @Override - public void putFloat(long offset, float value) { - _buffer.putFloat(offset, value); - } - - @Override - public double getDouble(int offset) { - return _buffer.getDouble(offset); - } - - @Override - public double getDouble(long offset) { - return _buffer.getDouble(offset); - } - - @Override - public void putDouble(int offset, double value) { - _buffer.putDouble(offset, value); - } - - @Override - public void putDouble(long offset, double value) { - _buffer.putDouble(offset, value); - } - - @Override - public ByteOrder order() { - return NATIVE_ORDER; - } -} diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotNonNativeOrderLBuffer.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotNonNativeOrderLBuffer.java deleted file mode 100644 index 42d77b74e9c7..000000000000 --- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/PinotNonNativeOrderLBuffer.java +++ /dev/null @@ -1,183 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pinot.segment.spi.memory; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteOrder; -import javax.annotation.concurrent.ThreadSafe; -import xerial.larray.buffer.LBuffer; -import xerial.larray.buffer.LBufferAPI; -import xerial.larray.mmap.MMapBuffer; -import xerial.larray.mmap.MMapMode; - - -@ThreadSafe -public class PinotNonNativeOrderLBuffer extends BasePinotLBuffer { - - static PinotNonNativeOrderLBuffer allocateDirect(long size) { - LBufferAPI buffer = new LBuffer(size); - return new PinotNonNativeOrderLBuffer(buffer, true, false); - } - - static PinotNonNativeOrderLBuffer loadFile(File file, long offset, long size) - throws IOException { - PinotNonNativeOrderLBuffer buffer = allocateDirect(size); - buffer.readFrom(0, file, offset, size); - return buffer; - } - - public static PinotNonNativeOrderLBuffer mapFile(File file, boolean readOnly, long offset, long size) - throws IOException { - if (readOnly) { - return new PinotNonNativeOrderLBuffer(new MMapBuffer(file, offset, size, MMapMode.READ_ONLY), true, false); - } else { - return new PinotNonNativeOrderLBuffer(new MMapBuffer(file, offset, size, MMapMode.READ_WRITE), true, true); - } - } - - PinotNonNativeOrderLBuffer(LBufferAPI buffer, boolean closeable, boolean flushable) { - super(buffer, closeable, flushable); - } - - @Override - public char getChar(int offset) { - return Character.reverseBytes(_buffer.getChar(offset)); - } - - @Override - public char getChar(long offset) { - return Character.reverseBytes(_buffer.getChar(offset)); - } - - @Override - public void putChar(int offset, char value) { - _buffer.putChar(offset, Character.reverseBytes(value)); - } - - @Override - public void putChar(long offset, char value) { - _buffer.putChar(offset, Character.reverseBytes(value)); - } - - @Override - public short getShort(int offset) { - return Short.reverseBytes(_buffer.getShort(offset)); - } - - @Override - public short getShort(long offset) { - return Short.reverseBytes(_buffer.getShort(offset)); - } - - @Override - public void putShort(int offset, short value) { - _buffer.putShort(offset, Short.reverseBytes(value)); - } - - @Override - public void putShort(long offset, short value) { - _buffer.putShort(offset, Short.reverseBytes(value)); - } - - @Override - public int getInt(int offset) { - return Integer.reverseBytes(_buffer.getInt(offset)); - } - - @Override - public int getInt(long offset) { - return Integer.reverseBytes(_buffer.getInt(offset)); - } - - @Override - public void putInt(int offset, int value) { - _buffer.putInt(offset, Integer.reverseBytes(value)); - } - - @Override - public void putInt(long offset, int value) { - _buffer.putInt(offset, Integer.reverseBytes(value)); - } - - @Override - public long getLong(int offset) { - return Long.reverseBytes(_buffer.getLong(offset)); - } - - @Override - public long getLong(long offset) { - return Long.reverseBytes(_buffer.getLong(offset)); - } - - @Override - public void putLong(int offset, long value) { - _buffer.putLong(offset, Long.reverseBytes(value)); - } - - @Override - public void putLong(long offset, long value) { - _buffer.putLong(offset, Long.reverseBytes(value)); - } - - @Override - public float getFloat(int offset) { - return Float.intBitsToFloat(Integer.reverseBytes(_buffer.getInt(offset))); - } - - @Override - public float getFloat(long offset) { - return Float.intBitsToFloat(Integer.reverseBytes(_buffer.getInt(offset))); - } - - @Override - public void putFloat(int offset, float value) { - _buffer.putInt(offset, Integer.reverseBytes(Float.floatToRawIntBits(value))); - } - - @Override - public void putFloat(long offset, float value) { - _buffer.putInt(offset, Integer.reverseBytes(Float.floatToRawIntBits(value))); - } - - @Override - public double getDouble(int offset) { - return Double.longBitsToDouble(Long.reverseBytes(_buffer.getLong(offset))); - } - - @Override - public double getDouble(long offset) { - return Double.longBitsToDouble(Long.reverseBytes(_buffer.getLong(offset))); - } - - @Override - public void putDouble(int offset, double value) { - _buffer.putLong(offset, Long.reverseBytes(Double.doubleToRawLongBits(value))); - } - - @Override - public void putDouble(long offset, double value) { - _buffer.putLong(offset, Long.reverseBytes(Double.doubleToRawLongBits(value))); - } - - @Override - public ByteOrder order() { - return NON_NATIVE_ORDER; - } -} diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/unsafe/UnsafePinotBufferFactory.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/unsafe/UnsafePinotBufferFactory.java index 3b7cce2b0028..f9406289c718 100644 --- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/unsafe/UnsafePinotBufferFactory.java +++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/memory/unsafe/UnsafePinotBufferFactory.java @@ -21,15 +21,15 @@ import java.io.File; import java.io.IOException; import java.nio.ByteOrder; +import org.apache.commons.lang3.SystemUtils; import org.apache.pinot.segment.spi.memory.NonNativePinotDataBuffer; import org.apache.pinot.segment.spi.memory.PinotBufferFactory; import org.apache.pinot.segment.spi.memory.PinotDataBuffer; -import xerial.larray.impl.OSInfo; public class UnsafePinotBufferFactory implements PinotBufferFactory { public UnsafePinotBufferFactory() { - if (OSInfo.isWindows()) { + if (SystemUtils.IS_OS_WINDOWS) { throw new IllegalStateException(getClass().getCanonicalName() + " cannot be used in Windows"); } } diff --git a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotLArrayByteBufferTest.java b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotLArrayByteBufferTest.java deleted file mode 100644 index 5ba1ddc2fabb..000000000000 --- a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/memory/PinotLArrayByteBufferTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pinot.segment.spi.memory; - -import net.openhft.chronicle.core.Jvm; -import org.apache.pinot.segment.spi.utils.JavaVersion; -import org.testng.SkipException; -import org.testng.annotations.BeforeClass; - - -public class PinotLArrayByteBufferTest extends PinotDataBufferTest { - public PinotLArrayByteBufferTest() { - super(new LArrayPinotBufferFactory()); - } - - @Override - protected boolean prioritizeByteBuffer() { - return false; - } - - @BeforeClass - public void abortOnModernJava() { - //larray isn't supported on Mac/aarch64 - if (Jvm.isMacArm()) { - throw new SkipException("Skipping LArray tests because they cannot run on Mac/aarch64"); - } - if (JavaVersion.VERSION > 15) { - throw new SkipException("Skipping LArray tests because they cannot run in Java " + JavaVersion.VERSION); - } - } -} diff --git a/pom.xml b/pom.xml index 4f1973bbd1f8..5040b87e45fc 100644 --- a/pom.xml +++ b/pom.xml @@ -990,11 +990,6 @@ snakeyaml 2.4 - - org.xerial.larray - larray-mmap - 0.4.1 - org.apache.zookeeper zookeeper