Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import io.druid.collections.bitmap.MutableBitmap;
import io.druid.query.aggregation.Aggregator;
import io.druid.segment.DimensionSelector;
import io.druid.segment.data.IndexedInts;

public class DistinctCountAggregator implements Aggregator
{
Expand All @@ -41,7 +42,9 @@ public DistinctCountAggregator(
@Override
public void aggregate()
{
for (final Integer index : selector.getRow()) {
IndexedInts row = selector.getRow();
for (int i = 0; i < row.size(); i++) {
int index = row.get(i);
mutableBitmap.add(index);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import io.druid.query.aggregation.BufferAggregator;
import io.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import io.druid.segment.DimensionSelector;
import io.druid.segment.data.IndexedInts;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;

Expand Down Expand Up @@ -51,7 +52,9 @@ public void init(ByteBuffer buf, int position)
public void aggregate(ByteBuffer buf, int position)
{
MutableBitmap mutableBitmap = getMutableBitmap(buf, position);
for (final Integer index : selector.getRow()) {
IndexedInts row = selector.getRow();
for (int i = 0; i < row.size(); i++) {
int index = row.get(i);
mutableBitmap.add(index);
}
buf.putLong(position, mutableBitmap.size());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import io.druid.query.aggregation.cardinality.CardinalityAggregator;
import io.druid.segment.DimensionSelector;
import io.druid.segment.data.IndexedInts;
import it.unimi.dsi.fastutil.ints.IntIterator;

import java.util.Arrays;

Expand Down Expand Up @@ -62,8 +61,9 @@ public void hashRow(DimensionSelector dimSelector, Hasher hasher)
@Override
public void hashValues(DimensionSelector dimSelector, HyperLogLogCollector collector)
{
for (IntIterator rowIt = dimSelector.getRow().iterator(); rowIt.hasNext(); ) {
int index = rowIt.nextInt();
IndexedInts row = dimSelector.getRow();
for (int i = 0; i < row.size(); i++) {
int index = row.get(i);
final String value = dimSelector.lookupName(index);
collector.add(CardinalityAggregator.hashFn.hashUnencodedChars(nullToSpecial(value)).asBytes());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -196,8 +196,9 @@ private List<ByteBuffer> updateValues(
newKey.putInt(MISSING_VALUE);
unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size()));
} else {
for (Integer dimValue : row) {
for (int i = 0; i < row.size(); i++) {
ByteBuffer newKey = key.duplicate();
int dimValue = row.get(i);
newKey.putInt(dimValue);
unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size()));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,9 @@
import io.druid.segment.data.CompressedObjectStrategy;
import io.druid.segment.data.CompressedVSizeIntsIndexedSupplier;
import io.druid.segment.data.IndexedInts;
import io.druid.segment.data.IndexedIntsIterator;
import io.druid.segment.data.IndexedIterable;
import io.druid.segment.data.IndexedMultivalue;
import io.druid.segment.data.WritableSupplier;
import it.unimi.dsi.fastutil.ints.IntIterator;

import java.io.IOException;
import java.nio.ByteBuffer;
Expand Down Expand Up @@ -206,12 +204,6 @@ public void close() throws IOException
// no-op
}

@Override
public IntIterator iterator()
{
return new IndexedIntsIterator(this);
}

@Override
public void inspectRuntimeShape(RuntimeShapeInspector inspector)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
package io.druid.segment;

import io.druid.segment.column.ColumnCapabilities;
import io.druid.segment.data.BitmapValues;
import io.druid.segment.data.Indexed;
import io.druid.segment.data.IndexedInts;
import org.joda.time.Interval;

import java.util.Map;
Expand All @@ -43,7 +43,7 @@ public interface IndexableAdapter

Iterable<Rowboat> getRows();

IndexedInts getBitmapIndex(String dimension, int dictId);
BitmapValues getBitmapValues(String dimension, int dictId);

String getMetricType(String metric);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public static int skip(IntIterator it, int n)
}
int skipped = 0;
while (skipped < n && it.hasNext()) {
it.next();
it.nextInt();
skipped++;
}
return skipped;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,9 @@
import io.druid.segment.column.IndexedFloatsGenericColumn;
import io.druid.segment.column.IndexedLongsGenericColumn;
import io.druid.segment.column.ValueType;
import io.druid.segment.data.BitmapCompressedIndexedInts;
import io.druid.segment.data.EmptyIndexedInts;
import io.druid.segment.data.ImmutableBitmapValues;
import io.druid.segment.data.BitmapValues;
import io.druid.segment.data.Indexed;
import io.druid.segment.data.IndexedInts;
import io.druid.segment.data.IndexedIterable;
import io.druid.segment.data.ListIndexed;
import org.joda.time.Interval;
Expand Down Expand Up @@ -300,23 +299,6 @@ public void remove()
};
}

@VisibleForTesting
IndexedInts getBitmapIndex(String dimension, String value)
{
final Column column = input.getColumn(dimension);

if (column == null) {
return EmptyIndexedInts.EMPTY_INDEXED_INTS;
}

final BitmapIndex bitmaps = column.getBitmapIndex();
if (bitmaps == null) {
return EmptyIndexedInts.EMPTY_INDEXED_INTS;
}

return new BitmapCompressedIndexedInts(bitmaps.getBitmap(bitmaps.getIndex(value)));
}

@Override
public String getMetricType(String metric)
{
Expand Down Expand Up @@ -347,25 +329,42 @@ public ColumnCapabilities getCapabilities(String column)
}

@Override
public IndexedInts getBitmapIndex(String dimension, int dictId)
public BitmapValues getBitmapValues(String dimension, int dictId)
{
final Column column = input.getColumn(dimension);
if (column == null) {
return EmptyIndexedInts.EMPTY_INDEXED_INTS;
return BitmapValues.EMPTY;
}

final BitmapIndex bitmaps = column.getBitmapIndex();
if (bitmaps == null) {
return EmptyIndexedInts.EMPTY_INDEXED_INTS;
return BitmapValues.EMPTY;
}

if (dictId >= 0) {
return new BitmapCompressedIndexedInts(bitmaps.getBitmap(dictId));
return new ImmutableBitmapValues(bitmaps.getBitmap(dictId));
} else {
return EmptyIndexedInts.EMPTY_INDEXED_INTS;
return BitmapValues.EMPTY;
}
}

@VisibleForTesting
BitmapValues getBitmapIndex(String dimension, String value)
{
final Column column = input.getColumn(dimension);

if (column == null) {
return BitmapValues.EMPTY;
}

final BitmapIndex bitmaps = column.getBitmapIndex();
if (bitmaps == null) {
return BitmapValues.EMPTY;
}

return new ImmutableBitmapValues(bitmaps.getBitmap(bitmaps.getIndex(value)));
}

@Override
public Metadata getMetadata()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import io.druid.segment.column.ColumnCapabilities;
import io.druid.segment.data.BitmapValues;
import io.druid.segment.data.Indexed;
import io.druid.segment.data.IndexedInts;
import org.joda.time.Interval;

import java.util.Map;
Expand Down Expand Up @@ -90,9 +90,9 @@ public ColumnCapabilities getCapabilities(String column)
}

@Override
public IndexedInts getBitmapIndex(String dimension, int dictId)
public BitmapValues getBitmapValues(String dimension, int dictId)
{
return baseAdapter.getBitmapIndex(dimension, dictId);
return baseAdapter.getBitmapValues(dimension, dictId);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import io.druid.segment.column.ValueType;
import io.druid.segment.data.ArrayIndexed;
import io.druid.segment.data.BitmapSerdeFactory;
import io.druid.segment.data.BitmapValues;
import io.druid.segment.data.ByteBufferWriter;
import io.druid.segment.data.CompressedObjectStrategy;
import io.druid.segment.data.CompressedVSizeIndexedV3Writer;
Expand All @@ -48,7 +49,6 @@
import io.druid.segment.data.GenericIndexedWriter;
import io.druid.segment.data.IOPeon;
import io.druid.segment.data.Indexed;
import io.druid.segment.data.IndexedInts;
import io.druid.segment.data.IndexedIntsWriter;
import io.druid.segment.data.IndexedRTree;
import io.druid.segment.data.VSizeIndexedIntsWriter;
Expand All @@ -58,6 +58,7 @@
import it.unimi.dsi.fastutil.ints.IntIterable;
import it.unimi.dsi.fastutil.ints.IntIterator;

import javax.annotation.Nonnull;
import java.io.Closeable;
import java.io.File;
import java.io.FileOutputStream;
Expand Down Expand Up @@ -370,21 +371,22 @@ static void mergeBitmaps(
GenericIndexedWriter<ImmutableBitmap> bitmapWriter
) throws IOException
{
List<ConvertingIndexedInts> convertedInvertedIndexesToMerge = Lists.newArrayListWithCapacity(adapters.size());
List<ConvertingBitmapValues> convertedInvertedIndexesToMerge = Lists.newArrayListWithCapacity(adapters.size());
for (int j = 0; j < adapters.size(); ++j) {
int seekedDictId = dictIdSeeker[j].seek(dictId);
if (seekedDictId != IndexSeeker.NOT_EXIST) {
convertedInvertedIndexesToMerge.add(
new ConvertingIndexedInts(
adapters.get(j).getBitmapIndex(dimensionName, seekedDictId), segmentRowNumConversions.get(j)
new ConvertingBitmapValues(
adapters.get(j).getBitmapValues(dimensionName, seekedDictId),
segmentRowNumConversions.get(j)
)
);
}
}

MutableBitmap mergedIndexes = bmpFactory.makeEmptyMutableBitmap();
List<IntIterator> convertedInvertedIndexesIterators = new ArrayList<>(convertedInvertedIndexesToMerge.size());
for (ConvertingIndexedInts convertedInvertedIndexes : convertedInvertedIndexesToMerge) {
for (ConvertingBitmapValues convertedInvertedIndexes : convertedInvertedIndexesToMerge) {
convertedInvertedIndexesIterators.add(convertedInvertedIndexes.iterator());
}

Expand Down Expand Up @@ -538,34 +540,27 @@ public int seek(int dictId)
}
}

public static class ConvertingIndexedInts implements IntIterable
public static class ConvertingBitmapValues implements IntIterable
{
private final IndexedInts baseIndex;
private final BitmapValues baseValues;
private final IntBuffer conversionBuffer;

public ConvertingIndexedInts(
IndexedInts baseIndex,
IntBuffer conversionBuffer
)
ConvertingBitmapValues(BitmapValues baseValues, IntBuffer conversionBuffer)
{
this.baseIndex = baseIndex;
this.baseValues = baseValues;
this.conversionBuffer = conversionBuffer;
}

public int size()
{
return baseIndex.size();
}

public int get(int index)
{
return conversionBuffer.get(baseIndex.get(index));
return baseValues.size();
}

@Nonnull
@Override
public IntIterator iterator()
{
final IntIterator baseIterator = baseIndex.iterator();
final IntIterator baseIterator = baseValues.iterator();
return new AbstractIntIterator()
{
@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@
import io.druid.java.util.common.IAE;
import io.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import it.unimi.dsi.fastutil.ints.IntArrays;
import it.unimi.dsi.fastutil.ints.IntIterator;
import it.unimi.dsi.fastutil.ints.IntIterators;

import java.io.IOException;

Expand Down Expand Up @@ -76,12 +74,6 @@ public int get(int index)
return expansion[index];
}

@Override
public IntIterator iterator()
{
return IntIterators.wrap(expansion, 0, size);
}

@Override
public void close() throws IOException
{
Expand Down
Loading