Skip to content

remove druid.processing.columnCache.sizeBytes and CachingIndexed, combine string column implementations #14500

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
clintropolis merged 11 commits into from
Jul 3, 2023
Merged

remove druid.processing.columnCache.sizeBytes and CachingIndexed, combine string column implementations #14500

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
c375b94
combine string column implementations
clintropolis Jun 29, 2023
a33dab6
fix style
clintropolis Jun 29, 2023
d1b0185
remove unused
clintropolis Jun 29, 2023
c37da40
Merge remote-tracking branch 'upstream/master' into there-can-be-only…
clintropolis Jun 29, 2023
9e85ba0
more unused
clintropolis Jun 29, 2023
0a3d7c3
maybe this will pass inspections
clintropolis Jun 29, 2023
8716027
adjust
clintropolis Jun 29, 2023
2759bd4
revert
clintropolis Jun 29, 2023
126e77c
additional cleanup
clintropolis Jun 30, 2023
f01622f
fix style
clintropolis Jun 30, 2023
779f6c1
why do static checks hate me
clintropolis Jun 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 2 additions & 6 deletions benchmarks/src/test/java/org/apache/druid/benchmark/BoundFilterBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
import org.apache.druid.segment.filter.BoundFilter;
import org.apache.druid.segment.filter.Filters;
import org.apache.druid.segment.serde.DictionaryEncodedStringIndexSupplier;
import org.apache.druid.segment.serde.StringUtf8ColumnIndexSupplier;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
Expand Down Expand Up @@ -161,10 +161,6 @@ public void setup()
final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
final BitmapSerdeFactory serdeFactory = RoaringBitmapSerdeFactory.getInstance();
final List<Integer> ints = generateInts();
final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(
FluentIterable.from(ints).transform(Object::toString),
GenericIndexed.STRING_STRATEGY
);
final GenericIndexed<ImmutableBitmap> bitmaps = GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(
Expand All @@ -183,7 +179,7 @@ public void setup()
);
selector = new MockColumnIndexSelector(
bitmapFactory,
new DictionaryEncodedStringIndexSupplier(bitmapFactory, dictionary, dictionaryUtf8, bitmaps, null)
new StringUtf8ColumnIndexSupplier<>(bitmapFactory, dictionaryUtf8::singleThreaded, bitmaps, null)
);
}

Expand Down
11 changes: 3 additions & 8 deletions ...c/test/java/org/apache/druid/benchmark/DictionaryEncodedStringIndexSupplierBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
import org.apache.druid.segment.data.BitmapSerdeFactory;
import org.apache.druid.segment.data.GenericIndexed;
import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
import org.apache.druid.segment.serde.DictionaryEncodedStringIndexSupplier;
import org.apache.druid.segment.serde.StringUtf8ColumnIndexSupplier;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
Expand Down Expand Up @@ -93,11 +93,6 @@ public void setup()
final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
final BitmapSerdeFactory serdeFactory = RoaringBitmapSerdeFactory.getInstance();
final Iterable<Integer> ints = intGenerator();
final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(Object::toString),
GenericIndexed.STRING_STRATEGY
);
final GenericIndexed<ByteBuffer> dictionaryUtf8 = GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(i -> ByteBuffer.wrap(StringUtils.toUtf8(String.valueOf(i)))),
Expand All @@ -115,8 +110,8 @@ public void setup()
.iterator(),
serdeFactory.getObjectStrategy()
);
DictionaryEncodedStringIndexSupplier indexSupplier =
new DictionaryEncodedStringIndexSupplier(bitmapFactory, dictionary, dictionaryUtf8, bitmaps, null);
StringUtf8ColumnIndexSupplier<?> indexSupplier =
new StringUtf8ColumnIndexSupplier<>(bitmapFactory, dictionaryUtf8::singleThreaded, bitmaps, null);
stringValueSetIndex = (IndexedUtf8ValueSetIndex<?>) indexSupplier.as(StringValueSetIndex.class);
List<Integer> filterValues = new ArrayList<>();
List<Integer> nonFilterValues = new ArrayList<>();
Expand Down
9 changes: 2 additions & 7 deletions benchmarks/src/test/java/org/apache/druid/benchmark/DimensionPredicateFilterBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
import org.apache.druid.segment.filter.DimensionPredicateFilter;
import org.apache.druid.segment.filter.Filters;
import org.apache.druid.segment.serde.DictionaryEncodedStringIndexSupplier;
import org.apache.druid.segment.serde.StringUtf8ColumnIndexSupplier;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
Expand Down Expand Up @@ -122,11 +122,6 @@ public void setup()
final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
final BitmapSerdeFactory serdeFactory = RoaringBitmapSerdeFactory.getInstance();
final List<Integer> ints = generateInts();
final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(Object::toString),
GenericIndexed.STRING_STRATEGY
);
final GenericIndexed<ByteBuffer> dictionaryUtf8 = GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(i -> ByteBuffer.wrap(StringUtils.toUtf8(String.valueOf(i)))),
Expand All @@ -145,7 +140,7 @@ public void setup()
);
selector = new MockColumnIndexSelector(
bitmapFactory,
new DictionaryEncodedStringIndexSupplier(bitmapFactory, dictionary, dictionaryUtf8, bitmaps, null)
new StringUtf8ColumnIndexSupplier<>(bitmapFactory, dictionaryUtf8::singleThreaded, bitmaps, null)
);
}

Expand Down
5 changes: 0 additions & 5 deletions benchmarks/src/test/java/org/apache/druid/benchmark/FilterPartitionBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,11 +137,6 @@ public class FilterPartitionBenchmark
JSON_MAPPER,
new ColumnConfig()
{
@Override
public int columnCacheSizeBytes()
{
return 0;
}
}
);
INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance());
Expand Down
5 changes: 0 additions & 5 deletions benchmarks/src/test/java/org/apache/druid/benchmark/FilteredAggregatorBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,11 +140,6 @@ public class FilteredAggregatorBenchmark
JSON_MAPPER,
new ColumnConfig()
{
@Override
public int columnCacheSizeBytes()
{
return 0;
}
}
);
INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance());
Expand Down
5 changes: 0 additions & 5 deletions benchmarks/src/test/java/org/apache/druid/benchmark/GroupByTypeInterfaceBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,11 +158,6 @@ public class GroupByTypeInterfaceBenchmark
JSON_MAPPER,
new ColumnConfig()
{
@Override
public int columnCacheSizeBytes()
{
return 0;
}
}
);
INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance());
Expand Down
9 changes: 2 additions & 7 deletions benchmarks/src/test/java/org/apache/druid/benchmark/InFilterBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
import org.apache.druid.segment.data.GenericIndexed;
import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
import org.apache.druid.segment.filter.Filters;
import org.apache.druid.segment.serde.DictionaryEncodedStringIndexSupplier;
import org.apache.druid.segment.serde.StringUtf8ColumnIndexSupplier;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
Expand Down Expand Up @@ -85,11 +85,6 @@ public void setup()
final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
final BitmapSerdeFactory serdeFactory = RoaringBitmapSerdeFactory.getInstance();
final Iterable<Integer> ints = intGenerator();
final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(Object::toString),
GenericIndexed.STRING_STRATEGY
);
final GenericIndexed<ByteBuffer> dictionaryUtf8 = GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(i -> ByteBuffer.wrap(StringUtils.toUtf8(String.valueOf(i)))),
Expand All @@ -109,7 +104,7 @@ public void setup()
);
selector = new MockColumnIndexSelector(
bitmapFactory,
new DictionaryEncodedStringIndexSupplier(bitmapFactory, dictionary, dictionaryUtf8, bitmaps, null)
new StringUtf8ColumnIndexSupplier<>(bitmapFactory, dictionaryUtf8::singleThreaded, bitmaps, null)
);
inFilter = new InDimFilter(
"dummy",
Expand Down
6 changes: 1 addition & 5 deletions benchmarks/src/test/java/org/apache/druid/benchmark/JoinAndLookupBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,6 @@ public class JoinAndLookupBenchmark
@Param({"500000"})
int rows;

@Param({"0", "16384"})
int columnCacheSizeBytes;

private File tmpDir = null;
private QueryableIndex index = null;
private Segment baseSegment = null;
Expand All @@ -123,8 +120,7 @@ public void tearDown() throws IOException
public void setup() throws IOException
{
tmpDir = FileUtils.createTempDir();
ColumnConfig columnConfig = () -> columnCacheSizeBytes;
index = JoinTestHelper.createFactIndexBuilder(columnConfig, tmpDir, rows).buildMMappedIndex();
index = JoinTestHelper.createFactIndexBuilder(ColumnConfig.DEFAULT, tmpDir, rows).buildMMappedIndex();

final String prefix = "c.";

Expand Down
9 changes: 2 additions & 7 deletions benchmarks/src/test/java/org/apache/druid/benchmark/LikeFilterBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
import org.apache.druid.segment.data.GenericIndexed;
import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
import org.apache.druid.segment.filter.Filters;
import org.apache.druid.segment.serde.DictionaryEncodedStringIndexSupplier;
import org.apache.druid.segment.serde.StringUtf8ColumnIndexSupplier;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
Expand Down Expand Up @@ -122,11 +122,6 @@ public void setup()
final BitmapFactory bitmapFactory = new RoaringBitmapFactory();
final BitmapSerdeFactory serdeFactory = RoaringBitmapSerdeFactory.getInstance();
final List<Integer> ints = generateInts();
final GenericIndexed<String> dictionary = GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(Object::toString),
GenericIndexed.STRING_STRATEGY
);
final GenericIndexed<ByteBuffer> dictionaryUtf8 = GenericIndexed.fromIterable(
FluentIterable.from(ints)
.transform(i -> ByteBuffer.wrap(StringUtils.toUtf8(String.valueOf(i)))),
Expand All @@ -145,7 +140,7 @@ public void setup()
);
selector = new MockColumnIndexSelector(
bitmapFactory,
new DictionaryEncodedStringIndexSupplier(bitmapFactory, dictionary, dictionaryUtf8, bitmaps, null)
new StringUtf8ColumnIndexSupplier<>(bitmapFactory, dictionaryUtf8::singleThreaded, bitmaps, null)
);
}

Expand Down
5 changes: 0 additions & 5 deletions benchmarks/src/test/java/org/apache/druid/benchmark/TopNTypeInterfaceBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,11 +138,6 @@ public class TopNTypeInterfaceBenchmark
JSON_MAPPER,
new ColumnConfig()
{
@Override
public int columnCacheSizeBytes()
{
return 0;
}
}
);
INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance());
Expand Down
6 changes: 2 additions & 4 deletions ...a/org/apache/druid/benchmark/compression/ColumnarLongsEncodeDataFromSegmentBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.druid.segment.IndexIO;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnConfig;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.LongsColumn;
import org.apache.druid.segment.column.ValueType;
Expand Down Expand Up @@ -118,10 +119,7 @@ private void initializeSegmentValueIntermediaryFile() throws IOException
File dataFile = new File(dir, getColumnDataFileName(segmentName, columnName));

if (!dataFile.exists()) {
final IndexIO indexIO = new IndexIO(
new DefaultObjectMapper(),
() -> 0
);
final IndexIO indexIO = new IndexIO(new DefaultObjectMapper(), ColumnConfig.DEFAULT);
try (final QueryableIndex index = indexIO.loadIndex(new File(segmentPath))) {
final Set<String> columnNames = new LinkedHashSet<>();
columnNames.add(ColumnHolder.TIME_COLUMN_NAME);
Expand Down
6 changes: 2 additions & 4 deletions benchmarks/src/test/java/org/apache/druid/benchmark/indexing/IndexMergeBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.apache.druid.segment.IndexMergerV9;
import org.apache.druid.segment.IndexSpec;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.column.ColumnConfig;
import org.apache.druid.segment.generator.DataGenerator;
import org.apache.druid.segment.generator.GeneratorBasicSchemas;
import org.apache.druid.segment.generator.GeneratorSchemaInfo;
Expand Down Expand Up @@ -105,10 +106,7 @@ public class IndexMergeBenchmark
InjectableValues.Std injectableValues = new InjectableValues.Std();
injectableValues.addValue(ExprMacroTable.class, ExprMacroTable.nil());
JSON_MAPPER.setInjectableValues(injectableValues);
INDEX_IO = new IndexIO(
JSON_MAPPER,
() -> 0
);
INDEX_IO = new IndexIO(JSON_MAPPER, ColumnConfig.DEFAULT);
}

@Setup
Expand Down
6 changes: 2 additions & 4 deletions benchmarks/src/test/java/org/apache/druid/benchmark/indexing/IndexPersistBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.druid.segment.IndexIO;
import org.apache.druid.segment.IndexMergerV9;
import org.apache.druid.segment.IndexSpec;
import org.apache.druid.segment.column.ColumnConfig;
import org.apache.druid.segment.generator.DataGenerator;
import org.apache.druid.segment.generator.GeneratorBasicSchemas;
import org.apache.druid.segment.generator.GeneratorSchemaInfo;
Expand Down Expand Up @@ -74,10 +75,7 @@ public class IndexPersistBenchmark
static {
NullHandling.initializeForTests();
JSON_MAPPER = new DefaultObjectMapper();
INDEX_IO = new IndexIO(
JSON_MAPPER,
() -> 0
);
INDEX_IO = new IndexIO(JSON_MAPPER, ColumnConfig.DEFAULT);
INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance());
}

Expand Down
5 changes: 0 additions & 5 deletions benchmarks/src/test/java/org/apache/druid/benchmark/query/GroupByBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,11 +174,6 @@ public class GroupByBenchmark
),
new ColumnConfig()
{
@Override
public int columnCacheSizeBytes()
{
return 0;
}
}
);
INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance());
Expand Down
6 changes: 2 additions & 4 deletions benchmarks/src/test/java/org/apache/druid/benchmark/query/ScanBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
import org.apache.druid.segment.IndexSpec;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.QueryableIndexSegment;
import org.apache.druid.segment.column.ColumnConfig;
import org.apache.druid.segment.generator.DataGenerator;
import org.apache.druid.segment.generator.GeneratorBasicSchemas;
import org.apache.druid.segment.generator.GeneratorSchemaInfo;
Expand Down Expand Up @@ -136,10 +137,7 @@ public class ScanBenchmark

static {
JSON_MAPPER = new DefaultObjectMapper();
INDEX_IO = new IndexIO(
JSON_MAPPER,
() -> 0
);
INDEX_IO = new IndexIO(JSON_MAPPER, ColumnConfig.DEFAULT);
INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance());
}

Expand Down
5 changes: 0 additions & 5 deletions benchmarks/src/test/java/org/apache/druid/benchmark/query/SearchBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,11 +143,6 @@ public class SearchBenchmark
JSON_MAPPER,
new ColumnConfig()
{
@Override
public int columnCacheSizeBytes()
{
return 0;
}
}
);
INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance());
Expand Down
5 changes: 0 additions & 5 deletions benchmarks/src/test/java/org/apache/druid/benchmark/query/TimeseriesBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,11 +136,6 @@ public class TimeseriesBenchmark
JSON_MAPPER,
new ColumnConfig()
{
@Override
public int columnCacheSizeBytes()
{
return 0;
}
}
);
INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance());
Expand Down
5 changes: 0 additions & 5 deletions benchmarks/src/test/java/org/apache/druid/benchmark/query/TopNBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,11 +134,6 @@ public class TopNBenchmark
JSON_MAPPER,
new ColumnConfig()
{
@Override
public int columnCacheSizeBytes()
{
return 0;
}
}
);
INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance());
Expand Down
5 changes: 0 additions & 5 deletions ...arks/src/test/java/org/apache/druid/benchmark/query/timecompare/TimeCompareBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,11 +160,6 @@ public class TimeCompareBenchmark
JSON_MAPPER,
new ColumnConfig()
{
@Override
public int columnCacheSizeBytes()
{
return 0;
}
}
);
INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance());
Expand Down
Loading