Skip to content

transition away from StorageAdapter #16985

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
clintropolis merged 16 commits into from
Sep 9, 2024
Merged

transition away from StorageAdapter #16985

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
0bdcc3f
transition away from StorageAdapter
clintropolis Sep 2, 2024
c0e6ae3
tweaks
clintropolis Sep 2, 2024
4200318
imports
clintropolis Sep 2, 2024
0ca4027
remove default implementation of asCursorFactory
clintropolis Sep 2, 2024
cee3e44
adjust self review
clintropolis Sep 3, 2024
79c474e
Merge remote-tracking branch 'upstream/master' into pining-for-the-fj…
clintropolis Sep 3, 2024
d96befb
isolate cardinality
clintropolis Sep 3, 2024
24c38f7
style
clintropolis Sep 3, 2024
2f54491
make QueryableIndexCursorFactory in constructor
clintropolis Sep 4, 2024
b872846
Merge remote-tracking branch 'upstream/master' into pining-for-the-fj…
clintropolis Sep 4, 2024
5d1ddc8
Merge remote-tracking branch 'upstream/master' into pining-for-the-fj…
clintropolis Sep 5, 2024
1964dae
Merge remote-tracking branch 'upstream/master' into pining-for-the-fj…
clintropolis Sep 9, 2024
b948662
review stuff
clintropolis Sep 9, 2024
036cbbd
oops
clintropolis Sep 9, 2024
26d2eda
fix test
clintropolis Sep 9, 2024
af8f0c7
fix join
clintropolis Sep 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionAggregationBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import org.apache.druid.segment.CursorBuildSpec;
import org.apache.druid.segment.CursorHolder;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.QueryableIndexStorageAdapter;
import org.apache.druid.segment.QueryableIndexCursorFactory;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.generator.GeneratorColumnSchema;
import org.apache.druid.segment.generator.GeneratorSchemaInfo;
Expand Down Expand Up @@ -161,9 +161,8 @@ public void queryUsingNative(Blackhole blackhole)

private double compute(final Function<ColumnSelectorFactory, BufferAggregator> aggregatorFactory)
{
final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(index);

try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final Cursor cursor = cursorHolder.asCursor();

final BufferAggregator bufferAggregator = aggregatorFactory.apply(cursor.getColumnSelectorFactory());
Expand Down
10 changes: 7 additions & 3 deletions benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionFilterBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
import org.apache.druid.segment.CursorBuildSpec;
import org.apache.druid.segment.CursorHolder;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.QueryableIndexStorageAdapter;
import org.apache.druid.segment.QueryableIndexCursorFactory;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.generator.GeneratorColumnSchema;
import org.apache.druid.segment.generator.GeneratorSchemaInfo;
Expand Down Expand Up @@ -148,7 +148,9 @@ public void expressionFilter(Blackhole blackhole)
final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
.setFilter(expressionFilter.toFilter())
.build();
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);

try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();


Expand All @@ -166,7 +168,9 @@ public void nativeFilter(Blackhole blackhole)
final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
.setFilter(nativeFilter.toFilter())
.build();
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);

try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("x");
while (!cursor.isDone()) {
Expand Down
51 changes: 32 additions & 19 deletions benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionSelectorBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,8 @@
import org.apache.druid.segment.Cursors;
import org.apache.druid.segment.DimensionSelector;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.QueryableIndexStorageAdapter;
import org.apache.druid.segment.QueryableIndexCursorFactory;
import org.apache.druid.segment.QueryableIndexTimeBoundaryInspector;
import org.apache.druid.segment.StorageAdapter;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnType;
Expand Down Expand Up @@ -160,7 +159,8 @@ public void timeFloorUsingExpression(Blackhole blackhole)
)
)
.build();
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();

final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
Expand All @@ -171,7 +171,8 @@ public void timeFloorUsingExpression(Blackhole blackhole)
@Benchmark
public void timeFloorUsingExtractionFn(Blackhole blackhole)
{
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final Cursor cursor = cursorHolder.asCursor();

final DimensionSelector selector = cursor
Expand All @@ -190,15 +191,15 @@ public void timeFloorUsingExtractionFn(Blackhole blackhole)
@Benchmark
public void timeFloorUsingCursor(Blackhole blackhole)
{
final StorageAdapter adapter = new QueryableIndexStorageAdapter(index);
try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final Cursor cursor = cursorHolder.asCursor();
final CursorGranularizer granularizer = CursorGranularizer.create(
cursor,
QueryableIndexTimeBoundaryInspector.create(index),
Cursors.getTimeOrdering(index.getOrdering()),
Granularities.HOUR,
adapter.getInterval()
index.getDataInterval()
);
final Sequence<Long> results =
Sequences.simple(granularizer.getBucketIterable())
Expand Down Expand Up @@ -241,7 +242,8 @@ public void timeFormatUsingExpression(Blackhole blackhole)
)
.build();

try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final DimensionSelector selector = cursor.getColumnSelectorFactory().makeDimensionSelector(
DefaultDimensionSpec.of("v")
Expand All @@ -253,7 +255,8 @@ public void timeFormatUsingExpression(Blackhole blackhole)
@Benchmark
public void timeFormatUsingExtractionFn(Blackhole blackhole)
{
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final Cursor cursor = cursorHolder.asCursor();
final DimensionSelector selector = cursor
.getColumnSelectorFactory()
Expand Down Expand Up @@ -284,7 +287,8 @@ public void strlenUsingExpressionAsLong(Blackhole blackhole)
)
.build();

try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
Expand All @@ -307,7 +311,8 @@ public void strlenUsingExpressionAsString(Blackhole blackhole)
)
.build();

try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final DimensionSelector selector = cursor
.getColumnSelectorFactory()
Expand All @@ -320,7 +325,8 @@ public void strlenUsingExpressionAsString(Blackhole blackhole)
@Benchmark
public void strlenUsingExtractionFn(Blackhole blackhole)
{
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
final Cursor cursor = cursorHolder.asCursor();
final DimensionSelector selector = cursor
.getColumnSelectorFactory()
Expand All @@ -346,7 +352,8 @@ public void arithmeticOnLong(Blackhole blackhole)
)
.build();

try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
Expand All @@ -368,7 +375,8 @@ public void stringConcatAndCompareOnLong(Blackhole blackhole)
)
)
.build();
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
Expand All @@ -390,7 +398,8 @@ public void caseSearched1(Blackhole blackhole)
)
)
.build();
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
Expand All @@ -412,7 +421,8 @@ public void caseSearched2(Blackhole blackhole)
)
)
.build();
try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
Expand Down Expand Up @@ -447,7 +457,8 @@ public void caseSearched100(Blackhole blackhole)
)
.build();

try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
Expand Down Expand Up @@ -476,7 +487,8 @@ public void caseSearchedWithLookup(Blackhole blackhole)
)
.build();

try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
Expand Down Expand Up @@ -513,7 +525,8 @@ public void caseSearchedWithLookup2(Blackhole blackhole)
)
.build();

try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
final Cursor cursor = cursorHolder.asCursor();
final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
consumeLong(cursor, selector, blackhole);
Expand Down
4 changes: 2 additions & 2 deletions benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
import org.apache.druid.segment.CursorBuildSpec;
import org.apache.druid.segment.CursorHolder;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.QueryableIndexStorageAdapter;
import org.apache.druid.segment.QueryableIndexCursorFactory;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.generator.GeneratorBasicSchemas;
import org.apache.druid.segment.generator.GeneratorSchemaInfo;
Expand Down Expand Up @@ -155,7 +155,7 @@ public void scan(Blackhole blackhole)
.setVirtualColumns(virtualColumns)
.build();
final CursorHolder cursorHolder = closer.register(
new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)
new QueryableIndexCursorFactory(index).makeCursorHolder(buildSpec)
);
if (vectorize) {
VectorCursor cursor = cursorHolder.asVectorCursor();
Expand Down
Loading