From 58a4cbdff6956009b66750d6fef7fa76056739fd Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 30 Jul 2025 03:21:45 -0700 Subject: [PATCH 1/9] add support for filtered projections --- .../benchmark/query/SqlBenchmarkDatasets.java | 3 + .../segment/DatasketchesProjectionTest.java | 2 + .../catalog/storage/TableManagerTest.java | 1 + .../apache/druid/msq/exec/MSQInsertTest.java | 4 + .../msq/indexing/MSQCompactionRunnerTest.java | 1 + .../DataSourceMSQDestinationTest.java | 2 + .../indexing/common/task/CompactionTask.java | 1 + .../task/CompactionTaskParallelRunTest.java | 2 + .../common/task/CompactionTaskTest.java | 1 + .../input/impl/AggregateProjectionSpec.java | 29 +++- .../segment/AggregateProjectionMetadata.java | 149 ++++++++++++++++-- .../druid/segment/filter/IsBooleanFilter.java | 10 ++ .../druid/segment/filter/TrueFilter.java | 2 +- .../OnHeapAggregateProjection.java | 38 ++++- .../segment/projections/Projections.java | 15 ++ .../impl/AggregateProjectionSpecTest.java | 8 + .../query/metadata/SegmentAnalysisTest.java | 1 + ...egmentMetadataQueryQueryToolChestTest.java | 2 + .../AggregateProjectionMetadataTest.java | 62 +++++++- .../segment/CursorFactoryProjectionTest.java | 70 ++++++++ .../druid/segment/IndexMergerTestBase.java | 2 + .../apache/druid/segment/MetadataTest.java | 5 + .../QueryableIndexCursorHolderTest.java | 1 + .../org/apache/druid/segment/TestIndex.java | 2 + .../OnheapIncrementalIndexTest.java | 11 ++ .../druid/segment/indexing/DataSchema.java | 2 +- .../segment/indexing/DataSchemaTest.java | 7 + .../compaction/CompactionStatusTest.java | 3 + ...CatalogDataSourceCompactionConfigTest.java | 1 + .../coordinator/duty/CompactSegmentsTest.java | 2 + 30 files changed, 418 insertions(+), 21 deletions(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java index 7b6b48802f34..8892aef8bc3e 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java @@ -164,6 +164,7 @@ public class SqlBenchmarkDatasets Arrays.asList( new AggregateProjectionSpec( "string2_hourly_sums_hll", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, "__gran") ), @@ -179,6 +180,7 @@ public class SqlBenchmarkDatasets ), new AggregateProjectionSpec( "string2_long2_sums", + null, VirtualColumns.EMPTY, Arrays.asList( new StringDimensionSchema("string2"), @@ -409,6 +411,7 @@ public BenchmarkSchema asAutoDimensions() projections.stream() .map(projection -> new AggregateProjectionSpec( projection.getName(), + null, projection.getVirtualColumns(), projection.getGroupingColumns() .stream() diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/segment/DatasketchesProjectionTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/segment/DatasketchesProjectionTest.java index e0548babab3e..589fa3dad0c1 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/segment/DatasketchesProjectionTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/segment/DatasketchesProjectionTest.java @@ -100,6 +100,7 @@ public class DatasketchesProjectionTest extends InitializedNullHandlingTest private static final List PROJECTIONS = Collections.singletonList( new AggregateProjectionSpec( "a_projection", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, "__gran") ), @@ -120,6 +121,7 @@ public class DatasketchesProjectionTest extends InitializedNullHandlingTest private static final List AUTO_PROJECTIONS = PROJECTIONS.stream().map(projection -> { return new AggregateProjectionSpec( projection.getName(), + projection.getFilter(), projection.getVirtualColumns(), projection.getGroupingColumns() .stream() diff --git a/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java b/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java index 0967cde0bf41..ba835ab24004 100644 --- a/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java +++ b/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java @@ -96,6 +96,7 @@ public void testCreate() throws DuplicateKeyException, NotFoundException final DatasourceProjectionMetadata projectionMetadata = new DatasourceProjectionMetadata( new AggregateProjectionSpec( "projection", + null, VirtualColumns.create( Granularities.toVirtualColumn( Granularities.HOUR, diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java index 719f096a4e93..b8262df7954c 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java @@ -198,6 +198,7 @@ protected CatalogResolver createMockCatalogResolver() new DatasourceProjectionMetadata( new AggregateProjectionSpec( "channel_added_hourly", + null, VirtualColumns.create( Granularities.toVirtualColumn( Granularities.HOUR, @@ -216,6 +217,7 @@ protected CatalogResolver createMockCatalogResolver() new DatasourceProjectionMetadata( new AggregateProjectionSpec( "channel_delta_daily", + null, VirtualColumns.create( Granularities.toVirtualColumn( Granularities.DAY, @@ -553,6 +555,7 @@ public void testInsertOnExternalDataSourceWithCatalogProjections(String contextN new AggregateProjectionMetadata.Schema( "channel_added_hourly", Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, + null, VirtualColumns.create( Granularities.toVirtualColumn( Granularities.HOUR, @@ -574,6 +577,7 @@ public void testInsertOnExternalDataSourceWithCatalogProjections(String contextN new AggregateProjectionMetadata.Schema( "channel_delta_daily", Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, + null, VirtualColumns.create( Granularities.toVirtualColumn( Granularities.DAY, diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java index 0d737321e32c..7bcd3a041507 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java @@ -121,6 +121,7 @@ public class MSQCompactionRunnerTest ); private static final AggregateProjectionSpec PROJECTION_SPEC = new AggregateProjectionSpec( "projection", + null, VirtualColumns.create( Granularities.toVirtualColumn( Granularities.HOUR, diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java index 5d19bc4fc547..a7c5970a829e 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java @@ -71,6 +71,7 @@ public void testEquals() ImmutableList.of( new AggregateProjectionSpec( "projection1", + null, VirtualColumns.EMPTY, ImmutableList.of( new StringDimensionSchema("region") @@ -83,6 +84,7 @@ public void testEquals() ImmutableList.of( new AggregateProjectionSpec( "projection2", + null, VirtualColumns.create( Granularities.toVirtualColumn( Granularities.HOUR, diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java index de2f068d73bc..dc9a3033f18d 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java @@ -1180,6 +1180,7 @@ private void processProjections(final QueryableIndex index) schema.getName(), new AggregateProjectionSpec( schema.getName(), + schema.getFilter(), schema.getVirtualColumns(), columnSchemas, schema.getAggregators() diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java index fcb3a1cf5d6d..118011c2f26e 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java @@ -120,6 +120,7 @@ public static Iterable constructorFeeder() private static final AggregateProjectionSpec PROJECTION_SPEC = new AggregateProjectionSpec( "projection1", + null, VirtualColumns.create( Granularities.toVirtualColumn( Granularities.HOUR, @@ -959,6 +960,7 @@ public void testRunParallelAddProjections() ); final AggregateProjectionSpec addProjection = new AggregateProjectionSpec( "projection2", + null, VirtualColumns.EMPTY, null, new AggregatorFactory[]{ diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java index 783214d77dd7..b0ae79988b3a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java @@ -576,6 +576,7 @@ public void testSerdeWithProjections() throws IOException final List projections = ImmutableList.of( new AggregateProjectionSpec( "test", + null, VirtualColumns.EMPTY, ImmutableList.of( new StringDimensionSchema("dim1"), diff --git a/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java b/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java index 925fd26f4fe1..7539ff3b0f3f 100644 --- a/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java +++ b/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java @@ -31,6 +31,7 @@ import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.OrderBy; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.filter.DimFilter; import org.apache.druid.segment.AggregateProjectionMetadata; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.VirtualColumns; @@ -57,8 +58,10 @@ public class AggregateProjectionSpec public static final String TYPE_NAME = "aggregate"; private final String name; - private final List groupingColumns; + @Nullable + private final DimFilter filter; private final VirtualColumns virtualColumns; + private final List groupingColumns; private final AggregatorFactory[] aggregators; private final List ordering; @Nullable @@ -67,6 +70,7 @@ public class AggregateProjectionSpec @JsonCreator public AggregateProjectionSpec( @JsonProperty("name") String name, + @JsonProperty("filter") @Nullable DimFilter filter, @JsonProperty("virtualColumns") @Nullable VirtualColumns virtualColumns, @JsonProperty("groupingColumns") @Nullable List groupingColumns, @JsonProperty("aggregators") @Nullable AggregatorFactory[] aggregators @@ -77,10 +81,14 @@ public AggregateProjectionSpec( } this.name = name; if (CollectionUtils.isNullOrEmpty(groupingColumns) && (aggregators == null || aggregators.length == 0)) { - throw InvalidInput.exception("projection[%s] groupingColumns and aggregators must not both be null or empty", name); + throw InvalidInput.exception( + "projection[%s] groupingColumns and aggregators must not both be null or empty", + name + ); } - this.groupingColumns = groupingColumns == null ? Collections.emptyList() : groupingColumns; + this.filter = filter; this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns; + this.groupingColumns = groupingColumns == null ? Collections.emptyList() : groupingColumns; // in the future this should be expanded to support user specified ordering, but for now we compute it based on // the grouping columns, which is consistent with how rollup ordering works for incremental index base table final ProjectionOrdering ordering = computeOrdering(this.virtualColumns, this.groupingColumns); @@ -95,6 +103,14 @@ public String getName() return name; } + @Nullable + @JsonProperty + @JsonInclude(JsonInclude.Include.NON_NULL) + public DimFilter getFilter() + { + return filter; + } + @JsonProperty @JsonInclude(JsonInclude.Include.NON_DEFAULT) public VirtualColumns getVirtualColumns() @@ -128,6 +144,7 @@ public AggregateProjectionMetadata.Schema toMetadataSchema() return new AggregateProjectionMetadata.Schema( name, timeColumnName, + filter, virtualColumns, groupingColumns.stream().map(DimensionSchema::getName).collect(Collectors.toList()), aggregators, @@ -146,6 +163,7 @@ public boolean equals(Object o) } AggregateProjectionSpec that = (AggregateProjectionSpec) o; return Objects.equals(name, that.name) + && Objects.equals(filter, that.filter) && Objects.equals(groupingColumns, that.groupingColumns) && Objects.equals(virtualColumns, that.virtualColumns) && Objects.deepEquals(aggregators, that.aggregators) @@ -155,7 +173,7 @@ public boolean equals(Object o) @Override public int hashCode() { - return Objects.hash(name, groupingColumns, virtualColumns, Arrays.hashCode(aggregators), ordering); + return Objects.hash(name, filter, virtualColumns, groupingColumns, Arrays.hashCode(aggregators), ordering); } @Override @@ -163,8 +181,9 @@ public String toString() { return "AggregateProjectionSpec{" + "name='" + name + '\'' + - ", groupingColumns=" + groupingColumns + + ", filter=" + filter + ", virtualColumns=" + virtualColumns + + ", groupingColumns=" + groupingColumns + ", aggregators=" + Arrays.toString(aggregators) + ", ordering=" + ordering + '}'; diff --git a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java index 83b762b629dc..840766de94f2 100644 --- a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java +++ b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java @@ -34,7 +34,12 @@ import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.OrderBy; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.filter.DimFilter; +import org.apache.druid.query.filter.Filter; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.filter.AndFilter; +import org.apache.druid.segment.filter.IsBooleanFilter; +import org.apache.druid.segment.filter.TrueFilter; import org.apache.druid.segment.projections.Projections; import org.apache.druid.utils.CollectionUtils; @@ -42,7 +47,9 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; @@ -59,6 +66,7 @@ public class AggregateProjectionMetadata { private static final Interner SCHEMA_INTERNER = Interners.newWeakInterner(); + public static final Comparator COMPARATOR = (o1, o2) -> { int rowCompare = Integer.compare(o1.numRows, o2.numRows); if (rowCompare != 0) { @@ -130,10 +138,10 @@ public static class Schema */ public static final Comparator COMPARATOR = (o1, o2) -> { // coarsest granularity first - if (o1.getGranularity().isFinerThan(o2.getGranularity())) { + if (o1.getEffectiveGranularity().isFinerThan(o2.getEffectiveGranularity())) { return 1; } - if (o2.getGranularity().isFinerThan(o1.getGranularity())) { + if (o2.getEffectiveGranularity().isFinerThan(o1.getEffectiveGranularity())) { return -1; } // fewer dimensions first @@ -163,6 +171,8 @@ public static class Schema private final String name; @Nullable private final String timeColumnName; + @Nullable + private final DimFilter filter; private final VirtualColumns virtualColumns; private final List groupingColumns; private final AggregatorFactory[] aggregators; @@ -171,12 +181,13 @@ public static class Schema // computed fields private final int timeColumnPosition; - private final Granularity granularity; + private final Granularity effectiveGranularity; @JsonCreator public Schema( @JsonProperty("name") String name, @JsonProperty("timeColumnName") @Nullable String timeColumnName, + @JsonProperty("filter") @Nullable DimFilter filter, @JsonProperty("virtualColumns") @Nullable VirtualColumns virtualColumns, @JsonProperty("groupingColumns") @Nullable List groupingColumns, @JsonProperty("aggregators") @Nullable AggregatorFactory[] aggregators, @@ -188,11 +199,15 @@ public Schema( } this.name = name; if (CollectionUtils.isNullOrEmpty(groupingColumns) && (aggregators == null || aggregators.length == 0)) { - throw DruidException.defensive("projection schema[%s] groupingColumns and aggregators must not both be null or empty", name); + throw DruidException.defensive( + "projection schema[%s] groupingColumns and aggregators must not both be null or empty", + name + ); } if (ordering == null) { throw DruidException.defensive("projection schema[%s] ordering must not be null", name); } + this.filter = filter; this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns; this.groupingColumns = groupingColumns == null ? Collections.emptyList() : groupingColumns; this.aggregators = aggregators == null ? new AggregatorFactory[0] : aggregators; @@ -219,7 +234,7 @@ public Schema( } this.timeColumnName = timeColumnName; this.timeColumnPosition = foundTimePosition; - this.granularity = granularity == null ? Granularities.ALL : granularity; + this.effectiveGranularity = granularity == null ? Granularities.ALL : granularity; } @JsonProperty @@ -235,6 +250,13 @@ public String getTimeColumnName() return timeColumnName; } + @JsonProperty + @Nullable + public DimFilter getFilter() + { + return filter; + } + @JsonProperty @JsonInclude(JsonInclude.Include.NON_DEFAULT) public VirtualColumns getVirtualColumns() @@ -275,9 +297,9 @@ public int getTimeColumnPosition() } @JsonIgnore - public Granularity getGranularity() + public Granularity getEffectiveGranularity() { - return granularity; + return effectiveGranularity; } /** @@ -396,6 +418,45 @@ public Projections.ProjectionMatch matches( } } } + + // if the projection has a filter, the query must contain this filter match + if (filter != null) { + final Filter queryFilter = queryCursorBuildSpec.getFilter(); + if (queryFilter != null) { + // try to rewrite the query filter into a projection filter, if the rewrite is valid, we can proceed + final Filter projectionFilter = filter.toOptimizedFilter(false); + final Map filterRewrites = new HashMap<>(); + // start with identity + for (String required : queryFilter.getRequiredColumns()) { + filterRewrites.put(required, required); + } + // overlay projection rewrites + filterRewrites.putAll(matchBuilder.getRemapColumns()); + + final Filter remappedQueryFilter = queryFilter.rewriteRequiredColumns(filterRewrites); + + final Filter rewritten = rewriteFilter(projectionFilter, remappedQueryFilter); + // if the filter does not contain the projection filter, we cannot match this projection + if (rewritten == null) { + return null; + } + //noinspection ObjectEquality + if (rewritten == ProjectionFilterMatch.INSTANCE) { + // we can remove the whole thing since the query filter exactly matches the projection filter + matchBuilder.rewriteFilter(null); + } else { + // otherwise, we partially rewrote the query filter to eliminate the projection filter since it is baked in + matchBuilder.rewriteFilter(rewritten); + } + } else { + // projection has a filter, but the query doesn't, no good + return null; + } + } else { + // projection doesn't have a filter, retain the original + matchBuilder.rewriteFilter(queryCursorBuildSpec.getFilter()); + } + return matchBuilder.build(queryCursorBuildSpec); } @@ -457,18 +518,18 @@ private Projections.ProjectionMatchBuilder matchRequiredColumn( // virtual column and underlying expression itself, but this will do for now final Granularity virtualGranularity = Granularities.fromVirtualColumn(queryVirtualColumn); if (virtualGranularity != null) { - if (virtualGranularity.isFinerThan(granularity)) { + if (virtualGranularity.isFinerThan(effectiveGranularity)) { return null; } // same granularity, replace virtual column directly by remapping it to the physical column - if (granularity.equals(virtualGranularity)) { + if (effectiveGranularity.equals(virtualGranularity)) { return matchBuilder.remapColumn(column, ColumnHolder.TIME_COLUMN_NAME) .addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); } return matchBuilder.addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); } else { // anything else with __time requires none granularity - if (Granularities.NONE.equals(granularity)) { + if (Granularities.NONE.equals(effectiveGranularity)) { return matchBuilder.addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); } return null; @@ -524,6 +585,7 @@ public boolean equals(Object o) Schema schema = (Schema) o; return Objects.equals(name, schema.name) && Objects.equals(timeColumnName, schema.timeColumnName) + && Objects.equals(filter, schema.filter) && Objects.equals(virtualColumns, schema.virtualColumns) && Objects.equals(groupingColumns, schema.groupingColumns) && Objects.deepEquals(aggregators, schema.aggregators) @@ -536,6 +598,7 @@ public int hashCode() return Objects.hash( name, timeColumnName, + filter, virtualColumns, groupingColumns, Arrays.hashCode(aggregators), @@ -554,9 +617,73 @@ public String toString() ", aggregators=" + Arrays.toString(aggregators) + ", ordering=" + ordering + ", timeColumnPosition=" + timeColumnPosition + - ", granularity=" + granularity + + ", effectiveGranularity=" + effectiveGranularity + ", orderingWithTimeSubstitution=" + orderingWithTimeSubstitution + '}'; } } + + /** + * Rewrites a query {@link Filter} if possible, removing the {@link Filter} of a projection. To match a projection + * filter, the query filter must be equal to the projection filter, or must contain the projection filter as the child + * of an AND filter. This method returns null + * indicating that a rewrite is impossible with the implication that the query cannot use the projection because the + * projection doesn't contain all the rows the query would match if not using the projection. + */ + @Nullable + public static Filter rewriteFilter(@Nullable Filter projectionFilter, @Nullable Filter queryFilter) + { + if (projectionFilter == null || queryFilter == null) { + return queryFilter; + } + if (queryFilter.equals(projectionFilter)) { + return ProjectionFilterMatch.INSTANCE; + } + if (queryFilter instanceof IsBooleanFilter && ((IsBooleanFilter) queryFilter).isTrue()) { + final IsBooleanFilter isTrueFilter = (IsBooleanFilter) queryFilter; + final Filter rewritten = rewriteFilter(projectionFilter, isTrueFilter.getBaseFilter()); + if (rewritten == null) { + return null; + } + //noinspection ObjectEquality + if (rewritten == ProjectionFilterMatch.INSTANCE) { + return ProjectionFilterMatch.INSTANCE; + } + return new IsBooleanFilter(rewritten, true); + } + if (queryFilter instanceof AndFilter) { + AndFilter andFilter = (AndFilter) queryFilter; + List newChildren = Lists.newArrayListWithExpectedSize(andFilter.getFilters().size()); + boolean childRewritten = false; + for (Filter filter : andFilter.getFilters()) { + Filter rewritten = rewriteFilter(projectionFilter, filter); + //noinspection ObjectEquality + if (rewritten == ProjectionFilterMatch.INSTANCE) { + childRewritten = true; + } else { + if (rewritten != null) { + newChildren.add(rewritten); + childRewritten = true; + } else { + newChildren.add(filter); + } + } + } + // at least one child must have been rewritten to rewrite the AND + if (childRewritten) { + if (newChildren.size() > 1) { + return new AndFilter(newChildren); + } else { + return newChildren.get(0); + } + } + return null; + } + return null; + } + + static final class ProjectionFilterMatch extends TrueFilter + { + private static final ProjectionFilterMatch INSTANCE = new ProjectionFilterMatch(); + } } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java index 813d4ac2f5ae..792b305cccb7 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java @@ -62,6 +62,16 @@ public IsBooleanFilter(Filter baseFilter, boolean isTrue) this.isTrue = isTrue; } + public boolean isTrue() + { + return isTrue; + } + + public Filter getBaseFilter() + { + return baseFilter; + } + @Nullable @Override public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector) diff --git a/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java index 3dcf59ee47eb..69bad38af069 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java @@ -46,7 +46,7 @@ public static TrueFilter instance() return INSTANCE; } - private TrueFilter() + protected TrueFilter() { } diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/OnHeapAggregateProjection.java b/processing/src/main/java/org/apache/druid/segment/incremental/OnHeapAggregateProjection.java index ed104ef1ddfc..5ac8f4a71748 100644 --- a/processing/src/main/java/org/apache/druid/segment/incremental/OnHeapAggregateProjection.java +++ b/processing/src/main/java/org/apache/druid/segment/incremental/OnHeapAggregateProjection.java @@ -30,11 +30,14 @@ import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorAndSize; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.segment.AggregateProjectionMetadata; import org.apache.druid.segment.AutoTypeColumnIndexer; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.EncodedKeyComponent; +import org.apache.druid.segment.RowAdapters; +import org.apache.druid.segment.RowBasedColumnSelectorFactory; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.CapabilitiesBasedFormat; @@ -43,6 +46,7 @@ import org.apache.druid.segment.column.ColumnFormat; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; @@ -78,6 +82,8 @@ public class OnHeapAggregateProjection implements IncrementalIndexRowSelector private final long minTimestamp; private final AtomicInteger rowCounter = new AtomicInteger(0); private final AtomicInteger numEntries = new AtomicInteger(0); + @Nullable + private final ValueMatcher valueMatcher; public OnHeapAggregateProjection( AggregateProjectionSpec projectionSpec, @@ -121,6 +127,33 @@ public OnHeapAggregateProjection( this.aggregatorsMap = new LinkedHashMap<>(); this.aggregatorFactories = new AggregatorFactory[projectionSchema.getAggregators().length]; initializeAndValidateAggregators(projectionSpec, getBaseTableDimensionDesc, getBaseTableAggregatorFactory); + + if (projectionSpec.getFilter() != null) { + RowSignature.Builder bob = RowSignature.builder(); + if (projectionSchema.getTimeColumnPosition() < 0) { + bob.addTimeColumn(); + } + for (String groupingColumn : projectionSchema.getGroupingColumns()) { + if (projectionSchema.getTimeColumnName().equals(groupingColumn)) { + bob.addTimeColumn(); + } else { + bob.add(groupingColumn, dimensionsMap.get(groupingColumn).getCapabilities().toColumnType()); + } + } + valueMatcher = projectionSchema.getFilter() + .toFilter() + .makeMatcher( + RowBasedColumnSelectorFactory.create( + RowAdapters.standardRow(), + inputRowHolder::getRow, + bob.build(), + false, + false + ) + ); + } else { + valueMatcher = null; + } } /** @@ -134,6 +167,9 @@ public void addToFacts( ) { inputRowHolder.set(inputRow); + if (valueMatcher != null && !valueMatcher.matches(false)) { + return; + } final Object[] projectionDims = new Object[dimensions.size()]; for (int i = 0; i < projectionDims.length; i++) { int parentDimIndex = parentDimensionIndex[i]; @@ -153,7 +189,7 @@ public void addToFacts( final long timestamp; if (projectionSchema.getTimeColumnName() != null) { - timestamp = projectionSchema.getGranularity().bucketStart(DateTimes.utc(key.getTimestamp())).getMillis(); + timestamp = projectionSchema.getEffectiveGranularity().bucketStart(DateTimes.utc(key.getTimestamp())).getMillis(); if (timestamp < minTimestamp) { throw DruidException.defensive( "Cannot add row[%s] to projection[%s] because projection effective timestamp[%s] is below the minTimestamp[%s]", diff --git a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java index db067bde85b7..081ece8b72fa 100644 --- a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java +++ b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java @@ -26,6 +26,7 @@ import org.apache.druid.math.expr.ExprEval; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.filter.Filter; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.AggregateProjectionMetadata; import org.apache.druid.segment.ColumnValueSelector; @@ -183,6 +184,8 @@ public static final class ProjectionMatchBuilder private final Map remapColumns; private final List combiningFactories; private final Set matchedQueryColumns; + @Nullable + private Filter rewriteFilter; public ProjectionMatchBuilder() { @@ -250,6 +253,17 @@ public ProjectionMatchBuilder addMatchedQueryColumns(Collection queryCol return this; } + public ProjectionMatchBuilder rewriteFilter(Filter rewriteFilter) + { + this.rewriteFilter = rewriteFilter; + return this; + } + + public Map getRemapColumns() + { + return remapColumns; + } + public Set getMatchedQueryColumns() { return matchedQueryColumns; @@ -259,6 +273,7 @@ public ProjectionMatch build(CursorBuildSpec queryCursorBuildSpec) { return new ProjectionMatch( CursorBuildSpec.builder(queryCursorBuildSpec) + .setFilter(rewriteFilter) .setPhysicalColumns(referencedPhysicalColumns) .setVirtualColumns(VirtualColumns.fromIterable(referencedVirtualColumns)) .setAggregators(combiningFactories) diff --git a/processing/src/test/java/org/apache/druid/data/input/impl/AggregateProjectionSpecTest.java b/processing/src/test/java/org/apache/druid/data/input/impl/AggregateProjectionSpecTest.java index dcb47416321b..7c43e8cf4adf 100644 --- a/processing/src/test/java/org/apache/druid/data/input/impl/AggregateProjectionSpecTest.java +++ b/processing/src/test/java/org/apache/druid/data/input/impl/AggregateProjectionSpecTest.java @@ -27,8 +27,10 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.filter.EqualityFilter; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.VirtualColumns; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -46,6 +48,7 @@ void testSerde() throws JsonProcessingException { AggregateProjectionSpec spec = new AggregateProjectionSpec( "some_projection", + new EqualityFilter("a", ColumnType.STRING, "a", null), VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, "time") ), @@ -69,6 +72,7 @@ void testComputeOrdering_noOrdering() { AggregateProjectionSpec spec = new AggregateProjectionSpec( "some_projection", + null, VirtualColumns.EMPTY, List.of(), new AggregatorFactory[] { @@ -85,6 +89,7 @@ void testMissingName() Throwable t = Assertions.assertThrows( DruidException.class, () -> new AggregateProjectionSpec( + null, null, VirtualColumns.EMPTY, List.of(new StringDimensionSchema("string")), @@ -98,6 +103,7 @@ void testMissingName() DruidException.class, () -> new AggregateProjectionSpec( "", + null, VirtualColumns.EMPTY, List.of(new StringDimensionSchema("string")), null @@ -115,6 +121,7 @@ void testInvalidGrouping() "other_projection", null, null, + null, null ) ); @@ -125,6 +132,7 @@ void testInvalidGrouping() () -> new AggregateProjectionSpec( "other_projection", null, + null, Collections.emptyList(), null ) diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java index 56800c4c3c41..5096fe0c883f 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java @@ -86,6 +86,7 @@ public void testSerde() throws Exception new AggregateProjectionMetadata.Schema( "channel_added_hourly", Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, + null, VirtualColumns.create( Granularities.toVirtualColumn( Granularities.HOUR, diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java index 69be9165ac9d..9cdabc3d1e14 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java @@ -73,6 +73,7 @@ public class SegmentMetadataQueryQueryToolChestTest private static final AggregateProjectionMetadata.Schema PROJECTION_CHANNEL_ADDED_HOURLY = new AggregateProjectionMetadata.Schema( "name1-does-not-matter", Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, + null, VirtualColumns.create(Granularities.toVirtualColumn( Granularities.HOUR, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME @@ -89,6 +90,7 @@ public class SegmentMetadataQueryQueryToolChestTest private static final AggregateProjectionMetadata.Schema PROJECTION_CHANNEL_ADDED_DAILY = new AggregateProjectionMetadata.Schema( "name2-does-not-matter", Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, + null, VirtualColumns.create(Granularities.toVirtualColumn( Granularities.DAY, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME diff --git a/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java b/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java index 8e025600686b..238477866bc7 100644 --- a/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java +++ b/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java @@ -32,7 +32,13 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.filter.EqualityFilter; +import org.apache.druid.query.filter.Filter; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.filter.AndFilter; +import org.apache.druid.segment.filter.IsBooleanFilter; +import org.apache.druid.segment.filter.OrFilter; import org.apache.druid.segment.projections.Projections; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.jupiter.api.Assertions; @@ -54,6 +60,7 @@ void testSerde() throws JsonProcessingException new AggregateProjectionMetadata.Schema( "some_projection", "time", + new EqualityFilter("a", ColumnType.STRING, "a", null), VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, "time") ), @@ -87,6 +94,7 @@ void testComparator() new AggregateProjectionMetadata.Schema( "good", "theTime", + null, VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "theTime")), Arrays.asList("theTime", "a", "b", "c"), new AggregatorFactory[]{new CountAggregatorFactory("chocula")}, @@ -104,6 +112,7 @@ void testComparator() new AggregateProjectionMetadata.Schema( "betterLessGroupingColumns", "theTime", + null, VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "theTime")), Arrays.asList("c", "d", "theTime"), new AggregatorFactory[]{new CountAggregatorFactory("chocula")}, @@ -120,6 +129,7 @@ void testComparator() new AggregateProjectionMetadata.Schema( "evenBetterMoreAggs", "theTime", + null, VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "theTime")), Arrays.asList("c", "d", "theTime"), new AggregatorFactory[]{ @@ -139,6 +149,7 @@ void testComparator() new AggregateProjectionMetadata.Schema( "best", null, + null, VirtualColumns.EMPTY, Arrays.asList("f", "g"), new AggregatorFactory[0], @@ -168,6 +179,7 @@ void testInvalidName() null, null, null, + null, new AggregatorFactory[]{new CountAggregatorFactory("count")}, List.of(OrderBy.ascending(ColumnHolder.TIME_COLUMN_NAME), OrderBy.ascending("count")) ), @@ -187,6 +199,7 @@ void testInvalidName() null, null, null, + null, new AggregatorFactory[]{new CountAggregatorFactory("count")}, List.of(OrderBy.ascending(ColumnHolder.TIME_COLUMN_NAME), OrderBy.ascending("count")) ), @@ -211,6 +224,7 @@ void testInvalidGrouping() null, null, null, + null, null ), 0 @@ -228,6 +242,7 @@ void testInvalidGrouping() "other_projection", null, null, + null, Collections.emptyList(), null, null @@ -252,6 +267,7 @@ void testInvalidOrdering() null, null, null, + null, new AggregatorFactory[]{new CountAggregatorFactory("count")}, null ), @@ -271,6 +287,7 @@ void testInvalidOrdering() null, null, null, + null, new AggregatorFactory[]{new CountAggregatorFactory("count")}, List.of(OrderBy.ascending(ColumnHolder.TIME_COLUMN_NAME), OrderBy.ascending("count")) ), @@ -293,19 +310,20 @@ void testEqualsAndHashcode() void testEqualsAndHashcodeSchema() { EqualsVerifier.forClass(AggregateProjectionMetadata.Schema.class) - .withIgnoredFields("orderingWithTimeSubstitution", "timeColumnPosition", "granularity") + .withIgnoredFields("orderingWithTimeSubstitution", "timeColumnPosition", "effectiveGranularity") .usingGetClass() .verify(); } @Test - public void testSchemaMatchSimple() + void testSchemaMatchSimple() { // arrange AggregateProjectionMetadata spec = new AggregateProjectionMetadata( new AggregateProjectionMetadata.Schema( "some_projection", null, + null, VirtualColumns.EMPTY, Arrays.asList("a", "b"), new AggregatorFactory[]{new LongSumAggregatorFactory("a_projection", "a")}, @@ -333,4 +351,44 @@ public void testSchemaMatchSimple() ); Assertions.assertEquals(expected, projectionMatch); } + + @Test + void testRewriteFilter() + { + Filter xeqfoo = new EqualityFilter("x", ColumnType.STRING, "foo", null); + Filter xeqfoo2 = new EqualityFilter("x", ColumnType.STRING, "foo", null); + Filter xeqbar = new EqualityFilter("x", ColumnType.STRING, "bar", null); + Filter yeqbar = new EqualityFilter("y", ColumnType.STRING, "bar", null); + Filter zeq123 = new EqualityFilter("z", ColumnType.LONG, 123L, null); + + Filter queryFilter = xeqfoo2; + Assertions.assertInstanceOf( + AggregateProjectionMetadata.ProjectionFilterMatch.class, + AggregateProjectionMetadata.rewriteFilter(xeqfoo, queryFilter) + ); + + queryFilter = yeqbar; + Assertions.assertNull(AggregateProjectionMetadata.rewriteFilter(xeqfoo, queryFilter)); + + queryFilter = new AndFilter(List.of(xeqfoo, yeqbar)); + Assertions.assertEquals( + yeqbar, + AggregateProjectionMetadata.rewriteFilter(xeqfoo, queryFilter) + ); + + queryFilter = new AndFilter(List.of(new OrFilter(List.of(xeqfoo, xeqbar)), yeqbar)); + Assertions.assertNull(AggregateProjectionMetadata.rewriteFilter(xeqfoo, queryFilter)); + + queryFilter = new AndFilter(List.of(new IsBooleanFilter(xeqfoo, true), yeqbar)); + Assertions.assertEquals(yeqbar, AggregateProjectionMetadata.rewriteFilter(xeqfoo, queryFilter)); + + queryFilter = new AndFilter(List.of(new IsBooleanFilter(xeqfoo, false), yeqbar)); + Assertions.assertNull(AggregateProjectionMetadata.rewriteFilter(xeqfoo, queryFilter)); + + queryFilter = new AndFilter(List.of(new AndFilter(List.of(xeqfoo, yeqbar)), zeq123)); + Assertions.assertEquals( + new AndFilter(List.of(yeqbar, zeq123)), + AggregateProjectionMetadata.rewriteFilter(xeqfoo, queryFilter) + ); + } } diff --git a/processing/src/test/java/org/apache/druid/segment/CursorFactoryProjectionTest.java b/processing/src/test/java/org/apache/druid/segment/CursorFactoryProjectionTest.java index 32f695b76b1b..5eb949b6645f 100644 --- a/processing/src/test/java/org/apache/druid/segment/CursorFactoryProjectionTest.java +++ b/processing/src/test/java/org/apache/druid/segment/CursorFactoryProjectionTest.java @@ -169,6 +169,7 @@ public static List makeRows(List dimensions) private static final List PROJECTIONS = Arrays.asList( new AggregateProjectionSpec( "ab_hourly_cd_sum", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, "__gran") ), @@ -184,6 +185,7 @@ public static List makeRows(List dimensions) ), new AggregateProjectionSpec( "a_hourly_c_sum_with_count_latest", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, "__gran") ), @@ -199,6 +201,7 @@ public static List makeRows(List dimensions) ), new AggregateProjectionSpec( "b_hourly_c_sum_non_time_ordered", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, "__gran") ), @@ -214,6 +217,7 @@ public static List makeRows(List dimensions) ), new AggregateProjectionSpec( "bf_daily_c_sum", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.DAY, "__gran") ), @@ -228,6 +232,7 @@ public static List makeRows(List dimensions) ), new AggregateProjectionSpec( "b_c_sum", + null, VirtualColumns.EMPTY, List.of(new StringDimensionSchema("b")), new AggregatorFactory[]{ @@ -237,6 +242,7 @@ public static List makeRows(List dimensions) new AggregateProjectionSpec( "ab", null, + null, Arrays.asList( new StringDimensionSchema("a"), new StringDimensionSchema("b") @@ -245,6 +251,7 @@ public static List makeRows(List dimensions) ), new AggregateProjectionSpec( "abfoo", + null, VirtualColumns.create( new ExpressionVirtualColumn( "bfoo", @@ -261,6 +268,7 @@ public static List makeRows(List dimensions) ), new AggregateProjectionSpec( "c_sum_daily", + null, VirtualColumns.create(Granularities.toVirtualColumn(Granularities.DAY, "__gran")), Collections.singletonList(new LongDimensionSchema("__gran")), new AggregatorFactory[]{ @@ -269,6 +277,7 @@ public static List makeRows(List dimensions) ), new AggregateProjectionSpec( "c_sum", + null, VirtualColumns.EMPTY, Collections.emptyList(), new AggregatorFactory[]{ @@ -277,17 +286,35 @@ public static List makeRows(List dimensions) ), new AggregateProjectionSpec( "missing_column", + null, VirtualColumns.EMPTY, List.of(new StringDimensionSchema("missing")), new AggregatorFactory[]{ new LongSumAggregatorFactory("csum", "c") } + ), + new AggregateProjectionSpec( + "ab_filter_baaonly_hourly_cd_sum", + new EqualityFilter("b", ColumnType.STRING, "aa", null), + VirtualColumns.create( + Granularities.toVirtualColumn(Granularities.HOUR, "__gran") + ), + Arrays.asList( + new StringDimensionSchema("a"), + new StringDimensionSchema("b"), + new LongDimensionSchema("__gran") + ), + new AggregatorFactory[]{ + new LongSumAggregatorFactory("_c_sum", "c"), + new DoubleSumAggregatorFactory("d", "d") + } ) ); private static final List ROLLUP_PROJECTIONS = Arrays.asList( new AggregateProjectionSpec( "a_hourly_c_sum_with_count", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, "__gran") ), @@ -302,6 +329,7 @@ public static List makeRows(List dimensions) ), new AggregateProjectionSpec( "afoo", + null, VirtualColumns.create( new ExpressionVirtualColumn( "afoo", @@ -323,6 +351,7 @@ public static List makeRows(List dimensions) PROJECTIONS.stream() .map(projection -> new AggregateProjectionSpec( projection.getName(), + projection.getFilter(), projection.getVirtualColumns(), projection.getGroupingColumns() .stream() @@ -336,6 +365,7 @@ public static List makeRows(List dimensions) ROLLUP_PROJECTIONS.stream() .map(projection -> new AggregateProjectionSpec( projection.getName(), + projection.getFilter(), projection.getVirtualColumns(), projection.getGroupingColumns() .stream() @@ -1686,6 +1716,46 @@ public void testProjectionSingleDimVirtualColumnRollupTable() } } + + @Test + public void testProjectionFilter() + { + // since d isn't present on the smaller projection, cant use it, but can still use the larger projection + final GroupByQuery query = + GroupByQuery.builder() + .setDataSource("test") + .setGranularity(Granularities.ALL) + .setInterval(Intervals.ETERNITY) + .addDimension("a") + .setDimFilter(new EqualityFilter("b", ColumnType.STRING, "aa", null)) + .addAggregator(new LongSumAggregatorFactory("c_sum", "c")) + .addAggregator(new DoubleSumAggregatorFactory("d_sum", "d")) + .build(); + final CursorBuildSpec buildSpec = GroupingEngine.makeCursorBuildSpec(query, null); + try (final CursorHolder cursorHolder = projectionsCursorFactory.makeCursorHolder(buildSpec)) { + final Cursor cursor = cursorHolder.asCursor(); + int rowCount = 0; + while (!cursor.isDone()) { + rowCount++; + cursor.advance(); + } + Assert.assertEquals(3, rowCount); + } + + final Sequence resultRows = groupingEngine.process( + query, + projectionsCursorFactory, + projectionsTimeBoundaryInspector, + nonBlockingPool, + null + ); + + final List results = resultRows.toList(); + Assert.assertEquals(2, results.size()); + Assert.assertArrayEquals(new Object[]{"a", 2L, 2.1}, results.get(0).getArray()); + Assert.assertArrayEquals(new Object[]{"b", 7L, 7.7}, results.get(1).getArray()); + } + private static IndexBuilder makeBuilder(DimensionsSpec dimensionsSpec, boolean autoSchema, boolean writeNullColumns) { File tmp = FileUtils.createTempDir(); diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java index 87959a63b6b2..19439bfcb29d 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java @@ -3013,6 +3013,7 @@ public void testMergeProjections() throws IOException List projections = Arrays.asList( new AggregateProjectionSpec( "a_hourly_c_sum", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, "__gran") ), @@ -3026,6 +3027,7 @@ public void testMergeProjections() throws IOException ), new AggregateProjectionSpec( "a_c_sum", + null, VirtualColumns.EMPTY, Collections.singletonList( new StringDimensionSchema("a") diff --git a/processing/src/test/java/org/apache/druid/segment/MetadataTest.java b/processing/src/test/java/org/apache/druid/segment/MetadataTest.java index 23318406bb47..b5cc7621e764 100644 --- a/processing/src/test/java/org/apache/druid/segment/MetadataTest.java +++ b/processing/src/test/java/org/apache/druid/segment/MetadataTest.java @@ -104,6 +104,7 @@ public void testMerge() new AggregateProjectionMetadata.Schema( "some_projection", "__gran", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, "__gran") ), @@ -282,6 +283,7 @@ public void testMergeProjectionsUnexpectedMismatch() new AggregateProjectionMetadata.Schema( "some_projection", "__gran", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, "__gran") ), @@ -300,6 +302,7 @@ public void testMergeProjectionsUnexpectedMismatch() new AggregateProjectionMetadata.Schema( "some_projection", "__gran", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, "__gran") ), @@ -318,6 +321,7 @@ public void testMergeProjectionsUnexpectedMismatch() new AggregateProjectionMetadata.Schema( "some_projection", "__gran", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, "__gran") ), @@ -333,6 +337,7 @@ public void testMergeProjectionsUnexpectedMismatch() new AggregateProjectionMetadata.Schema( "some_projection2", "__gran", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.DAY, "__gran") ), diff --git a/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorHolderTest.java b/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorHolderTest.java index d0be594844dd..8ac1a26bb4f8 100644 --- a/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorHolderTest.java +++ b/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorHolderTest.java @@ -184,6 +184,7 @@ public void testProjectionTimeBoundaryInspector() Collections.singletonList( new AggregateProjectionSpec( "ab_hourly_cd_sum_time_ordered", + null, VirtualColumns.create( Granularities.toVirtualColumn( Granularities.HOUR, diff --git a/processing/src/test/java/org/apache/druid/segment/TestIndex.java b/processing/src/test/java/org/apache/druid/segment/TestIndex.java index 712d5d950027..c5bc6c7041af 100644 --- a/processing/src/test/java/org/apache/druid/segment/TestIndex.java +++ b/processing/src/test/java/org/apache/druid/segment/TestIndex.java @@ -188,12 +188,14 @@ public class TestIndex public static final ImmutableList PROJECTIONS = ImmutableList.of( new AggregateProjectionSpec( "daily_market_maxQuality", + null, VirtualColumns.create(Granularities.toVirtualColumn(Granularities.DAY, "__gran")), List.of(new LongDimensionSchema("__gran"), new StringDimensionSchema("market")), new AggregatorFactory[]{new LongMaxAggregatorFactory("maxQuality", "qualityLong")} ), new AggregateProjectionSpec( "daily_countAndQualityCardinalityAndMaxLongNullable", + null, VirtualColumns.create(Granularities.toVirtualColumn(Granularities.DAY, "__gran")), List.of(new LongDimensionSchema("__gran")), new AggregatorFactory[]{ diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/OnheapIncrementalIndexTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/OnheapIncrementalIndexTest.java index 3ec43db9054c..02d93e0afed3 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/OnheapIncrementalIndexTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/OnheapIncrementalIndexTest.java @@ -73,6 +73,7 @@ public void testProjectionHappyPath() AggregatorFactory aggregatorFactory = new DoubleSumAggregatorFactory("double", "double"); AggregateProjectionSpec projectionSpec = new AggregateProjectionSpec( "proj", + null, VirtualColumns.EMPTY, ImmutableList.of(new StringDimensionSchema("string")), new AggregatorFactory[]{ @@ -100,12 +101,14 @@ public void testProjectionDuplicatedName() AggregatorFactory aggregatorFactory = new DoubleSumAggregatorFactory("double", "double"); AggregateProjectionSpec projectionSpec1 = new AggregateProjectionSpec( "proj", + null, VirtualColumns.EMPTY, ImmutableList.of(), new AggregatorFactory[]{new DoubleSumAggregatorFactory("double", "double")} ); AggregateProjectionSpec projectionSpec2 = new AggregateProjectionSpec( "proj", + null, VirtualColumns.EMPTY, ImmutableList.of(), new AggregatorFactory[]{new DoubleSumAggregatorFactory("double", "double")} @@ -160,6 +163,7 @@ public void testBadProjectionMismatchedDimensionTypes() ImmutableList.of( new AggregateProjectionSpec( "mismatched dims", + null, VirtualColumns.EMPTY, ImmutableList.of( new LongDimensionSchema("string") @@ -200,6 +204,7 @@ public void testBadProjectionDimensionNoVirtualColumnOrBaseTable() ImmutableList.of( new AggregateProjectionSpec( "sad grouping column", + null, VirtualColumns.create( new ExpressionVirtualColumn( "v0", @@ -248,6 +253,7 @@ public void testBadProjectionVirtualColumnNoDimension() ImmutableList.of( new AggregateProjectionSpec( "sad virtual column", + null, VirtualColumns.create( new ExpressionVirtualColumn( "v0", @@ -299,6 +305,7 @@ public void testBadProjectionRollupMismatchedAggType() ImmutableList.of( new AggregateProjectionSpec( "mismatched agg", + null, VirtualColumns.EMPTY, ImmutableList.of( new StringDimensionSchema("string") @@ -348,6 +355,7 @@ public void testBadProjectionRollupBadAggInput() ImmutableList.of( new AggregateProjectionSpec( "renamed agg", + null, VirtualColumns.EMPTY, ImmutableList.of( new StringDimensionSchema("string") @@ -391,6 +399,7 @@ public void testBadProjectionVirtualColumnAggInput() ImmutableList.of( new AggregateProjectionSpec( "sad agg virtual column", + null, VirtualColumns.create( new ExpressionVirtualColumn( "v0", @@ -431,6 +440,7 @@ public void testTimestampOutOfRange() AggregatorFactory aggregatorFactory = new DoubleSumAggregatorFactory("double", "double"); AggregateProjectionSpec projectionSpec = new AggregateProjectionSpec( "proj", + null, VirtualColumns.EMPTY, ImmutableList.of(new StringDimensionSchema("string")), new AggregatorFactory[]{ @@ -491,6 +501,7 @@ public void testTimestampOutOfRange() AggregateProjectionSpec projectionSpecYear = new AggregateProjectionSpec( "proj", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.YEAR, "g") ), diff --git a/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java b/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java index 169d5a309495..782ed8580b71 100644 --- a/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java +++ b/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java @@ -464,7 +464,7 @@ public static void validateProjections( if (schema.getTimeColumnName() == null) { continue; } - final Granularity projectionGranularity = schema.getGranularity(); + final Granularity projectionGranularity = schema.getEffectiveGranularity(); if (segmentGranularity != null) { if (segmentGranularity.isFinerThan(projectionGranularity)) { throw InvalidInput.exception( diff --git a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java index 360fcf455fa3..8e2fd5dde90a 100644 --- a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java +++ b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java @@ -639,6 +639,7 @@ public void testSerdeWithProjections() throws Exception AggregateProjectionSpec projectionSpec = new AggregateProjectionSpec( "ab_count_projection", null, + null, Arrays.asList( new StringDimensionSchema("a"), new LongDimensionSchema("b") @@ -878,12 +879,14 @@ void testInvalidProjectionDupeNames() List.of( new AggregateProjectionSpec( "some projection", + null, VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "g")), List.of(new LongDimensionSchema("g")), new AggregatorFactory[]{new CountAggregatorFactory("count")} ), new AggregateProjectionSpec( "some projection", + null, VirtualColumns.create(Granularities.toVirtualColumn(Granularities.MINUTE, "g")), List.of(new LongDimensionSchema("g")), new AggregatorFactory[]{new CountAggregatorFactory("count")} @@ -918,24 +921,28 @@ void testInvalidProjectionGranularity() List.of( new AggregateProjectionSpec( "ok granularity", + null, VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "g")), List.of(new LongDimensionSchema("g")), new AggregatorFactory[]{new CountAggregatorFactory("count")} ), new AggregateProjectionSpec( "acceptable granularity", + null, VirtualColumns.create(Granularities.toVirtualColumn(Granularities.MINUTE, "g")), List.of(new LongDimensionSchema("g")), new AggregatorFactory[]{new CountAggregatorFactory("count")} ), new AggregateProjectionSpec( "not having a time column is ok too", + null, VirtualColumns.EMPTY, null, new AggregatorFactory[]{new CountAggregatorFactory("count")} ), new AggregateProjectionSpec( "bad granularity", + null, VirtualColumns.create(Granularities.toVirtualColumn(Granularities.DAY, "g")), List.of(new LongDimensionSchema("g")), new AggregatorFactory[]{new CountAggregatorFactory("count")} diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java index 41d9a8d3d5fb..df71bf21f4ca 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java @@ -338,6 +338,7 @@ public void testStatusWhenProjectionsMatch() = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); final AggregateProjectionSpec projection1 = new AggregateProjectionSpec( "foo", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) ), @@ -385,6 +386,7 @@ public void testStatusWhenProjectionsMismatch() = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); final AggregateProjectionSpec projection1 = new AggregateProjectionSpec( "1", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) ), @@ -398,6 +400,7 @@ public void testStatusWhenProjectionsMismatch() ); final AggregateProjectionSpec projection2 = new AggregateProjectionSpec( "2", + null, VirtualColumns.EMPTY, Collections.emptyList(), new AggregatorFactory[]{ diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java b/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java index fdd606344607..f8eff3df4b0d 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java @@ -55,6 +55,7 @@ public class CatalogDataSourceCompactionConfigTest private static final AggregateProjectionSpec TEST_PROJECTION_SPEC_1 = new AggregateProjectionSpec( "string_sum_long_hourly", + null, VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) ), diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java index 228229e7b737..0303ebd28d33 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java @@ -945,6 +945,7 @@ public void testCompactWithProjections() final List projections = ImmutableList.of( new AggregateProjectionSpec( dataSource + "_projection", + null, VirtualColumns.create( Granularities.toVirtualColumn( Granularities.HOUR, @@ -1001,6 +1002,7 @@ public void testCompactWithCatalogProjections() final ArgumentCaptor payloadCaptor = setUpMockClient(mockClient); final AggregateProjectionSpec projectionSpec = new AggregateProjectionSpec( dataSource + "_projection", + null, VirtualColumns.create( Granularities.toVirtualColumn( Granularities.HOUR, From b47b5e5298db910a45b08cf34c511b5d21982321 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 30 Jul 2025 03:44:33 -0700 Subject: [PATCH 2/9] fix style --- .../org/apache/druid/segment/AggregateProjectionMetadata.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java index 840766de94f2..30100514b7b4 100644 --- a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java +++ b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java @@ -640,7 +640,7 @@ public static Filter rewriteFilter(@Nullable Filter projectionFilter, @Nullable return ProjectionFilterMatch.INSTANCE; } if (queryFilter instanceof IsBooleanFilter && ((IsBooleanFilter) queryFilter).isTrue()) { - final IsBooleanFilter isTrueFilter = (IsBooleanFilter) queryFilter; + final IsBooleanFilter isTrueFilter = (IsBooleanFilter) queryFilter; final Filter rewritten = rewriteFilter(projectionFilter, isTrueFilter.getBaseFilter()); if (rewritten == null) { return null; From 280c8ce412d730b9c6448d2e47713043b05de6da Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 30 Jul 2025 03:47:17 -0700 Subject: [PATCH 3/9] remove mistake --- .../org/apache/druid/segment/AggregateProjectionMetadata.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java index 30100514b7b4..6cb463261594 100644 --- a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java +++ b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java @@ -65,8 +65,7 @@ public class AggregateProjectionMetadata { private static final Interner SCHEMA_INTERNER = Interners.newWeakInterner(); - - + public static final Comparator COMPARATOR = (o1, o2) -> { int rowCompare = Integer.compare(o1.numRows, o2.numRows); if (rowCompare != 0) { From b591181f3024142c3cb4fd41f5fdec27225a4b0c Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 4 Aug 2025 12:45:44 -0700 Subject: [PATCH 4/9] move match logic into Projections instead of living in Schema, WIP --- .../segment/AggregateProjectionMetadata.java | 340 +----------- .../org/apache/druid/segment/IndexIO.java | 3 +- ...larNestedCommonFormatColumnSerializer.java | 20 +- .../projections/ConstantTimeColumn.java | 100 ++++ .../segment/projections/Projections.java | 496 +++++++++++++++--- .../AggregateProjectionMetadataTest.java | 23 +- 6 files changed, 547 insertions(+), 435 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/segment/projections/ConstantTimeColumn.java diff --git a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java index 6cb463261594..b6f872cc50c7 100644 --- a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java +++ b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java @@ -35,23 +35,15 @@ import org.apache.druid.query.OrderBy; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.filter.DimFilter; -import org.apache.druid.query.filter.Filter; import org.apache.druid.segment.column.ColumnHolder; -import org.apache.druid.segment.filter.AndFilter; -import org.apache.druid.segment.filter.IsBooleanFilter; -import org.apache.druid.segment.filter.TrueFilter; -import org.apache.druid.segment.projections.Projections; import org.apache.druid.utils.CollectionUtils; import javax.annotation.Nullable; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.Set; /** * Aggregate projection schema and row count information to store in {@link Metadata} which itself is stored inside a @@ -301,270 +293,16 @@ public Granularity getEffectiveGranularity() return effectiveGranularity; } - /** - * Check if this projection "matches" a {@link CursorBuildSpec} for a query to see if we can use a projection - * instead. For a projection to match, all grouping columns of the build spec must match, virtual columns of the - * build spec must either be available as a physical column on the projection, or the inputs to the virtual column - * must be available on the projection, and all aggregators must be compatible with pre-aggregated columns of the - * projection per {@link AggregatorFactory#substituteCombiningFactory(AggregatorFactory)}. If the projection - * matches, this method returns a {@link Projections.ProjectionMatch} which contains an updated - * {@link CursorBuildSpec} which has the remaining virtual columns from the original build spec which must still be - * computed and the 'combining' aggregator factories to process the pre-aggregated data from the projection, as well - * as a mapping of query column names to projection column names. - * - * @param queryCursorBuildSpec the {@link CursorBuildSpec} that contains the required inputs to build a - * {@link CursorHolder} for a query - * @param physicalColumnChecker Helper utility which can determine if a physical column required by - * queryCursorBuildSpec is available on the projection OR does not exist on the base - * table either - * @return a {@link Projections.ProjectionMatch} if the {@link CursorBuildSpec} matches the projection, which - * contains information such as which - */ - @Nullable - public Projections.ProjectionMatch matches( - CursorBuildSpec queryCursorBuildSpec, - Projections.PhysicalColumnChecker physicalColumnChecker - ) - { - if (!queryCursorBuildSpec.isCompatibleOrdering(orderingWithTimeSubstitution)) { - return null; - } - Projections.ProjectionMatchBuilder matchBuilder = new Projections.ProjectionMatchBuilder(); - - final List queryGrouping = queryCursorBuildSpec.getGroupingColumns(); - if (queryGrouping != null) { - for (String queryColumn : queryGrouping) { - matchBuilder = matchRequiredColumn( - matchBuilder, - queryColumn, - queryCursorBuildSpec.getVirtualColumns(), - physicalColumnChecker - ); - if (matchBuilder == null) { - return null; - } - // a query grouping column must also be defined as a projection grouping column - if (isInvalidGrouping(queryColumn) || isInvalidGrouping(matchBuilder.getRemapValue(queryColumn))) { - return null; - } - } - } - if (queryCursorBuildSpec.getFilter() != null) { - for (String queryColumn : queryCursorBuildSpec.getFilter().getRequiredColumns()) { - matchBuilder = matchRequiredColumn( - matchBuilder, - queryColumn, - queryCursorBuildSpec.getVirtualColumns(), - physicalColumnChecker - ); - if (matchBuilder == null) { - return null; - } - } - } - if (!CollectionUtils.isNullOrEmpty(queryCursorBuildSpec.getAggregators())) { - boolean allMatch = true; - for (AggregatorFactory queryAgg : queryCursorBuildSpec.getAggregators()) { - boolean foundMatch = false; - for (AggregatorFactory projectionAgg : aggregators) { - final AggregatorFactory combining = queryAgg.substituteCombiningFactory(projectionAgg); - if (combining != null) { - matchBuilder.remapColumn(queryAgg.getName(), projectionAgg.getName()) - .addReferencedPhysicalColumn(projectionAgg.getName()) - .addPreAggregatedAggregator(combining) - .addMatchedQueryColumns(queryAgg.requiredFields()); - foundMatch = true; - break; - } - } - allMatch = allMatch && foundMatch; - } - if (!allMatch) { - return null; - } - } - // validate physical and virtual columns have all been accounted for - final Set matchedQueryColumns = matchBuilder.getMatchedQueryColumns(); - if (queryCursorBuildSpec.getPhysicalColumns() != null) { - for (String queryColumn : queryCursorBuildSpec.getPhysicalColumns()) { - // time is special handled - if (ColumnHolder.TIME_COLUMN_NAME.equals(queryColumn)) { - continue; - } - if (!matchedQueryColumns.contains(queryColumn)) { - matchBuilder = matchRequiredColumn( - matchBuilder, - queryColumn, - queryCursorBuildSpec.getVirtualColumns(), - physicalColumnChecker - ); - if (matchBuilder == null) { - return null; - } - } - } - for (VirtualColumn vc : queryCursorBuildSpec.getVirtualColumns().getVirtualColumns()) { - if (!matchedQueryColumns.contains(vc.getOutputName())) { - matchBuilder = matchRequiredColumn( - matchBuilder, - vc.getOutputName(), - queryCursorBuildSpec.getVirtualColumns(), - physicalColumnChecker - ); - if (matchBuilder == null) { - return null; - } - } - } - } - - // if the projection has a filter, the query must contain this filter match - if (filter != null) { - final Filter queryFilter = queryCursorBuildSpec.getFilter(); - if (queryFilter != null) { - // try to rewrite the query filter into a projection filter, if the rewrite is valid, we can proceed - final Filter projectionFilter = filter.toOptimizedFilter(false); - final Map filterRewrites = new HashMap<>(); - // start with identity - for (String required : queryFilter.getRequiredColumns()) { - filterRewrites.put(required, required); - } - // overlay projection rewrites - filterRewrites.putAll(matchBuilder.getRemapColumns()); - - final Filter remappedQueryFilter = queryFilter.rewriteRequiredColumns(filterRewrites); - - final Filter rewritten = rewriteFilter(projectionFilter, remappedQueryFilter); - // if the filter does not contain the projection filter, we cannot match this projection - if (rewritten == null) { - return null; - } - //noinspection ObjectEquality - if (rewritten == ProjectionFilterMatch.INSTANCE) { - // we can remove the whole thing since the query filter exactly matches the projection filter - matchBuilder.rewriteFilter(null); - } else { - // otherwise, we partially rewrote the query filter to eliminate the projection filter since it is baked in - matchBuilder.rewriteFilter(rewritten); - } - } else { - // projection has a filter, but the query doesn't, no good - return null; - } - } else { - // projection doesn't have a filter, retain the original - matchBuilder.rewriteFilter(queryCursorBuildSpec.getFilter()); - } - - return matchBuilder.build(queryCursorBuildSpec); - } - - /** - * Ensure that the projection has the specified column required by a {@link CursorBuildSpec} in one form or another. - * If the column is a {@link VirtualColumn} on the build spec, ensure that the projection has an equivalent virtual - * column, or has the required inputs to compute the virtual column. If an equivalent virtual column exists, its - * name will be added to {@link Projections.ProjectionMatchBuilder#remapColumn(String, String)} so the query - * virtual column name can be mapped to the projection physical column name. If no equivalent virtual column exists, - * but the inputs are available on the projection to compute it, it will be added to - * {@link Projections.ProjectionMatchBuilder#addReferenceedVirtualColumn(VirtualColumn)}. - *

- * Finally, if the column is not a virtual column in the query, it is checked with - * {@link Projections.PhysicalColumnChecker} which true if the column is present on the projection OR if the column - * is NOT present on the base table (meaning missing columns that do not exist anywhere do not disqualify a - * projection from being used). - * - * @param matchBuilder match state to add mappings of query virtual columns to projection physical columns - * and query virtual columns which still must be computed from projection physical - * columns - * @param column Column name to check - * @param queryVirtualColumns {@link VirtualColumns} from the {@link CursorBuildSpec} required by the query - * @param physicalColumnChecker Helper to check if the physical column exists on a projection, or does not exist on - * the base table - * @return {@link Projections.ProjectionMatchBuilder} with updated state per the rules described above, or null - * if the column cannot be matched - */ - @Nullable - private Projections.ProjectionMatchBuilder matchRequiredColumn( - Projections.ProjectionMatchBuilder matchBuilder, - String column, - VirtualColumns queryVirtualColumns, - Projections.PhysicalColumnChecker physicalColumnChecker - ) - { - final VirtualColumn queryVirtualColumn = queryVirtualColumns.getVirtualColumn(column); - if (queryVirtualColumn != null) { - matchBuilder.addMatchedQueryColumn(column) - .addMatchedQueryColumns(queryVirtualColumn.requiredColumns()); - // check to see if we have an equivalent virtual column defined in the projection, if so we can - final VirtualColumn projectionEquivalent = virtualColumns.findEquivalent(queryVirtualColumn); - if (projectionEquivalent != null) { - final String remapColumnName; - if (Objects.equals(projectionEquivalent.getOutputName(), timeColumnName)) { - remapColumnName = ColumnHolder.TIME_COLUMN_NAME; - } else { - remapColumnName = projectionEquivalent.getOutputName(); - } - if (!queryVirtualColumn.getOutputName().equals(remapColumnName)) { - matchBuilder.remapColumn(queryVirtualColumn.getOutputName(), remapColumnName); - } - return matchBuilder.addReferencedPhysicalColumn(remapColumnName); - } - - matchBuilder.addReferenceedVirtualColumn(queryVirtualColumn); - final List requiredInputs = queryVirtualColumn.requiredColumns(); - if (requiredInputs.size() == 1 && ColumnHolder.TIME_COLUMN_NAME.equals(requiredInputs.get(0))) { - // special handle time granularity. in the future this should be reworked to push this concept into the - // virtual column and underlying expression itself, but this will do for now - final Granularity virtualGranularity = Granularities.fromVirtualColumn(queryVirtualColumn); - if (virtualGranularity != null) { - if (virtualGranularity.isFinerThan(effectiveGranularity)) { - return null; - } - // same granularity, replace virtual column directly by remapping it to the physical column - if (effectiveGranularity.equals(virtualGranularity)) { - return matchBuilder.remapColumn(column, ColumnHolder.TIME_COLUMN_NAME) - .addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); - } - return matchBuilder.addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); - } else { - // anything else with __time requires none granularity - if (Granularities.NONE.equals(effectiveGranularity)) { - return matchBuilder.addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); - } - return null; - } - } else { - for (String required : requiredInputs) { - matchBuilder = matchRequiredColumn( - matchBuilder, - required, - queryVirtualColumns, - physicalColumnChecker - ); - if (matchBuilder == null) { - return null; - } - } - return matchBuilder; - } - } else { - if (physicalColumnChecker.check(name, column)) { - return matchBuilder.addMatchedQueryColumn(column) - .addReferencedPhysicalColumn(column); - } - return null; - } - } - /** * Check if a column is either part of {@link #groupingColumns}, or at least is not present in - * {@link #virtualColumns}. Naively we would just check that grouping column contains the column in question, - * however we can also use a projection when a column is truly missing. {@link #matchRequiredColumn} returns a - * match builder if the column is present as either a physical column, or a virtual column, but a virtual column - * could also be present for an aggregator input, so we must further check that a column not in the grouping list - * is also not a virtual column, the implication being that it is a missing column. + * {@link #virtualColumns}. Naively, we would just check that grouping column contains the column in question, + * however, we can also use a projection when a column is truly missing. + * {@link org.apache.druid.segment.projections.Projections#matchAggregateProjection} returns a match builder if the + * column is present as either a physical column, or a virtual column, but a virtual column could also be present + * for an aggregator input, so we must further check that a column not in the grouping list is also not a virtual + * column, the implication being that it is a missing column. */ - private boolean isInvalidGrouping(@Nullable String columnName) + public boolean isInvalidGrouping(@Nullable String columnName) { if (columnName == null) { return false; @@ -621,68 +359,4 @@ public String toString() '}'; } } - - /** - * Rewrites a query {@link Filter} if possible, removing the {@link Filter} of a projection. To match a projection - * filter, the query filter must be equal to the projection filter, or must contain the projection filter as the child - * of an AND filter. This method returns null - * indicating that a rewrite is impossible with the implication that the query cannot use the projection because the - * projection doesn't contain all the rows the query would match if not using the projection. - */ - @Nullable - public static Filter rewriteFilter(@Nullable Filter projectionFilter, @Nullable Filter queryFilter) - { - if (projectionFilter == null || queryFilter == null) { - return queryFilter; - } - if (queryFilter.equals(projectionFilter)) { - return ProjectionFilterMatch.INSTANCE; - } - if (queryFilter instanceof IsBooleanFilter && ((IsBooleanFilter) queryFilter).isTrue()) { - final IsBooleanFilter isTrueFilter = (IsBooleanFilter) queryFilter; - final Filter rewritten = rewriteFilter(projectionFilter, isTrueFilter.getBaseFilter()); - if (rewritten == null) { - return null; - } - //noinspection ObjectEquality - if (rewritten == ProjectionFilterMatch.INSTANCE) { - return ProjectionFilterMatch.INSTANCE; - } - return new IsBooleanFilter(rewritten, true); - } - if (queryFilter instanceof AndFilter) { - AndFilter andFilter = (AndFilter) queryFilter; - List newChildren = Lists.newArrayListWithExpectedSize(andFilter.getFilters().size()); - boolean childRewritten = false; - for (Filter filter : andFilter.getFilters()) { - Filter rewritten = rewriteFilter(projectionFilter, filter); - //noinspection ObjectEquality - if (rewritten == ProjectionFilterMatch.INSTANCE) { - childRewritten = true; - } else { - if (rewritten != null) { - newChildren.add(rewritten); - childRewritten = true; - } else { - newChildren.add(filter); - } - } - } - // at least one child must have been rewritten to rewrite the AND - if (childRewritten) { - if (newChildren.size() > 1) { - return new AndFilter(newChildren); - } else { - return newChildren.get(0); - } - } - return null; - } - return null; - } - - static final class ProjectionFilterMatch extends TrueFilter - { - private static final ProjectionFilterMatch INSTANCE = new ProjectionFilterMatch(); - } } diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index f1faed65da14..95d39521908b 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -65,6 +65,7 @@ import org.apache.druid.segment.data.IndexedIterable; import org.apache.druid.segment.data.ListIndexed; import org.apache.druid.segment.data.VSizeColumnarMultiInts; +import org.apache.druid.segment.projections.ConstantTimeColumn; import org.apache.druid.segment.projections.Projections; import org.apache.druid.segment.serde.ComplexColumnPartSupplier; import org.apache.druid.segment.serde.FloatNumericColumnSupplier; @@ -742,7 +743,7 @@ private Map> readProjectionColumns( if (projectionSpec.getSchema().getTimeColumnName() == null) { projectionColumns.put( ColumnHolder.TIME_COLUMN_NAME, - Projections.makeConstantTimeSupplier(projectionSpec.getNumRows(), dataInterval.getStartMillis()) + ConstantTimeColumn.makeConstantTimeSupplier(projectionSpec.getNumRows(), dataInterval.getStartMillis()) ); } return projectionColumns; diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarNestedCommonFormatColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarNestedCommonFormatColumnSerializer.java index dc18970ad799..271d9c9b33f2 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarNestedCommonFormatColumnSerializer.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarNestedCommonFormatColumnSerializer.java @@ -212,23 +212,29 @@ public void writeTo( bitmapIndexWriter.setObjectsNotSorted(); final MutableBitmap[] bitmaps; bitmaps = new MutableBitmap[getCardinality()]; - for (int i = 0; i < bitmaps.length; i++) { - bitmaps[i] = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap(); - } final IntIterator rows = intermediateValueWriter.getIterator(); int rowCount = 0; while (rows.hasNext()) { final int dictId = rows.nextInt(); encodedValueSerializer.addValue(dictId); - bitmaps[dictId].add(rowCount++); + MutableBitmap b = bitmaps[dictId]; + if (b == null) { + b = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap(); + bitmaps[dictId] = b; + } + b.add(rowCount++); } for (int i = 0; i < bitmaps.length; i++) { final MutableBitmap bitmap = bitmaps[i]; - bitmapIndexWriter.write( - indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeImmutableBitmap(bitmap) - ); + if (bitmap == null) { + bitmapIndexWriter.write(null); + } else { + bitmapIndexWriter.write( + indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeImmutableBitmap(bitmap) + ); + } bitmaps[i] = null; // Reclaim memory } diff --git a/processing/src/main/java/org/apache/druid/segment/projections/ConstantTimeColumn.java b/processing/src/main/java/org/apache/druid/segment/projections/ConstantTimeColumn.java new file mode 100644 index 000000000000..6485cb9578de --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/projections/ConstantTimeColumn.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.projections; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.ConstantExprEvalSelector; +import org.apache.druid.segment.column.CapabilitiesBasedFormat; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; +import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.NumericColumn; +import org.apache.druid.segment.data.ReadableOffset; +import org.apache.druid.segment.vector.ConstantVectorSelectors; +import org.apache.druid.segment.vector.ReadableVectorOffset; +import org.apache.druid.segment.vector.VectorValueSelector; + +public class ConstantTimeColumn implements NumericColumn +{ + public static Supplier makeConstantTimeSupplier(int numRows, long constant) + { + return Suppliers.memoize( + () -> new ColumnBuilder().setNumericColumnSupplier(() -> new ConstantTimeColumn(numRows, constant)) + .setColumnFormat( + new CapabilitiesBasedFormat( + ColumnCapabilitiesImpl.createDefault().setType(ColumnType.LONG) + ) + ) + .setType(ColumnType.LONG) + .setHasNulls(false) + .build() + ); + } + + private final int numRows; + private final long constant; + + public ConstantTimeColumn(int numRows, long constant) + { + this.numRows = numRows; + this.constant = constant; + } + + @Override + public int length() + { + return numRows; + } + + @Override + public long getLongSingleValueRow(int rowNum) + { + return constant; + } + + @Override + public void close() + { + // nothing to close + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + + } + + @Override + public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + { + return new ConstantExprEvalSelector(ExprEval.ofLong(constant)); + } + + @Override + public VectorValueSelector makeVectorValueSelector(ReadableVectorOffset offset) + { + return ConstantVectorSelectors.vectorValueSelector(offset, constant); + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java index 081ece8b72fa..a1faf165f499 100644 --- a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java +++ b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java @@ -19,31 +19,24 @@ package org.apache.druid.segment.projections; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; +import com.google.common.collect.Lists; import org.apache.druid.data.input.impl.AggregateProjectionSpec; import org.apache.druid.error.InvalidInput; -import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.filter.Filter; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.AggregateProjectionMetadata; -import org.apache.druid.segment.ColumnValueSelector; -import org.apache.druid.segment.ConstantExprEvalSelector; import org.apache.druid.segment.CursorBuildSpec; +import org.apache.druid.segment.CursorHolder; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.VirtualColumns; -import org.apache.druid.segment.column.CapabilitiesBasedFormat; -import org.apache.druid.segment.column.ColumnBuilder; -import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnHolder; -import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.NumericColumn; -import org.apache.druid.segment.data.ReadableOffset; -import org.apache.druid.segment.vector.ConstantVectorSelectors; -import org.apache.druid.segment.vector.ReadableVectorOffset; -import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.segment.filter.AndFilter; +import org.apache.druid.segment.filter.IsBooleanFilter; +import org.apache.druid.segment.filter.TrueFilter; +import org.apache.druid.utils.CollectionUtils; import javax.annotation.Nullable; import java.util.ArrayList; @@ -59,21 +52,6 @@ public class Projections { - public static Supplier makeConstantTimeSupplier(int numRows, long constant) - { - return Suppliers.memoize( - () -> new ColumnBuilder().setNumericColumnSupplier(() -> new ConstantTimeColumn(numRows, constant)) - .setColumnFormat( - new CapabilitiesBasedFormat( - ColumnCapabilitiesImpl.createDefault().setType(ColumnType.LONG) - ) - ) - .setType(ColumnType.LONG) - .setHasNulls(false) - .build() - ); - } - @Nullable public static QueryableProjection findMatchingProjection( CursorBuildSpec cursorBuildSpec, @@ -92,7 +70,7 @@ public static QueryableProjection findMatchingProjection( if (name != null && !name.equals(spec.getSchema().getName())) { continue; } - final ProjectionMatch match = spec.getSchema().matches(cursorBuildSpec, physicalChecker); + final ProjectionMatch match = matchAggregateProjection(spec.getSchema(), cursorBuildSpec, physicalChecker); if (match != null) { if (cursorBuildSpec.getQueryMetrics() != null) { cursorBuildSpec.getQueryMetrics().projection(spec.getSchema().getName()); @@ -114,26 +92,402 @@ public static QueryableProjection findMatchingProjection( return null; } - public static String getProjectionSmooshV9FileName(AggregateProjectionMetadata projectionSpec, String columnName) + + /** + * Check if this projection "matches" a {@link CursorBuildSpec} for a query to see if we can use a projection + * instead. For a projection to match, all grouping columns of the build spec must match, virtual columns of the + * build spec must either be available as a physical column on the projection, or the inputs to the virtual column + * must be available on the projection, and all aggregators must be compatible with pre-aggregated columns of the + * projection per {@link AggregatorFactory#substituteCombiningFactory(AggregatorFactory)}. If the projection + * matches, this method returns a {@link ProjectionMatch} which contains an updated {@link CursorBuildSpec} which has + * the remaining virtual columns from the original build spec which must still be computed and the 'combining' + * aggregator factories to process the pre-aggregated data from the projection, as well as a mapping of query column + * names to projection column names. + * + * @param queryCursorBuildSpec the {@link CursorBuildSpec} that contains the required inputs to build a + * {@link CursorHolder} for a query + * @param physicalColumnChecker Helper utility which can determine if a physical column required by + * queryCursorBuildSpec is available on the projection OR does not exist on the base + * table either + * @return a {@link ProjectionMatch} if the {@link CursorBuildSpec} matches the projection, which contains + * information such as which + */ + @Nullable + public static ProjectionMatch matchAggregateProjection( + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + PhysicalColumnChecker physicalColumnChecker + ) { - return getProjectionSmooshV9Prefix(projectionSpec) + columnName; + if (!queryCursorBuildSpec.isCompatibleOrdering(projection.getOrderingWithTimeColumnSubstitution())) { + return null; + } + ProjectionMatchBuilder matchBuilder = new ProjectionMatchBuilder(); + + matchBuilder = matchGrouping(projection, queryCursorBuildSpec, physicalColumnChecker, matchBuilder); + if (matchBuilder == null) { + return null; + } + + if (matchAggregators(projection, queryCursorBuildSpec, matchBuilder)) { + return null; + } + + matchBuilder = matchFilter(projection, queryCursorBuildSpec, physicalColumnChecker, matchBuilder); + if (matchBuilder == null) { + return null; + } + + matchBuilder = matchRemainingRequiredColumns(projection, queryCursorBuildSpec, physicalColumnChecker, matchBuilder); + if (matchBuilder == null) { + return null; + } + + // todo (clint): combine this with match filter, but need to figure out how to handle remapping since ideally all + // the remapping has been done before we match filters so the filter required column rewrites are chill + // if the projection has a filter, the query must contain this filter match + if (projection.getFilter() != null) { + final Filter queryFilter = queryCursorBuildSpec.getFilter(); + if (queryFilter != null) { + // try to rewrite the query filter into a projection filter, if the rewrite is valid, we can proceed + final Filter projectionFilter = projection.getFilter().toOptimizedFilter(false); + final Map filterRewrites = new HashMap<>(); + // start with identity + for (String required : queryFilter.getRequiredColumns()) { + filterRewrites.put(required, required); + } + // overlay projection rewrites + filterRewrites.putAll(matchBuilder.getRemapColumns()); + + final Filter remappedQueryFilter = queryFilter.rewriteRequiredColumns(filterRewrites); + + final Filter rewritten = rewriteFilter(projectionFilter, remappedQueryFilter); + // if the filter does not contain the projection filter, we cannot match this projection + if (rewritten == null) { + return null; + } + //noinspection ObjectEquality + if (rewritten == ProjectionFilterMatch.INSTANCE) { + // we can remove the whole thing since the query filter exactly matches the projection filter + matchBuilder.rewriteFilter(null); + } else { + // otherwise, we partially rewrote the query filter to eliminate the projection filter since it is baked in + matchBuilder.rewriteFilter(rewritten); + } + } else { + // projection has a filter, but the query doesn't, no good + return null; + } + } else { + // projection doesn't have a filter, retain the original + matchBuilder.rewriteFilter(queryCursorBuildSpec.getFilter()); + } + + return matchBuilder.build(queryCursorBuildSpec); } - public static String getProjectionSmooshV9Prefix(AggregateProjectionMetadata projectionSpec) + @Nullable + private static ProjectionMatchBuilder matchGrouping( + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + PhysicalColumnChecker physicalColumnChecker, + ProjectionMatchBuilder matchBuilder + ) { - return projectionSpec.getSchema().getName() + "/"; + final List queryGrouping = queryCursorBuildSpec.getGroupingColumns(); + if (queryGrouping != null) { + for (String queryColumn : queryGrouping) { + matchBuilder = matchRequiredColumn( + projection, + matchBuilder, + queryColumn, + queryCursorBuildSpec.getVirtualColumns(), + physicalColumnChecker + ); + if (matchBuilder == null) { + return null; + } + // a query grouping column must also be defined as a projection grouping column + if (projection.isInvalidGrouping(queryColumn)) { + return null; + } + // even if remapped + if (projection.isInvalidGrouping(matchBuilder.getRemapValue(queryColumn))) { + return null; + } + } + } + return matchBuilder; + } + + + private static boolean matchAggregators( + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + ProjectionMatchBuilder matchBuilder + ) + { + if (!CollectionUtils.isNullOrEmpty(queryCursorBuildSpec.getAggregators())) { + boolean allMatch = true; + for (AggregatorFactory queryAgg : queryCursorBuildSpec.getAggregators()) { + boolean foundMatch = false; + for (AggregatorFactory projectionAgg : projection.getAggregators()) { + final AggregatorFactory combining = queryAgg.substituteCombiningFactory(projectionAgg); + if (combining != null) { + matchBuilder.remapColumn(queryAgg.getName(), projectionAgg.getName()) + .addReferencedPhysicalColumn(projectionAgg.getName()) + .addPreAggregatedAggregator(combining) + .addMatchedQueryColumns(queryAgg.requiredFields()); + foundMatch = true; + break; + } + } + allMatch = allMatch && foundMatch; + } + return !allMatch; + } + return false; + } + + @Nullable + private static ProjectionMatchBuilder matchFilter( + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + PhysicalColumnChecker physicalColumnChecker, + ProjectionMatchBuilder matchBuilder + ) + { + if (queryCursorBuildSpec.getFilter() != null) { + for (String queryColumn : queryCursorBuildSpec.getFilter().getRequiredColumns()) { + matchBuilder = matchRequiredColumn( + projection, + matchBuilder, + queryColumn, + queryCursorBuildSpec.getVirtualColumns(), + physicalColumnChecker + ); + if (matchBuilder == null) { + return null; + } + } + } + + return matchBuilder; + } + + @Nullable + private static ProjectionMatchBuilder matchRemainingRequiredColumns( + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + PhysicalColumnChecker physicalColumnChecker, + ProjectionMatchBuilder matchBuilder + ) + { + // validate physical and virtual columns have all been accounted for + final Set matchedQueryColumns = matchBuilder.getMatchedQueryColumns(); + if (queryCursorBuildSpec.getPhysicalColumns() != null) { + for (String queryColumn : queryCursorBuildSpec.getPhysicalColumns()) { + // a projection always has a __time column, it just might be a constant of the segment interval start if the + // projection itself did not transform the base table __time column + if (ColumnHolder.TIME_COLUMN_NAME.equals(queryColumn)) { + continue; + } + if (!matchedQueryColumns.contains(queryColumn)) { + matchBuilder = matchRequiredColumn( + projection, + matchBuilder, + queryColumn, + queryCursorBuildSpec.getVirtualColumns(), + physicalColumnChecker + ); + if (matchBuilder == null) { + return null; + } + } + } + for (VirtualColumn vc : queryCursorBuildSpec.getVirtualColumns().getVirtualColumns()) { + if (!matchedQueryColumns.contains(vc.getOutputName())) { + matchBuilder = matchRequiredColumn( + projection, + matchBuilder, + vc.getOutputName(), + queryCursorBuildSpec.getVirtualColumns(), + physicalColumnChecker + ); + if (matchBuilder == null) { + return null; + } + } + } + } + return matchBuilder; } /** - * Returns true if column is defined in {@link AggregateProjectionSpec#getGroupingColumns()} OR if the column does not - * exist in the base table. Part of determining if a projection can be used for a given {@link CursorBuildSpec}, + * Ensure that the projection has the specified column required by a {@link CursorBuildSpec} in one form or another. + * If the column is a {@link VirtualColumn} on the build spec, ensure that the projection has an equivalent virtual + * column, or has the required inputs to compute the virtual column. If an equivalent virtual column exists, its + * name will be added to {@link ProjectionMatchBuilder#remapColumn(String, String)} so the query virtual column name\ + * can be mapped to the projection physical column name. If no equivalent virtual column exists, but the inputs are + * available on the projection to compute it, it will be added to + * {@link ProjectionMatchBuilder#addReferenceedVirtualColumn(VirtualColumn)}. + *

+ * Finally, if the column is not a virtual column in the query, it is checked with {@link PhysicalColumnChecker} + * which true if the column is present on the projection OR if the column is NOT present on the base table (meaning + * missing columns that do not exist anywhere do not disqualify a projection from being used). * - * @see AggregateProjectionMetadata.Schema#matches(CursorBuildSpec, PhysicalColumnChecker) + * @param matchBuilder match state to add mappings of query virtual columns to projection physical columns + * and query virtual columns which still must be computed from projection physical + * columns + * @param column Column name to check + * @param queryVirtualColumns {@link VirtualColumns} from the {@link CursorBuildSpec} required by the query + * @param physicalColumnChecker Helper to check if the physical column exists on a projection, or does not exist on + * the base table + * @return {@link ProjectionMatchBuilder} with updated state per the rules described above, or null if the column + * cannot be matched */ - @FunctionalInterface - public interface PhysicalColumnChecker + @Nullable + private static ProjectionMatchBuilder matchRequiredColumn( + AggregateProjectionMetadata.Schema projection, + ProjectionMatchBuilder matchBuilder, + String column, + VirtualColumns queryVirtualColumns, + PhysicalColumnChecker physicalColumnChecker + ) { - boolean check(String projectionName, String columnName); + final VirtualColumn queryVirtualColumn = queryVirtualColumns.getVirtualColumn(column); + if (queryVirtualColumn != null) { + matchBuilder.addMatchedQueryColumn(column) + .addMatchedQueryColumns(queryVirtualColumn.requiredColumns()); + // check to see if we have an equivalent virtual column defined in the projection, if so we can + final VirtualColumn projectionEquivalent = projection.getVirtualColumns().findEquivalent(queryVirtualColumn); + if (projectionEquivalent != null) { + final String remapColumnName; + if (Objects.equals(projectionEquivalent.getOutputName(), projection.getTimeColumnName())) { + remapColumnName = ColumnHolder.TIME_COLUMN_NAME; + } else { + remapColumnName = projectionEquivalent.getOutputName(); + } + if (!queryVirtualColumn.getOutputName().equals(remapColumnName)) { + matchBuilder.remapColumn(queryVirtualColumn.getOutputName(), remapColumnName); + } + return matchBuilder.addReferencedPhysicalColumn(remapColumnName); + } + + matchBuilder.addReferenceedVirtualColumn(queryVirtualColumn); + final List requiredInputs = queryVirtualColumn.requiredColumns(); + if (requiredInputs.size() == 1 && ColumnHolder.TIME_COLUMN_NAME.equals(requiredInputs.get(0))) { + // special handle time granularity. in the future this should be reworked to push this concept into the + // virtual column and underlying expression itself, but this will do for now + final Granularity virtualGranularity = Granularities.fromVirtualColumn(queryVirtualColumn); + if (virtualGranularity != null) { + if (virtualGranularity.isFinerThan(projection.getEffectiveGranularity())) { + return null; + } + // same granularity, replace virtual column directly by remapping it to the physical column + if (projection.getEffectiveGranularity().equals(virtualGranularity)) { + return matchBuilder.remapColumn(column, ColumnHolder.TIME_COLUMN_NAME) + .addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); + } + return matchBuilder.addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); + } else { + // anything else with __time requires none granularity + if (Granularities.NONE.equals(projection.getEffectiveGranularity())) { + return matchBuilder.addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); + } + return null; + } + } else { + for (String required : requiredInputs) { + matchBuilder = matchRequiredColumn( + projection, + matchBuilder, + required, + queryVirtualColumns, + physicalColumnChecker + ); + if (matchBuilder == null) { + return null; + } + } + return matchBuilder; + } + } else { + if (physicalColumnChecker.check(projection.getName(), column)) { + return matchBuilder.addMatchedQueryColumn(column) + .addReferencedPhysicalColumn(column); + } + return null; + } + } + + /** + * Rewrites a query {@link Filter} if possible, removing the {@link Filter} of a projection. To match a projection + * filter, the query filter must be equal to the projection filter, or must contain the projection filter as the child + * of an AND filter. This method returns null + * indicating that a rewrite is impossible with the implication that the query cannot use the projection because the + * projection doesn't contain all the rows the query would match if not using the projection. + */ + @Nullable + public static Filter rewriteFilter(@Nullable Filter projectionFilter, @Nullable Filter queryFilter) + { + if (projectionFilter == null || queryFilter == null) { + return queryFilter; + } + if (queryFilter.equals(projectionFilter)) { + return ProjectionFilterMatch.INSTANCE; + } + if (queryFilter instanceof IsBooleanFilter && ((IsBooleanFilter) queryFilter).isTrue()) { + final IsBooleanFilter isTrueFilter = (IsBooleanFilter) queryFilter; + final Filter rewritten = rewriteFilter(projectionFilter, isTrueFilter.getBaseFilter()); + if (rewritten == null) { + return null; + } + //noinspection ObjectEquality + if (rewritten == ProjectionFilterMatch.INSTANCE) { + return ProjectionFilterMatch.INSTANCE; + } + return new IsBooleanFilter(rewritten, true); + } + if (queryFilter instanceof AndFilter) { + AndFilter andFilter = (AndFilter) queryFilter; + List newChildren = Lists.newArrayListWithExpectedSize(andFilter.getFilters().size()); + boolean childRewritten = false; + for (Filter filter : andFilter.getFilters()) { + Filter rewritten = rewriteFilter(projectionFilter, filter); + //noinspection ObjectEquality + if (rewritten == ProjectionFilterMatch.INSTANCE) { + childRewritten = true; + } else { + if (rewritten != null) { + newChildren.add(rewritten); + childRewritten = true; + } else { + newChildren.add(filter); + } + } + } + // at least one child must have been rewritten to rewrite the AND + if (childRewritten) { + if (newChildren.size() > 1) { + return new AndFilter(newChildren); + } else { + return newChildren.get(0); + } + } + return null; + } + return null; + } + + public static String getProjectionSmooshV9FileName(AggregateProjectionMetadata projectionSpec, String columnName) + { + return getProjectionSmooshV9Prefix(projectionSpec) + columnName; + } + + public static String getProjectionSmooshV9Prefix(AggregateProjectionMetadata projectionSpec) + { + return projectionSpec.getSchema().getName() + "/"; } public static final class ProjectionMatch @@ -283,51 +637,25 @@ public ProjectionMatch build(CursorBuildSpec queryCursorBuildSpec) } } - private static class ConstantTimeColumn implements NumericColumn + public static final class ProjectionFilterMatch extends TrueFilter { - private final int numRows; - private final long constant; - - private ConstantTimeColumn(int numRows, long constant) - { - this.numRows = numRows; - this.constant = constant; - } - - @Override - public int length() - { - return numRows; - } - - @Override - public long getLongSingleValueRow(int rowNum) - { - return constant; - } - - @Override - public void close() - { - // nothing to close - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - - } + private static final ProjectionFilterMatch INSTANCE = new ProjectionFilterMatch(); + } - @Override - public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) - { - return new ConstantExprEvalSelector(ExprEval.ofLong(constant)); - } + /** + * Returns true if column is defined in {@link AggregateProjectionSpec#getGroupingColumns()} OR if the column does not + * exist in the base table. Part of determining if a projection can be used for a given {@link CursorBuildSpec}, + * + * @see #matchAggregateProjection(AggregateProjectionMetadata.Schema, CursorBuildSpec, PhysicalColumnChecker) + */ + @FunctionalInterface + public interface PhysicalColumnChecker + { + boolean check(String projectionName, String columnName); + } - @Override - public VectorValueSelector makeVectorValueSelector(ReadableVectorOffset offset) - { - return ConstantVectorSelectors.vectorValueSelector(offset, constant); - } + private Projections() + { + // no instantiation } } diff --git a/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java b/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java index 238477866bc7..35fe050cd8a2 100644 --- a/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java +++ b/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java @@ -339,8 +339,11 @@ void testSchemaMatchSimple() ))) .build(); // act & assert - Projections.ProjectionMatch projectionMatch = spec.getSchema() - .matches(cursorBuildSpec, (projectionName, columnName) -> true); + Projections.ProjectionMatch projectionMatch = Projections.matchAggregateProjection( + spec.getSchema(), + cursorBuildSpec, + (projectionName, columnName) -> true + ); Projections.ProjectionMatch expected = new Projections.ProjectionMatch( CursorBuildSpec.builder() .setAggregators(ImmutableList.of(new LongSumAggregatorFactory("a", "a"))) @@ -363,32 +366,32 @@ void testRewriteFilter() Filter queryFilter = xeqfoo2; Assertions.assertInstanceOf( - AggregateProjectionMetadata.ProjectionFilterMatch.class, - AggregateProjectionMetadata.rewriteFilter(xeqfoo, queryFilter) + Projections.ProjectionFilterMatch.class, + Projections.rewriteFilter(xeqfoo, queryFilter) ); queryFilter = yeqbar; - Assertions.assertNull(AggregateProjectionMetadata.rewriteFilter(xeqfoo, queryFilter)); + Assertions.assertNull(Projections.rewriteFilter(xeqfoo, queryFilter)); queryFilter = new AndFilter(List.of(xeqfoo, yeqbar)); Assertions.assertEquals( yeqbar, - AggregateProjectionMetadata.rewriteFilter(xeqfoo, queryFilter) + Projections.rewriteFilter(xeqfoo, queryFilter) ); queryFilter = new AndFilter(List.of(new OrFilter(List.of(xeqfoo, xeqbar)), yeqbar)); - Assertions.assertNull(AggregateProjectionMetadata.rewriteFilter(xeqfoo, queryFilter)); + Assertions.assertNull(Projections.rewriteFilter(xeqfoo, queryFilter)); queryFilter = new AndFilter(List.of(new IsBooleanFilter(xeqfoo, true), yeqbar)); - Assertions.assertEquals(yeqbar, AggregateProjectionMetadata.rewriteFilter(xeqfoo, queryFilter)); + Assertions.assertEquals(yeqbar, Projections.rewriteFilter(xeqfoo, queryFilter)); queryFilter = new AndFilter(List.of(new IsBooleanFilter(xeqfoo, false), yeqbar)); - Assertions.assertNull(AggregateProjectionMetadata.rewriteFilter(xeqfoo, queryFilter)); + Assertions.assertNull(Projections.rewriteFilter(xeqfoo, queryFilter)); queryFilter = new AndFilter(List.of(new AndFilter(List.of(xeqfoo, yeqbar)), zeq123)); Assertions.assertEquals( new AndFilter(List.of(yeqbar, zeq123)), - AggregateProjectionMetadata.rewriteFilter(xeqfoo, queryFilter) + Projections.rewriteFilter(xeqfoo, queryFilter) ); } } From af66e88662b9bbfb2d83ff59e5060fd47c95441f Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 5 Aug 2025 18:54:23 -0700 Subject: [PATCH 5/9] move tests --- .../AggregateProjectionMetadataTest.java | 88 ------------- .../segment/projections/ProjectionsTest.java | 124 ++++++++++++++++++ 2 files changed, 124 insertions(+), 88 deletions(-) create mode 100644 processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java diff --git a/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java b/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java index 35fe050cd8a2..fc6dade3fc3c 100644 --- a/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java +++ b/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java @@ -21,9 +21,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import it.unimi.dsi.fastutil.objects.ObjectAVLTreeSet; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.error.DruidException; @@ -33,13 +30,8 @@ import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.filter.EqualityFilter; -import org.apache.druid.query.filter.Filter; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.filter.AndFilter; -import org.apache.druid.segment.filter.IsBooleanFilter; -import org.apache.druid.segment.filter.OrFilter; -import org.apache.druid.segment.projections.Projections; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -314,84 +306,4 @@ void testEqualsAndHashcodeSchema() .usingGetClass() .verify(); } - - @Test - void testSchemaMatchSimple() - { - // arrange - AggregateProjectionMetadata spec = new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "some_projection", - null, - null, - VirtualColumns.EMPTY, - Arrays.asList("a", "b"), - new AggregatorFactory[]{new LongSumAggregatorFactory("a_projection", "a")}, - Arrays.asList(OrderBy.ascending("a"), OrderBy.ascending("b")) - ), - 12345 - ); - CursorBuildSpec cursorBuildSpec = CursorBuildSpec.builder() - .setPreferredOrdering(ImmutableList.of()) - .setAggregators(ImmutableList.of(new LongSumAggregatorFactory( - "a", - "a" - ))) - .build(); - // act & assert - Projections.ProjectionMatch projectionMatch = Projections.matchAggregateProjection( - spec.getSchema(), - cursorBuildSpec, - (projectionName, columnName) -> true - ); - Projections.ProjectionMatch expected = new Projections.ProjectionMatch( - CursorBuildSpec.builder() - .setAggregators(ImmutableList.of(new LongSumAggregatorFactory("a", "a"))) - .setPhysicalColumns(ImmutableSet.of("a_projection")) - .setPreferredOrdering(ImmutableList.of()) - .build(), - ImmutableMap.of("a", "a_projection") - ); - Assertions.assertEquals(expected, projectionMatch); - } - - @Test - void testRewriteFilter() - { - Filter xeqfoo = new EqualityFilter("x", ColumnType.STRING, "foo", null); - Filter xeqfoo2 = new EqualityFilter("x", ColumnType.STRING, "foo", null); - Filter xeqbar = new EqualityFilter("x", ColumnType.STRING, "bar", null); - Filter yeqbar = new EqualityFilter("y", ColumnType.STRING, "bar", null); - Filter zeq123 = new EqualityFilter("z", ColumnType.LONG, 123L, null); - - Filter queryFilter = xeqfoo2; - Assertions.assertInstanceOf( - Projections.ProjectionFilterMatch.class, - Projections.rewriteFilter(xeqfoo, queryFilter) - ); - - queryFilter = yeqbar; - Assertions.assertNull(Projections.rewriteFilter(xeqfoo, queryFilter)); - - queryFilter = new AndFilter(List.of(xeqfoo, yeqbar)); - Assertions.assertEquals( - yeqbar, - Projections.rewriteFilter(xeqfoo, queryFilter) - ); - - queryFilter = new AndFilter(List.of(new OrFilter(List.of(xeqfoo, xeqbar)), yeqbar)); - Assertions.assertNull(Projections.rewriteFilter(xeqfoo, queryFilter)); - - queryFilter = new AndFilter(List.of(new IsBooleanFilter(xeqfoo, true), yeqbar)); - Assertions.assertEquals(yeqbar, Projections.rewriteFilter(xeqfoo, queryFilter)); - - queryFilter = new AndFilter(List.of(new IsBooleanFilter(xeqfoo, false), yeqbar)); - Assertions.assertNull(Projections.rewriteFilter(xeqfoo, queryFilter)); - - queryFilter = new AndFilter(List.of(new AndFilter(List.of(xeqfoo, yeqbar)), zeq123)); - Assertions.assertEquals( - new AndFilter(List.of(yeqbar, zeq123)), - Projections.rewriteFilter(xeqfoo, queryFilter) - ); - } } diff --git a/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java b/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java new file mode 100644 index 000000000000..76f58ca26b76 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.projections; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.apache.druid.query.OrderBy; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.filter.EqualityFilter; +import org.apache.druid.query.filter.Filter; +import org.apache.druid.segment.AggregateProjectionMetadata; +import org.apache.druid.segment.CursorBuildSpec; +import org.apache.druid.segment.VirtualColumns; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.filter.AndFilter; +import org.apache.druid.segment.filter.IsBooleanFilter; +import org.apache.druid.segment.filter.OrFilter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; + +class ProjectionsTest +{ + @Test + void testSchemaMatchSimple() + { + AggregateProjectionMetadata spec = new AggregateProjectionMetadata( + new AggregateProjectionMetadata.Schema( + "some_projection", + null, + null, + VirtualColumns.EMPTY, + Arrays.asList("a", "b"), + new AggregatorFactory[]{new LongSumAggregatorFactory("a_projection", "a")}, + Arrays.asList(OrderBy.ascending("a"), OrderBy.ascending("b")) + ), + 12345 + ); + CursorBuildSpec cursorBuildSpec = CursorBuildSpec.builder() + .setPreferredOrdering(ImmutableList.of()) + .setAggregators( + List.of( + new LongSumAggregatorFactory("a", "a") + ) + ) + .build(); + + Projections.ProjectionMatch projectionMatch = Projections.matchAggregateProjection( + spec.getSchema(), + cursorBuildSpec, + (projectionName, columnName) -> true + ); + Projections.ProjectionMatch expected = new Projections.ProjectionMatch( + CursorBuildSpec.builder() + .setAggregators(ImmutableList.of(new LongSumAggregatorFactory("a", "a"))) + .setPhysicalColumns(ImmutableSet.of("a_projection")) + .setPreferredOrdering(ImmutableList.of()) + .build(), + ImmutableMap.of("a", "a_projection") + ); + Assertions.assertEquals(expected, projectionMatch); + } + + @Test + void testRewriteFilter() + { + Filter xeqfoo = new EqualityFilter("x", ColumnType.STRING, "foo", null); + Filter xeqfoo2 = new EqualityFilter("x", ColumnType.STRING, "foo", null); + Filter xeqbar = new EqualityFilter("x", ColumnType.STRING, "bar", null); + Filter yeqbar = new EqualityFilter("y", ColumnType.STRING, "bar", null); + Filter zeq123 = new EqualityFilter("z", ColumnType.LONG, 123L, null); + + Filter queryFilter = xeqfoo2; + Assertions.assertInstanceOf( + Projections.ProjectionFilterMatch.class, + Projections.rewriteFilter(xeqfoo, queryFilter) + ); + + queryFilter = yeqbar; + Assertions.assertNull(Projections.rewriteFilter(xeqfoo, queryFilter)); + + queryFilter = new AndFilter(List.of(xeqfoo, yeqbar)); + Assertions.assertEquals( + yeqbar, + Projections.rewriteFilter(xeqfoo, queryFilter) + ); + + queryFilter = new AndFilter(List.of(new OrFilter(List.of(xeqfoo, xeqbar)), yeqbar)); + Assertions.assertNull(Projections.rewriteFilter(xeqfoo, queryFilter)); + + queryFilter = new AndFilter(List.of(new IsBooleanFilter(xeqfoo, true), yeqbar)); + Assertions.assertEquals(yeqbar, Projections.rewriteFilter(xeqfoo, queryFilter)); + + queryFilter = new AndFilter(List.of(new IsBooleanFilter(xeqfoo, false), yeqbar)); + Assertions.assertNull(Projections.rewriteFilter(xeqfoo, queryFilter)); + + queryFilter = new AndFilter(List.of(new AndFilter(List.of(xeqfoo, yeqbar)), zeq123)); + Assertions.assertEquals( + new AndFilter(List.of(yeqbar, zeq123)), + Projections.rewriteFilter(xeqfoo, queryFilter) + ); + } +} \ No newline at end of file From dcf8e2f254f01f472b343ae7f362955f682a9e49 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 6 Aug 2025 18:25:08 -0700 Subject: [PATCH 6/9] builders, improvements, fixes, etc --- .../benchmark/query/SqlBenchmarkDatasets.java | 93 +++-- .../segment/DatasketchesProjectionTest.java | 66 ++-- .../catalog/storage/TableManagerTest.java | 28 +- .../apache/druid/msq/exec/MSQInsertTest.java | 127 +++--- .../msq/indexing/MSQCompactionRunnerTest.java | 29 +- .../DataSourceMSQDestinationTest.java | 56 ++- .../indexing/common/task/CompactionTask.java | 14 +- .../task/CompactionTaskParallelRunTest.java | 45 +-- .../common/task/CompactionTaskTest.java | 28 +- .../input/impl/AggregateProjectionSpec.java | 82 +++- .../segment/AggregateProjectionMetadata.java | 105 ++++- .../apache/druid/segment/CursorBuildSpec.java | 16 + .../segment/projections/Projections.java | 365 +++++++++++------- .../query/metadata/SegmentAnalysisTest.java | 33 +- ...egmentMetadataQueryQueryToolChestTest.java | 62 ++- .../segment/CursorFactoryProjectionTest.java | 343 +++++++--------- .../druid/segment/IndexMergerTestBase.java | 37 +- .../apache/druid/segment/MetadataTest.java | 102 ++--- .../QueryableIndexCursorHolderTest.java | 63 ++- .../org/apache/druid/segment/TestIndex.java | 37 +- .../OnheapIncrementalIndexTest.java | 276 ++++++------- .../segment/projections/ProjectionsTest.java | 221 +++++++++-- .../segment/indexing/DataSchemaTest.java | 95 ++--- .../compaction/CompactionStatusTest.java | 75 ++-- ...CatalogDataSourceCompactionConfigTest.java | 26 +- .../coordinator/duty/CompactSegmentsTest.java | 56 ++- 26 files changed, 1344 insertions(+), 1136 deletions(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java index 8892aef8bc3e..9084c9fbaa17 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java @@ -35,7 +35,6 @@ import org.apache.druid.query.aggregation.datasketches.theta.SketchMergeAggregatorFactory; import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.segment.AutoTypeColumnSchema; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.generator.GeneratorBasicSchemas; import org.apache.druid.segment.generator.GeneratorSchemaInfo; import org.apache.druid.segment.transform.ExpressionTransform; @@ -162,36 +161,47 @@ public class SqlBenchmarkDatasets makeDimensionsSpec(expressionsSchema), expressionsSchema.getAggsArray(), Arrays.asList( - new AggregateProjectionSpec( - "string2_hourly_sums_hll", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList( - new StringDimensionSchema("string2"), - new LongDimensionSchema("__gran") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("long4_sum", "long4"), - new DoubleSumAggregatorFactory("double2_sum", "double2"), - new HllSketchBuildAggregatorFactory("hll_string5", "string5", null, null, null, false, true) - } - ), - new AggregateProjectionSpec( - "string2_long2_sums", - null, - VirtualColumns.EMPTY, - Arrays.asList( - new StringDimensionSchema("string2"), - new LongDimensionSchema("long2") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("long4_sum", "long4"), - new DoubleSumAggregatorFactory("double2_sum", "double2"), - new HllSketchBuildAggregatorFactory("hll_string5", "string5", null, null, null, false, true) - } - ) + AggregateProjectionSpec.builder("string2_hourly_sums_hll") + .virtualColumns( + Granularities.toVirtualColumn(Granularities.HOUR, "__gran") + ) + .groupingColumns( + new StringDimensionSchema("string2"), + new LongDimensionSchema("__gran") + ) + .aggregators( + new LongSumAggregatorFactory("long4_sum", "long4"), + new DoubleSumAggregatorFactory("double2_sum", "double2"), + new HllSketchBuildAggregatorFactory( + "hll_string5", + "string5", + null, + null, + null, + false, + true + ) + ) + .build(), + AggregateProjectionSpec.builder("string2_long2_sums") + .groupingColumns( + new StringDimensionSchema("string2"), + new LongDimensionSchema("long2") + ) + .aggregators( + new LongSumAggregatorFactory("long4_sum", "long4"), + new DoubleSumAggregatorFactory("double2_sum", "double2"), + new HllSketchBuildAggregatorFactory( + "hll_string5", + "string5", + null, + null, + null, + false, + true + ) + ) + .build() ), Granularities.NONE ) @@ -409,16 +419,17 @@ public BenchmarkSchema asAutoDimensions() ), aggregators, projections.stream() - .map(projection -> new AggregateProjectionSpec( - projection.getName(), - null, - projection.getVirtualColumns(), - projection.getGroupingColumns() - .stream() - .map(dim -> new AutoTypeColumnSchema(dim.getName(), null)) - .collect(Collectors.toList()), - projection.getAggregators() - )).collect(Collectors.toList()), + .map( + projection -> + AggregateProjectionSpec.builder(projection) + .groupingColumns( + projection.getGroupingColumns() + .stream() + .map(dim -> new AutoTypeColumnSchema(dim.getName(), null)) + .collect(Collectors.toList()) + ) + .build() + ).collect(Collectors.toList()), queryGranularity ); } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/segment/DatasketchesProjectionTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/segment/DatasketchesProjectionTest.java index 589fa3dad0c1..6073b0623032 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/segment/DatasketchesProjectionTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/segment/DatasketchesProjectionTest.java @@ -47,7 +47,6 @@ import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.query.DruidProcessingConfig; import org.apache.druid.query.QueryContexts; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.datasketches.hll.HllSketchBuildAggregatorFactory; import org.apache.druid.query.aggregation.datasketches.hll.HllSketchHolder; import org.apache.druid.query.aggregation.datasketches.hll.HllSketchModule; @@ -98,38 +97,43 @@ public class DatasketchesProjectionTest extends InitializedNullHandlingTest private static final Closer CLOSER = Closer.create(); private static final List PROJECTIONS = Collections.singletonList( - new AggregateProjectionSpec( - "a_projection", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList( - new LongDimensionSchema("__gran"), - new StringDimensionSchema("a") - ), - new AggregatorFactory[]{ - new HllSketchBuildAggregatorFactory("_b_hll", "b", null, null, null, null, false), - new SketchMergeAggregatorFactory("_b_theta", "b", null, null, false, null), - new DoublesSketchAggregatorFactory("_d_doubles", "d", null), - new ArrayOfDoublesSketchAggregatorFactory("_bcd_aod", "b", null, Arrays.asList("c", "d"), null), - new KllDoublesSketchAggregatorFactory("_d_kll", "d", null, null) - } - ) + AggregateProjectionSpec.builder("a_projection") + .virtualColumns( + Granularities.toVirtualColumn(Granularities.HOUR, "__gran") + ) + .groupingColumns( + new LongDimensionSchema("__gran"), + new StringDimensionSchema("a") + ) + .aggregators( + new HllSketchBuildAggregatorFactory("_b_hll", "b", null, null, null, null, false), + new SketchMergeAggregatorFactory("_b_theta", "b", null, null, false, null), + new DoublesSketchAggregatorFactory("_d_doubles", "d", null), + new ArrayOfDoublesSketchAggregatorFactory( + "_bcd_aod", + "b", + null, + Arrays.asList("c", "d"), + null + ), + new KllDoublesSketchAggregatorFactory("_d_kll", "d", null, null) + ) + .build() ); - private static final List AUTO_PROJECTIONS = PROJECTIONS.stream().map(projection -> { - return new AggregateProjectionSpec( - projection.getName(), - projection.getFilter(), - projection.getVirtualColumns(), - projection.getGroupingColumns() - .stream() - .map(x -> new AutoTypeColumnSchema(x.getName(), null)) - .collect(Collectors.toList()), - projection.getAggregators() - ); - }).collect(Collectors.toList()); + private static final List AUTO_PROJECTIONS = + PROJECTIONS.stream() + .map( + projection -> + AggregateProjectionSpec.builder(projection) + .groupingColumns( + projection.getGroupingColumns() + .stream() + .map(x -> new AutoTypeColumnSchema(x.getName(), null)) + .collect(Collectors.toList()) + ) + .build() + ).collect(Collectors.toList()); @Parameterized.Parameters(name = "name: {0}, sortByDim: {3}, autoSchema: {4}") public static Collection constructorFeeder() diff --git a/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java b/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java index ba835ab24004..8947a161558e 100644 --- a/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java +++ b/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java @@ -40,9 +40,7 @@ import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.metadata.TestDerbyConnector; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; -import org.apache.druid.segment.VirtualColumns; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -94,22 +92,16 @@ public void tearDown() public void testCreate() throws DuplicateKeyException, NotFoundException { final DatasourceProjectionMetadata projectionMetadata = new DatasourceProjectionMetadata( - new AggregateProjectionSpec( - "projection", - null, - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of( - new StringDimensionSchema("dim") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("count") - } - ) + AggregateProjectionSpec.builder("projection") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns(new StringDimensionSchema("dim")) + .aggregators(new CountAggregatorFactory("count")) + .build() ); List jsonProjectionSpec = JSON_MAPPER.convertValue(ImmutableList.of(projectionMetadata), List.class); diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java index b8262df7954c..4cfd1dcd363d 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java @@ -57,9 +57,7 @@ import org.apache.druid.msq.test.CounterSnapshotMatcher; import org.apache.druid.msq.test.MSQTestBase; import org.apache.druid.msq.util.MultiStageQueryContext; -import org.apache.druid.query.OrderBy; import org.apache.druid.query.QueryContexts; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; @@ -69,7 +67,6 @@ import org.apache.druid.query.groupby.orderby.OrderByColumnSpec; import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; import org.apache.druid.segment.AggregateProjectionMetadata; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.ValueType; @@ -196,42 +193,38 @@ protected CatalogResolver createMockCatalogResolver() DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, ImmutableList.of( new DatasourceProjectionMetadata( - new AggregateProjectionSpec( - "channel_added_hourly", - null, - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of( - new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - new StringDimensionSchema("channel") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_added", "added") - } - ) + AggregateProjectionSpec.builder("channel_added_hourly") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + new StringDimensionSchema("channel") + ) + .aggregators( + new LongSumAggregatorFactory("sum_added", "added") + ) + .build() ), new DatasourceProjectionMetadata( - new AggregateProjectionSpec( - "channel_delta_daily", - null, - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.DAY, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of( - new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - new StringDimensionSchema("channel") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_delta", "delta") - } - ) + AggregateProjectionSpec.builder("channel_delta_daily") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.DAY, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + new StringDimensionSchema("channel") + ) + .aggregators( + new LongSumAggregatorFactory("sum_delta", "delta") + ) + .build() ) ) ) @@ -552,47 +545,31 @@ public void testInsertOnExternalDataSourceWithCatalogProjections(String contextN .add("delta", ColumnType.LONG) .build(); AggregateProjectionMetadata expectedProjection = new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "channel_added_hourly", - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, - null, - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel"), - new AggregatorFactory[] { - new LongSumAggregatorFactory("sum_added", "added") - }, - ImmutableList.of( - OrderBy.ascending(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - OrderBy.ascending("channel") - ) - ), + AggregateProjectionMetadata.schemaBuilder("channel_added_hourly") + .timeColumnName(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupAndOrder(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel") + .aggregators(new LongSumAggregatorFactory("sum_added", "added")) + .build(), 16 ); AggregateProjectionMetadata expectedProjection2 = new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "channel_delta_daily", - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, - null, - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.DAY, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel"), - new AggregatorFactory[] { - new LongSumAggregatorFactory("sum_delta", "delta") - }, - ImmutableList.of( - OrderBy.ascending(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - OrderBy.ascending("channel") - ) - ), + AggregateProjectionMetadata.schemaBuilder("channel_delta_daily") + .timeColumnName(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.DAY, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupAndOrder(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel") + .aggregators(new LongSumAggregatorFactory("sum_delta", "delta")) + .build(), 11 ); diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java index 7bcd3a041507..03fec193603f 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java @@ -64,7 +64,6 @@ import org.apache.druid.segment.AutoTypeColumnSchema; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.NestedDataColumnSchema; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.data.CompressionFactory; @@ -81,7 +80,6 @@ import org.junit.Test; import javax.annotation.Nullable; - import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -119,20 +117,19 @@ public class MSQCompactionRunnerTest NESTED_DIMENSION, AUTO_DIMENSION ); - private static final AggregateProjectionSpec PROJECTION_SPEC = new AggregateProjectionSpec( - "projection", - null, - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of(STRING_DIMENSION), - new AggregatorFactory[]{ - new LongSumAggregatorFactory(LONG_DIMENSION.getName(), LONG_DIMENSION.getName()) - } - ); + private static final AggregateProjectionSpec PROJECTION_SPEC = + AggregateProjectionSpec.builder("projection") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns(STRING_DIMENSION) + .aggregators( + new LongSumAggregatorFactory(LONG_DIMENSION.getName(), LONG_DIMENSION.getName()) + ) + .build(); private static final Map INTERVAL_DATASCHEMAS = ImmutableMap.of( COMPACTION_INTERVAL, new CombinedDataSchema( diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java index a7c5970a829e..14160c5ff7a0 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.data.input.impl.AggregateProjectionSpec; @@ -29,10 +28,8 @@ import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; -import org.apache.druid.segment.VirtualColumns; import org.junit.Assert; import org.junit.Test; @@ -68,38 +65,29 @@ public void testEquals() ) .withPrefabValues( List.class, - ImmutableList.of( - new AggregateProjectionSpec( - "projection1", - null, - VirtualColumns.EMPTY, - ImmutableList.of( - new StringDimensionSchema("region") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("count") - } - ) + List.of( + AggregateProjectionSpec.builder("projection1") + .groupingColumns(new StringDimensionSchema("region")) + .aggregators(new CountAggregatorFactory("count")) + .build() ), - ImmutableList.of( - new AggregateProjectionSpec( - "projection2", - null, - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of( - new StringDimensionSchema("language"), - new StringDimensionSchema("region") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("count"), - new LongSumAggregatorFactory("sum_added", "added") - } - ) + List.of( + AggregateProjectionSpec.builder("projection2") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new StringDimensionSchema("language"), + new StringDimensionSchema("region") + ) + .aggregators( + new CountAggregatorFactory("count"), + new LongSumAggregatorFactory("sum_added", "added") + ) + .build() ) ) .usingGetClass() diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java index dc9a3033f18d..2a6457e36349 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java @@ -1178,13 +1178,13 @@ private void processProjections(final QueryableIndex index) } projections.put( schema.getName(), - new AggregateProjectionSpec( - schema.getName(), - schema.getFilter(), - schema.getVirtualColumns(), - columnSchemas, - schema.getAggregators() - ) + AggregateProjectionSpec.builder() + .name(schema.getName()) + .virtualColumns(schema.getVirtualColumns()) + .filter(schema.getFilter()) + .groupingColumns(columnSchemas) + .aggregators(schema.getAggregators()) + .build() ); } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java index 118011c2f26e..e78085b0b104 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java @@ -63,7 +63,6 @@ import org.apache.druid.query.filter.SelectorDimFilter; import org.apache.druid.segment.DataSegmentsWithSchemas; import org.apache.druid.segment.SegmentUtils; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.loading.NoopSegmentCacheManager; import org.apache.druid.segment.transform.CompactionTransformSpec; @@ -118,23 +117,20 @@ public static Iterable constructorFeeder() private static final String DATA_SOURCE = "test"; private static final Interval INTERVAL_TO_INDEX = Intervals.of("2014-01-01/2014-01-02"); - private static final AggregateProjectionSpec PROJECTION_SPEC = new AggregateProjectionSpec( - "projection1", - null, - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of( - new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - new StringDimensionSchema("dim", DimensionSchema.MultiValueHandling.ARRAY, null) - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("val", "val") - } - ); + private static final AggregateProjectionSpec PROJECTION_SPEC = + AggregateProjectionSpec.builder("projection1") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + new StringDimensionSchema("dim", DimensionSchema.MultiValueHandling.ARRAY, null) + ) + .aggregators(new LongSumAggregatorFactory("val", "val")) + .build(); private final LockGranularity lockGranularity; @@ -958,15 +954,10 @@ public void testRunParallelAddProjections() DATA_SOURCE, getSegmentCacheManagerFactory() ); - final AggregateProjectionSpec addProjection = new AggregateProjectionSpec( - "projection2", - null, - VirtualColumns.EMPTY, - null, - new AggregatorFactory[]{ - new LongSumAggregatorFactory("val", "val") - } - ); + final AggregateProjectionSpec addProjection = + AggregateProjectionSpec.builder("projection2") + .aggregators(new LongSumAggregatorFactory("val", "val")) + .build(); final CompactionTask compactionTask = builder .inputSpec(new CompactionIntervalSpec(INTERVAL_TO_INDEX, null)) .tuningConfig(AbstractParallelIndexSupervisorTaskTest.DEFAULT_TUNING_CONFIG_FOR_PARALLEL_INDEXING) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java index b0ae79988b3a..b14ae4c09e80 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java @@ -105,7 +105,6 @@ import org.apache.druid.segment.SegmentUtils; import org.apache.druid.segment.SimpleQueryableIndex; import org.apache.druid.segment.TestIndex; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.BaseColumn; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnCapabilitiesImpl; @@ -573,21 +572,18 @@ public void testSerdeWithProjections() throws IOException DATA_SOURCE, segmentCacheManagerFactory ); - final List projections = ImmutableList.of( - new AggregateProjectionSpec( - "test", - null, - VirtualColumns.EMPTY, - ImmutableList.of( - new StringDimensionSchema("dim1"), - new StringDimensionSchema("dim2"), - new StringDimensionSchema("dim3") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("count"), - new LongSumAggregatorFactory("sum_long_dim_1", "long_dim_1") - } - ) + final List projections = List.of( + AggregateProjectionSpec.builder("test") + .groupingColumns( + new StringDimensionSchema("dim1"), + new StringDimensionSchema("dim2"), + new StringDimensionSchema("dim3") + ) + .aggregators( + new CountAggregatorFactory("count"), + new LongSumAggregatorFactory("sum_long_dim_1", "long_dim_1") + ) + .build() ); final CompactionTask task = builder diff --git a/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java b/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java index 7539ff3b0f3f..d9e09cc6fba7 100644 --- a/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java +++ b/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java @@ -57,6 +57,25 @@ public class AggregateProjectionSpec { public static final String TYPE_NAME = "aggregate"; + public static Builder builder() + { + return new Builder(); + } + + public static Builder builder(String name) + { + return new Builder().name(name); + } + + public static Builder builder(AggregateProjectionSpec spec) + { + return new Builder().name(spec.getName()) + .virtualColumns(spec.getVirtualColumns()) + .filter(spec.getFilter()) + .groupingColumns(spec.getGroupingColumns()) + .aggregators(spec.getAggregators()); + } + private final String name; @Nullable private final DimFilter filter; @@ -189,7 +208,6 @@ public String toString() '}'; } - private static ProjectionOrdering computeOrdering(VirtualColumns virtualColumns, List groupingColumns) { if (groupingColumns.isEmpty()) { @@ -236,4 +254,66 @@ private ProjectionOrdering(List ordering, @Nullable String timeColumnNa this.timeColumnName = timeColumnName; } } + + public static final class Builder + { + private String name; + private DimFilter filter; + private VirtualColumns virtualColumns = VirtualColumns.EMPTY; + private List groupingColumns; + private AggregatorFactory[] aggregators; + + public Builder name(String name) + { + this.name = name; + return this; + } + + public Builder filter(@Nullable DimFilter filter) + { + this.filter = filter; + return this; + } + + public Builder virtualColumns(@Nullable VirtualColumns virtualColumns) + { + this.virtualColumns = virtualColumns; + return this; + } + + public Builder virtualColumns(VirtualColumn... virtualColumns) + { + this.virtualColumns = VirtualColumns.create(virtualColumns); + return this; + } + + public Builder groupingColumns(@Nullable List groupingColumns) + { + this.groupingColumns = groupingColumns; + return this; + } + + public Builder groupingColumns(DimensionSchema... groupingColumns) + { + this.groupingColumns = Arrays.asList(groupingColumns); + return this; + } + + public Builder aggregators(@Nullable AggregatorFactory... aggregators) + { + this.aggregators = aggregators; + return this; + } + + public AggregateProjectionSpec build() + { + return new AggregateProjectionSpec( + name, + filter, + virtualColumns, + groupingColumns, + aggregators + ); + } + } } diff --git a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java index b6f872cc50c7..642112cb3a12 100644 --- a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java +++ b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java @@ -36,6 +36,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.filter.DimFilter; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.projections.Projections; import org.apache.druid.utils.CollectionUtils; import javax.annotation.Nullable; @@ -44,6 +45,7 @@ import java.util.Comparator; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; /** * Aggregate projection schema and row count information to store in {@link Metadata} which itself is stored inside a @@ -66,6 +68,11 @@ public class AggregateProjectionMetadata return Schema.COMPARATOR.compare(o1.getSchema(), o2.getSchema()); }; + public static SchemaBuilder schemaBuilder(String name) + { + return new SchemaBuilder().name(name); + } + private final Schema schema; private final int numRows; @@ -297,10 +304,10 @@ public Granularity getEffectiveGranularity() * Check if a column is either part of {@link #groupingColumns}, or at least is not present in * {@link #virtualColumns}. Naively, we would just check that grouping column contains the column in question, * however, we can also use a projection when a column is truly missing. - * {@link org.apache.druid.segment.projections.Projections#matchAggregateProjection} returns a match builder if the - * column is present as either a physical column, or a virtual column, but a virtual column could also be present - * for an aggregator input, so we must further check that a column not in the grouping list is also not a virtual - * column, the implication being that it is a missing column. + * {@link Projections#matchAggregateProjection(Schema, CursorBuildSpec, Projections.PhysicalColumnChecker)} returns + * a match builder if the column is present as either a physical column, or a virtual column, but a virtual column + * could also be present for an aggregator input, so we must further check that a column not in the grouping list + * is also not a virtual column, the implication being that it is a missing column. */ public boolean isInvalidGrouping(@Nullable String columnName) { @@ -359,4 +366,94 @@ public String toString() '}'; } } + + public static class SchemaBuilder + { + /* + @JsonProperty("name") String name, + @JsonProperty("timeColumnName") @Nullable String timeColumnName, + @JsonProperty("filter") @Nullable DimFilter filter, + @JsonProperty("virtualColumns") @Nullable VirtualColumns virtualColumns, + @JsonProperty("groupingColumns") @Nullable List groupingColumns, + @JsonProperty("aggregators") @Nullable AggregatorFactory[] aggregators, + @JsonProperty("ordering") List ordering + */ + @Nullable + private String name; + @Nullable + private String timeColumnName; + private VirtualColumns virtualColumns = VirtualColumns.EMPTY; + @Nullable + private DimFilter filter; + private List groupingColumns; + private AggregatorFactory[] aggregators; + private List ordering; + + public SchemaBuilder name(@Nullable String name) + { + this.name = name; + return this; + } + + public SchemaBuilder timeColumnName(@Nullable String timeColumnName) + { + this.timeColumnName = timeColumnName; + return this; + } + + public SchemaBuilder virtualColumns(VirtualColumns virtualColumns) + { + this.virtualColumns = virtualColumns; + return this; + } + + public SchemaBuilder virtualColumns(VirtualColumn... virtualColumns) + { + this.virtualColumns = VirtualColumns.create(virtualColumns); + return this; + } + + public SchemaBuilder filter(@Nullable DimFilter filter) + { + this.filter = filter; + return this; + } + + public SchemaBuilder groupAndOrder(String... groupingColumns) + { + this.groupingColumns = Arrays.asList(groupingColumns); + return ordering(groupingColumns); + } + + public SchemaBuilder aggregators(@Nullable AggregatorFactory... aggregators) + { + this.aggregators = aggregators; + return this; + } + + public SchemaBuilder ordering(final String... columnNames) + { + this.ordering = Arrays.stream(columnNames).map(OrderBy::ascending).collect(Collectors.toList()); + return this; + } + + public SchemaBuilder ordering(List ordering) + { + this.ordering = ordering; + return this; + } + + public Schema build() + { + return new Schema( + name, + timeColumnName, + filter, + virtualColumns, + groupingColumns, + aggregators, + ordering + ); + } + } } diff --git a/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java b/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java index aca3743cf6c9..35e847818e30 100644 --- a/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java +++ b/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java @@ -278,6 +278,22 @@ public int hashCode() ); } + @Override + public String toString() + { + return "CursorBuildSpec{" + + "filter=" + filter + + ", interval=" + interval + + ", groupingColumns=" + groupingColumns + + ", virtualColumns=" + virtualColumns + + ", aggregators=" + aggregators + + ", preferredOrdering=" + preferredOrdering + + ", queryContext=" + queryContext + + ", isAggregate=" + isAggregate + + ", physicalColumns=" + physicalColumns + + '}'; + } + public static class CursorBuildSpecBuilder { @Nullable diff --git a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java index a1faf165f499..0c1857702801 100644 --- a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java +++ b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java @@ -104,6 +104,7 @@ public static QueryableProjection findMatchingProjection( * aggregator factories to process the pre-aggregated data from the projection, as well as a mapping of query column * names to projection column names. * + * @param projection the {@link AggregateProjectionMetadata.Schema} to check for match * @param queryCursorBuildSpec the {@link CursorBuildSpec} that contains the required inputs to build a * {@link CursorHolder} for a query * @param physicalColumnChecker Helper utility which can determine if a physical column required by @@ -124,27 +125,66 @@ public static ProjectionMatch matchAggregateProjection( } ProjectionMatchBuilder matchBuilder = new ProjectionMatchBuilder(); - matchBuilder = matchGrouping(projection, queryCursorBuildSpec, physicalColumnChecker, matchBuilder); + // match virtual columns first, which will populate the 'remapColumns' of the match builder + matchBuilder = matchQueryVirtualColumns(projection, queryCursorBuildSpec, physicalColumnChecker, matchBuilder); if (matchBuilder == null) { return null; } - if (matchAggregators(projection, queryCursorBuildSpec, matchBuilder)) { + matchBuilder = matchFilter(projection, queryCursorBuildSpec, physicalColumnChecker, matchBuilder); + if (matchBuilder == null) { return null; } - matchBuilder = matchFilter(projection, queryCursorBuildSpec, physicalColumnChecker, matchBuilder); + matchBuilder = matchGrouping(projection, queryCursorBuildSpec, physicalColumnChecker, matchBuilder); if (matchBuilder == null) { return null; } - matchBuilder = matchRemainingRequiredColumns(projection, queryCursorBuildSpec, physicalColumnChecker, matchBuilder); + matchBuilder = matchAggregators(projection, queryCursorBuildSpec, matchBuilder); if (matchBuilder == null) { return null; } - // todo (clint): combine this with match filter, but need to figure out how to handle remapping since ideally all - // the remapping has been done before we match filters so the filter required column rewrites are chill + matchBuilder = matchRemainingPhysicalColumns(projection, queryCursorBuildSpec, physicalColumnChecker, matchBuilder); + if (matchBuilder == null) { + return null; + } + + return matchBuilder.build(queryCursorBuildSpec); + } + + @Nullable + public static ProjectionMatchBuilder matchQueryVirtualColumns( + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + PhysicalColumnChecker physicalColumnChecker, + ProjectionMatchBuilder matchBuilder + ) + { + for (VirtualColumn vc : queryCursorBuildSpec.getVirtualColumns().getVirtualColumns()) { + matchBuilder = matchQueryVirtualColumn( + vc, + projection, + queryCursorBuildSpec, + physicalColumnChecker, + matchBuilder + ); + if (matchBuilder == null) { + return null; + } + } + return matchBuilder; + } + + @Nullable + public static ProjectionMatchBuilder matchFilter( + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + PhysicalColumnChecker physicalColumnChecker, + ProjectionMatchBuilder matchBuilder + ) + { // if the projection has a filter, the query must contain this filter match if (projection.getFilter() != null) { final Filter queryFilter = queryCursorBuildSpec.getFilter(); @@ -183,11 +223,27 @@ public static ProjectionMatch matchAggregateProjection( matchBuilder.rewriteFilter(queryCursorBuildSpec.getFilter()); } - return matchBuilder.build(queryCursorBuildSpec); + // now that filter has been possibly rewritten, make sure the projection actually has all the required columns. + if (matchBuilder.getRewriteFilter() != null) { + for (String queryColumn : matchBuilder.getRewriteFilter().getRequiredColumns()) { + matchBuilder = matchRequiredColumn( + queryColumn, + projection, + queryCursorBuildSpec, + physicalColumnChecker, + matchBuilder + ); + if (matchBuilder == null) { + return null; + } + } + } + + return matchBuilder; } @Nullable - private static ProjectionMatchBuilder matchGrouping( + public static ProjectionMatchBuilder matchGrouping( AggregateProjectionMetadata.Schema projection, CursorBuildSpec queryCursorBuildSpec, PhysicalColumnChecker physicalColumnChecker, @@ -198,11 +254,11 @@ private static ProjectionMatchBuilder matchGrouping( if (queryGrouping != null) { for (String queryColumn : queryGrouping) { matchBuilder = matchRequiredColumn( - projection, - matchBuilder, queryColumn, - queryCursorBuildSpec.getVirtualColumns(), - physicalColumnChecker + projection, + queryCursorBuildSpec, + physicalColumnChecker, + matchBuilder ); if (matchBuilder == null) { return null; @@ -220,63 +276,40 @@ private static ProjectionMatchBuilder matchGrouping( return matchBuilder; } - - private static boolean matchAggregators( - AggregateProjectionMetadata.Schema projection, - CursorBuildSpec queryCursorBuildSpec, - ProjectionMatchBuilder matchBuilder - ) - { - if (!CollectionUtils.isNullOrEmpty(queryCursorBuildSpec.getAggregators())) { - boolean allMatch = true; - for (AggregatorFactory queryAgg : queryCursorBuildSpec.getAggregators()) { - boolean foundMatch = false; - for (AggregatorFactory projectionAgg : projection.getAggregators()) { - final AggregatorFactory combining = queryAgg.substituteCombiningFactory(projectionAgg); - if (combining != null) { - matchBuilder.remapColumn(queryAgg.getName(), projectionAgg.getName()) - .addReferencedPhysicalColumn(projectionAgg.getName()) - .addPreAggregatedAggregator(combining) - .addMatchedQueryColumns(queryAgg.requiredFields()); - foundMatch = true; - break; - } - } - allMatch = allMatch && foundMatch; - } - return !allMatch; - } - return false; - } - @Nullable - private static ProjectionMatchBuilder matchFilter( + public static ProjectionMatchBuilder matchAggregators( AggregateProjectionMetadata.Schema projection, CursorBuildSpec queryCursorBuildSpec, - PhysicalColumnChecker physicalColumnChecker, ProjectionMatchBuilder matchBuilder ) { - if (queryCursorBuildSpec.getFilter() != null) { - for (String queryColumn : queryCursorBuildSpec.getFilter().getRequiredColumns()) { - matchBuilder = matchRequiredColumn( - projection, - matchBuilder, - queryColumn, - queryCursorBuildSpec.getVirtualColumns(), - physicalColumnChecker - ); - if (matchBuilder == null) { - return null; + if (CollectionUtils.isNullOrEmpty(queryCursorBuildSpec.getAggregators())) { + return matchBuilder; + } + boolean allMatch = true; + for (AggregatorFactory queryAgg : queryCursorBuildSpec.getAggregators()) { + boolean foundMatch = false; + for (AggregatorFactory projectionAgg : projection.getAggregators()) { + final AggregatorFactory combining = queryAgg.substituteCombiningFactory(projectionAgg); + if (combining != null) { + matchBuilder.remapColumn(queryAgg.getName(), projectionAgg.getName()) + .addReferencedPhysicalColumn(projectionAgg.getName()) + .addPreAggregatedAggregator(combining) + .addMatchedQueryColumns(queryAgg.requiredFields()); + foundMatch = true; + break; } } + allMatch = allMatch && foundMatch; } - - return matchBuilder; + if (allMatch) { + return matchBuilder; + } + return null; } @Nullable - private static ProjectionMatchBuilder matchRemainingRequiredColumns( + public static ProjectionMatchBuilder matchRemainingPhysicalColumns( AggregateProjectionMetadata.Schema projection, CursorBuildSpec queryCursorBuildSpec, PhysicalColumnChecker physicalColumnChecker, @@ -293,26 +326,11 @@ private static ProjectionMatchBuilder matchRemainingRequiredColumns( continue; } if (!matchedQueryColumns.contains(queryColumn)) { - matchBuilder = matchRequiredColumn( - projection, - matchBuilder, + matchBuilder = matchQueryPhysicalColumn( queryColumn, - queryCursorBuildSpec.getVirtualColumns(), - physicalColumnChecker - ); - if (matchBuilder == null) { - return null; - } - } - } - for (VirtualColumn vc : queryCursorBuildSpec.getVirtualColumns().getVirtualColumns()) { - if (!matchedQueryColumns.contains(vc.getOutputName())) { - matchBuilder = matchRequiredColumn( projection, - matchBuilder, - vc.getOutputName(), - queryCursorBuildSpec.getVirtualColumns(), - physicalColumnChecker + physicalColumnChecker, + matchBuilder ); if (matchBuilder == null) { return null; @@ -336,89 +354,123 @@ private static ProjectionMatchBuilder matchRemainingRequiredColumns( * which true if the column is present on the projection OR if the column is NOT present on the base table (meaning * missing columns that do not exist anywhere do not disqualify a projection from being used). * - * @param matchBuilder match state to add mappings of query virtual columns to projection physical columns - * and query virtual columns which still must be computed from projection physical - * columns * @param column Column name to check - * @param queryVirtualColumns {@link VirtualColumns} from the {@link CursorBuildSpec} required by the query + * @param projection {@link AggregateProjectionMetadata.Schema} to match against + * @param queryCursorBuildSpec the {@link CursorBuildSpec} required by the query * @param physicalColumnChecker Helper to check if the physical column exists on a projection, or does not exist on * the base table + * @param matchBuilder match state to add mappings of query virtual columns to projection physical columns + * and query virtual columns which still must be computed from projection physical + * columns * @return {@link ProjectionMatchBuilder} with updated state per the rules described above, or null if the column * cannot be matched */ @Nullable - private static ProjectionMatchBuilder matchRequiredColumn( - AggregateProjectionMetadata.Schema projection, - ProjectionMatchBuilder matchBuilder, + public static ProjectionMatchBuilder matchRequiredColumn( String column, - VirtualColumns queryVirtualColumns, - PhysicalColumnChecker physicalColumnChecker + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + PhysicalColumnChecker physicalColumnChecker, + ProjectionMatchBuilder matchBuilder ) { - final VirtualColumn queryVirtualColumn = queryVirtualColumns.getVirtualColumn(column); - if (queryVirtualColumn != null) { - matchBuilder.addMatchedQueryColumn(column) - .addMatchedQueryColumns(queryVirtualColumn.requiredColumns()); - // check to see if we have an equivalent virtual column defined in the projection, if so we can - final VirtualColumn projectionEquivalent = projection.getVirtualColumns().findEquivalent(queryVirtualColumn); - if (projectionEquivalent != null) { - final String remapColumnName; - if (Objects.equals(projectionEquivalent.getOutputName(), projection.getTimeColumnName())) { - remapColumnName = ColumnHolder.TIME_COLUMN_NAME; - } else { - remapColumnName = projectionEquivalent.getOutputName(); - } - if (!queryVirtualColumn.getOutputName().equals(remapColumnName)) { - matchBuilder.remapColumn(queryVirtualColumn.getOutputName(), remapColumnName); - } - return matchBuilder.addReferencedPhysicalColumn(remapColumnName); - } + if (matchBuilder.getMatchedQueryColumns().contains(column)) { + return matchBuilder; + } + final VirtualColumn virtualColumn = queryCursorBuildSpec.getVirtualColumns().getVirtualColumn(column); + if (virtualColumn != null) { + return matchQueryVirtualColumn( + virtualColumn, + projection, + queryCursorBuildSpec, + physicalColumnChecker, + matchBuilder + ); + } - matchBuilder.addReferenceedVirtualColumn(queryVirtualColumn); - final List requiredInputs = queryVirtualColumn.requiredColumns(); - if (requiredInputs.size() == 1 && ColumnHolder.TIME_COLUMN_NAME.equals(requiredInputs.get(0))) { - // special handle time granularity. in the future this should be reworked to push this concept into the - // virtual column and underlying expression itself, but this will do for now - final Granularity virtualGranularity = Granularities.fromVirtualColumn(queryVirtualColumn); - if (virtualGranularity != null) { - if (virtualGranularity.isFinerThan(projection.getEffectiveGranularity())) { - return null; - } - // same granularity, replace virtual column directly by remapping it to the physical column - if (projection.getEffectiveGranularity().equals(virtualGranularity)) { - return matchBuilder.remapColumn(column, ColumnHolder.TIME_COLUMN_NAME) - .addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); - } - return matchBuilder.addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); - } else { - // anything else with __time requires none granularity - if (Granularities.NONE.equals(projection.getEffectiveGranularity())) { - return matchBuilder.addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); - } + return matchQueryPhysicalColumn(column, projection, physicalColumnChecker, matchBuilder); + } + + @Nullable + public static ProjectionMatchBuilder matchQueryVirtualColumn( + VirtualColumn queryVirtualColumn, + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + PhysicalColumnChecker physicalColumnChecker, + ProjectionMatchBuilder matchBuilder + ) + { + // check to see if we have an equivalent virtual column defined in the projection, if so we can + final VirtualColumn projectionEquivalent = projection.getVirtualColumns().findEquivalent(queryVirtualColumn); + if (projectionEquivalent != null) { + final String remapColumnName; + if (Objects.equals(projectionEquivalent.getOutputName(), projection.getTimeColumnName())) { + remapColumnName = ColumnHolder.TIME_COLUMN_NAME; + } else { + remapColumnName = projectionEquivalent.getOutputName(); + } + if (!queryVirtualColumn.getOutputName().equals(remapColumnName)) { + matchBuilder.remapColumn(queryVirtualColumn.getOutputName(), remapColumnName); + } + return matchBuilder.addMatchedQueryColumn(queryVirtualColumn.getOutputName()) + .addMatchedQueryColumns(queryVirtualColumn.requiredColumns()) + .addReferencedPhysicalColumn(remapColumnName); + } + + matchBuilder.addMatchedQueryColumn(queryVirtualColumn.getOutputName()) + .addReferenceedVirtualColumn(queryVirtualColumn); + final List requiredInputs = queryVirtualColumn.requiredColumns(); + if (requiredInputs.size() == 1 && ColumnHolder.TIME_COLUMN_NAME.equals(requiredInputs.get(0))) { + // special handle time granularity. in the future this should be reworked to push this concept into the + // virtual column and underlying expression itself, but this will do for now + final Granularity virtualGranularity = Granularities.fromVirtualColumn(queryVirtualColumn); + if (virtualGranularity != null) { + if (virtualGranularity.isFinerThan(projection.getEffectiveGranularity())) { return null; } + // same granularity, replace virtual column directly by remapping it to the physical column + if (projection.getEffectiveGranularity().equals(virtualGranularity)) { + return matchBuilder.remapColumn(queryVirtualColumn.getOutputName(), ColumnHolder.TIME_COLUMN_NAME) + .addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); + } + return matchBuilder.addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); } else { - for (String required : requiredInputs) { - matchBuilder = matchRequiredColumn( - projection, - matchBuilder, - required, - queryVirtualColumns, - physicalColumnChecker - ); - if (matchBuilder == null) { - return null; - } + // anything else with __time requires none granularity + if (Granularities.NONE.equals(projection.getEffectiveGranularity())) { + return matchBuilder.addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); } - return matchBuilder; + return null; } } else { - if (physicalColumnChecker.check(projection.getName(), column)) { - return matchBuilder.addMatchedQueryColumn(column) - .addReferencedPhysicalColumn(column); + for (String required : requiredInputs) { + matchBuilder = matchRequiredColumn( + required, + projection, + queryCursorBuildSpec, + physicalColumnChecker, + matchBuilder + ); + if (matchBuilder == null) { + return null; + } } - return null; + return matchBuilder; + } + } + + @Nullable + public static ProjectionMatchBuilder matchQueryPhysicalColumn( + String column, + AggregateProjectionMetadata.Schema projection, + PhysicalColumnChecker physicalColumnChecker, + ProjectionMatchBuilder matchBuilder + ) + { + if (physicalColumnChecker.check(projection.getName(), column)) { + return matchBuilder.addMatchedQueryColumn(column) + .addReferencedPhysicalColumn(column); } + return null; } /** @@ -451,6 +503,26 @@ public static Filter rewriteFilter(@Nullable Filter projectionFilter, @Nullable } if (queryFilter instanceof AndFilter) { AndFilter andFilter = (AndFilter) queryFilter; + + // if both and filters, check to see if the query and filter contains all of the clauses of the projection and filter + if (projectionFilter instanceof AndFilter) { + AndFilter projectionAndFilter = (AndFilter) projectionFilter; + Filter rewritten = andFilter; + for (Filter filter : projectionAndFilter.getFilters()) { + rewritten = rewriteFilter(filter, rewritten); + if (rewritten != null) { + if (rewritten == ProjectionFilterMatch.INSTANCE) { + return ProjectionFilterMatch.INSTANCE; + } + } + } + if (rewritten != null) { + return rewritten; + } + return null; + } + + // else check to see if any clause of the query AND filter is the projection filter List newChildren = Lists.newArrayListWithExpectedSize(andFilter.getFilters().size()); boolean childRewritten = false; for (Filter filter : andFilter.getFilters()) { @@ -529,8 +601,18 @@ public int hashCode() { return Objects.hash(cursorBuildSpec, remapColumns); } + + @Override + public String toString() + { + return "ProjectionMatch{" + + "cursorBuildSpec=" + cursorBuildSpec + + ", remapColumns=" + remapColumns + + '}'; + } } + public static final class ProjectionMatchBuilder { private final Set referencedPhysicalColumns; @@ -613,6 +695,11 @@ public ProjectionMatchBuilder rewriteFilter(Filter rewriteFilter) return this; } + public Filter getRewriteFilter() + { + return rewriteFilter; + } + public Map getRemapColumns() { return remapColumns; diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java index 5096fe0c883f..df3abe0e53c3 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java @@ -26,15 +26,12 @@ import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.OrderBy; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.segment.AggregateProjectionMetadata; import org.apache.druid.segment.TestHelper; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; import org.junit.Assert; import org.junit.Test; @@ -83,25 +80,17 @@ public void testSerde() throws Exception 2, ImmutableMap.of("cnt", new CountAggregatorFactory("cnt")), ImmutableMap.of("channel_added_hourly", new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "channel_added_hourly", - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, - null, - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel"), - new AggregatorFactory[] { - new LongSumAggregatorFactory("sum_added", "added") - }, - ImmutableList.of( - OrderBy.ascending(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - OrderBy.ascending("channel") - ) - ), + AggregateProjectionMetadata.schemaBuilder("channel_added_hourly") + .timeColumnName(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupAndOrder(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel") + .aggregators(new LongSumAggregatorFactory("sum_added", "added")) + .build(), 16 )), new TimestampSpec(null, null, null), diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java index 9cdabc3d1e14..14a6b8765cb0 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java @@ -31,10 +31,8 @@ import org.apache.druid.query.CacheStrategy; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; -import org.apache.druid.query.OrderBy; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.UnionDataSource; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.DoubleMaxAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.query.aggregation.LongMaxAggregatorFactory; @@ -45,7 +43,6 @@ import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; import org.apache.druid.query.spec.LegacySegmentSpec; import org.apache.druid.segment.AggregateProjectionMetadata; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.ValueType; import org.apache.druid.timeline.LogicalSegment; @@ -70,40 +67,31 @@ public class SegmentMetadataQueryQueryToolChestTest private static final SegmentId TEST_SEGMENT_ID1 = SegmentId.of(TEST_DATASOURCE.toString(), INTERVAL_2020, "test", 0); private static final SegmentId TEST_SEGMENT_ID2 = SegmentId.of(TEST_DATASOURCE.toString(), INTERVAL_2021, "test", 0); - private static final AggregateProjectionMetadata.Schema PROJECTION_CHANNEL_ADDED_HOURLY = new AggregateProjectionMetadata.Schema( - "name1-does-not-matter", - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, - null, - VirtualColumns.create(Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - )), - ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel"), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("channel_sum", "channel") - }, - ImmutableList.of( - OrderBy.ascending(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - OrderBy.ascending("channel") - ) - ); - private static final AggregateProjectionMetadata.Schema PROJECTION_CHANNEL_ADDED_DAILY = new AggregateProjectionMetadata.Schema( - "name2-does-not-matter", - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, - null, - VirtualColumns.create(Granularities.toVirtualColumn( - Granularities.DAY, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - )), - ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel"), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("channel_sum", "channel") - }, - ImmutableList.of( - OrderBy.ascending(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - OrderBy.ascending("channel") - ) - ); + private static final AggregateProjectionMetadata.Schema PROJECTION_CHANNEL_ADDED_HOURLY = + AggregateProjectionMetadata.schemaBuilder("name1-does-not-matter") + .timeColumnName(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupAndOrder(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel") + .aggregators(new LongSumAggregatorFactory("channel_sum", "channel")) + .build(); + + private static final AggregateProjectionMetadata.Schema PROJECTION_CHANNEL_ADDED_DAILY = + AggregateProjectionMetadata.schemaBuilder("name2-does-not-matter") + .timeColumnName(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.DAY, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupAndOrder(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel") + .aggregators(new LongSumAggregatorFactory("channel_sum", "channel")) + .build(); @Test public void testCacheStrategy() throws Exception diff --git a/processing/src/test/java/org/apache/druid/segment/CursorFactoryProjectionTest.java b/processing/src/test/java/org/apache/druid/segment/CursorFactoryProjectionTest.java index bba15fa8f064..555afa1ae813 100644 --- a/processing/src/test/java/org/apache/druid/segment/CursorFactoryProjectionTest.java +++ b/processing/src/test/java/org/apache/druid/segment/CursorFactoryProjectionTest.java @@ -91,7 +91,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -170,221 +169,159 @@ public static List makeRows(List dimensions) static final List ROLLUP_ROWS = makeRows(ImmutableList.of("a", "b")); private static final List PROJECTIONS = Arrays.asList( - new AggregateProjectionSpec( - "ab_hourly_cd_sum", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList( - new StringDimensionSchema("a"), - new StringDimensionSchema("b"), - new LongDimensionSchema("__gran") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("_c_sum", "c"), - new DoubleSumAggregatorFactory("d", "d") - } - ), - new AggregateProjectionSpec( - "a_hourly_c_sum_with_count_latest", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList( - new LongDimensionSchema("__gran"), - new StringDimensionSchema("a") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("chocula"), - new LongSumAggregatorFactory("_c_sum", "c"), - new LongLastAggregatorFactory("_c_last", "c", null) - } - ), - new AggregateProjectionSpec( - "b_hourly_c_sum_non_time_ordered", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList( - new StringDimensionSchema("b"), - new LongDimensionSchema("__gran") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("chocula"), - new LongSumAggregatorFactory("_c_sum", "c"), - new LongLastAggregatorFactory("_c_last", "c", null) - } - ), - new AggregateProjectionSpec( - "bf_daily_c_sum", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.DAY, "__gran") - ), - Arrays.asList( - new LongDimensionSchema("__gran"), - new StringDimensionSchema("b"), - new FloatDimensionSchema("e") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("_c_sum", "c") - } - ), - new AggregateProjectionSpec( - "b_c_sum", - null, - VirtualColumns.EMPTY, - List.of(new StringDimensionSchema("b")), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("_c_sum", "c") - } - ), - new AggregateProjectionSpec( - "ab", - null, - null, - Arrays.asList( - new StringDimensionSchema("a"), - new StringDimensionSchema("b") - ), - null - ), - new AggregateProjectionSpec( - "abfoo", - null, - VirtualColumns.create( - new ExpressionVirtualColumn( - "bfoo", - "concat(b, 'foo')", - ColumnType.STRING, - TestExprMacroTable.INSTANCE - ) - ), - Arrays.asList( - new StringDimensionSchema("a"), - new StringDimensionSchema("bfoo") - ), - null - ), - new AggregateProjectionSpec( - "c_sum_daily", - null, - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.DAY, "__gran")), - Collections.singletonList(new LongDimensionSchema("__gran")), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("_c_sum", "c") - } - ), - new AggregateProjectionSpec( - "c_sum", - null, - VirtualColumns.EMPTY, - Collections.emptyList(), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("_c_sum", "c") - } - ), - new AggregateProjectionSpec( - "missing_column", - null, - VirtualColumns.EMPTY, - List.of(new StringDimensionSchema("missing")), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("csum", "c") - } - ), - new AggregateProjectionSpec( - "json", - null, - VirtualColumns.EMPTY, - List.of(new AutoTypeColumnSchema("f", null)), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("_c_sum", "c") - } - ), - new AggregateProjectionSpec( - "ab_filter_baaonly_hourly_cd_sum", - new EqualityFilter("b", ColumnType.STRING, "aa", null), - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList( - new StringDimensionSchema("a"), - new StringDimensionSchema("b"), - new LongDimensionSchema("__gran") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("_c_sum", "c"), - new DoubleSumAggregatorFactory("d", "d") - } - ) + AggregateProjectionSpec.builder("ab_hourly_cd_sum") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupingColumns( + new StringDimensionSchema("a"), + new StringDimensionSchema("b"), + new LongDimensionSchema("__gran") + ) + .aggregators( + new LongSumAggregatorFactory("_c_sum", "c"), + new DoubleSumAggregatorFactory("d", "d") + ) + .build(), + AggregateProjectionSpec.builder("a_hourly_c_sum_with_count_latest") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupingColumns( + new LongDimensionSchema("__gran"), + new StringDimensionSchema("a") + ) + .aggregators( + new CountAggregatorFactory("chocula"), + new LongSumAggregatorFactory("_c_sum", "c"), + new LongLastAggregatorFactory("_c_last", "c", null) + ) + .build(), + AggregateProjectionSpec.builder("b_hourly_c_sum_non_time_ordered") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupingColumns( + new StringDimensionSchema("b"), + new LongDimensionSchema("__gran") + ) + .aggregators( + new CountAggregatorFactory("chocula"), + new LongSumAggregatorFactory("_c_sum", "c"), + new LongLastAggregatorFactory("_c_last", "c", null) + ) + .build(), + AggregateProjectionSpec.builder("bf_daily_c_sum") + .virtualColumns(Granularities.toVirtualColumn(Granularities.DAY, "__gran")) + .groupingColumns( + new LongDimensionSchema("__gran"), + new StringDimensionSchema("b"), + new FloatDimensionSchema("e") + ) + .aggregators(new LongSumAggregatorFactory("_c_sum", "c")) + .build(), + AggregateProjectionSpec.builder("b_c_sum") + .groupingColumns(new StringDimensionSchema("b")) + .aggregators(new LongSumAggregatorFactory("_c_sum", "c")) + .build(), + AggregateProjectionSpec.builder("ab") + .groupingColumns( + new StringDimensionSchema("a"), + new StringDimensionSchema("b") + ) + .build(), + AggregateProjectionSpec.builder("abfoo") + .virtualColumns( + new ExpressionVirtualColumn( + "bfoo", + "concat(b, 'foo')", + ColumnType.STRING, + TestExprMacroTable.INSTANCE + ) + ) + .groupingColumns( + new StringDimensionSchema("a"), + new StringDimensionSchema("bfoo") + ) + .build(), + AggregateProjectionSpec.builder("c_sum_daily") + .virtualColumns(Granularities.toVirtualColumn(Granularities.DAY, "__gran")) + .groupingColumns(new LongDimensionSchema("__gran")) + .aggregators(new LongSumAggregatorFactory("_c_sum", "c")) + .build(), + AggregateProjectionSpec.builder("c_sum") + .aggregators(new LongSumAggregatorFactory("_c_sum", "c")) + .build(), + AggregateProjectionSpec.builder("missing_column") + .groupingColumns(new StringDimensionSchema("missing")) + .aggregators(new LongSumAggregatorFactory("csum", "c")) + .build(), + AggregateProjectionSpec.builder("json") + .groupingColumns(new AutoTypeColumnSchema("f", null)) + .aggregators(new LongSumAggregatorFactory("_c_sum", "c")) + .build(), + AggregateProjectionSpec.builder("a_filter_b_aaonly_hourly_cd_sum") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .filter(new EqualityFilter("b", ColumnType.STRING, "aa", null)) + .groupingColumns( + new StringDimensionSchema("a"), + new LongDimensionSchema("__gran") + ) + .aggregators( + new LongSumAggregatorFactory("_c_sum", "c"), + new DoubleSumAggregatorFactory("d", "d") + ) + .build() ); private static final List ROLLUP_PROJECTIONS = Arrays.asList( - new AggregateProjectionSpec( - "a_hourly_c_sum_with_count", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList( - new LongDimensionSchema("__gran"), - new StringDimensionSchema("a") - ), - new AggregatorFactory[]{ + AggregateProjectionSpec.builder("a_hourly_c_sum_with_count") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupingColumns( + new LongDimensionSchema("__gran"), + new StringDimensionSchema("a") + ) + .aggregators( new CountAggregatorFactory("chocula"), new LongSumAggregatorFactory("sum_c", "sum_c") - } - ), - new AggregateProjectionSpec( - "afoo", - null, - VirtualColumns.create( - new ExpressionVirtualColumn( - "afoo", - "concat(a, 'foo')", - ColumnType.STRING, - TestExprMacroTable.INSTANCE - ) - ), - List.of( - new StringDimensionSchema("afoo") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_c", "sum_c") - } - ) + ) + .build(), + AggregateProjectionSpec.builder("afoo") + .virtualColumns( + new ExpressionVirtualColumn( + "afoo", + "concat(a, 'foo')", + ColumnType.STRING, + TestExprMacroTable.INSTANCE + ) + ) + .groupingColumns(new StringDimensionSchema("afoo")) + .aggregators(new LongSumAggregatorFactory("sum_c", "sum_c")) + .build() ); private static final List AUTO_PROJECTIONS = PROJECTIONS.stream() - .map(projection -> new AggregateProjectionSpec( - projection.getName(), - projection.getFilter(), - projection.getVirtualColumns(), - projection.getGroupingColumns() - .stream() - .map(x -> new AutoTypeColumnSchema(x.getName(), null)) - .collect(Collectors.toList()), - projection.getAggregators() - )) + .map( + projection -> + AggregateProjectionSpec.builder(projection) + .groupingColumns( + projection.getGroupingColumns() + .stream() + .map(x -> new AutoTypeColumnSchema(x.getName(), null)) + .collect(Collectors.toList()) + ) + .build() + ) .collect(Collectors.toList()); private static final List AUTO_ROLLUP_PROJECTIONS = ROLLUP_PROJECTIONS.stream() - .map(projection -> new AggregateProjectionSpec( - projection.getName(), - projection.getFilter(), - projection.getVirtualColumns(), - projection.getGroupingColumns() - .stream() - .map(x -> new AutoTypeColumnSchema(x.getName(), null)) - .collect(Collectors.toList()), - projection.getAggregators() - )) + .map( + projection -> + AggregateProjectionSpec.builder(projection) + .groupingColumns( + projection.getGroupingColumns() + .stream() + .map(x -> new AutoTypeColumnSchema(x.getName(), null)) + .collect(Collectors.toList()) + ) + .build() + ) .collect(Collectors.toList()); @Parameterized.Parameters(name = "name: {0}, segmentTimeOrdered: {5}, autoSchema: {6}") diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java index 19439bfcb29d..b6a1dfa03909 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java @@ -3011,31 +3011,18 @@ public void testMergeProjections() throws IOException ); List projections = Arrays.asList( - new AggregateProjectionSpec( - "a_hourly_c_sum", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList( - new StringDimensionSchema("a"), - new LongDimensionSchema("__gran") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("c_sum", "c") - } - ), - new AggregateProjectionSpec( - "a_c_sum", - null, - VirtualColumns.EMPTY, - Collections.singletonList( - new StringDimensionSchema("a") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("c_sum", "c") - } - ) + AggregateProjectionSpec.builder("a_hourly_c_sum") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupingColumns( + new StringDimensionSchema("a"), + new LongDimensionSchema("__gran") + ) + .aggregators(new LongSumAggregatorFactory("c_sum", "c")) + .build(), + AggregateProjectionSpec.builder("a_c_sum") + .groupingColumns(new StringDimensionSchema("a")) + .aggregators(new LongSumAggregatorFactory("c_sum", "c")) + .build() ); IndexBuilder bob = IndexBuilder.create() diff --git a/processing/src/test/java/org/apache/druid/segment/MetadataTest.java b/processing/src/test/java/org/apache/druid/segment/MetadataTest.java index b5cc7621e764..b889d58eca3d 100644 --- a/processing/src/test/java/org/apache/druid/segment/MetadataTest.java +++ b/processing/src/test/java/org/apache/druid/segment/MetadataTest.java @@ -99,21 +99,15 @@ public void testMerge() AggregatorFactory[] aggs = new AggregatorFactory[]{ new LongMaxAggregatorFactory("n", "f") }; - List projectionSpecs = ImmutableList.of( + List projectionSpecs = List.of( new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "some_projection", - "__gran", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList("a", "b", "__gran"), - new AggregatorFactory[]{ - new LongLastAggregatorFactory("atLongLast", "d", null) - }, - makeOrderBy("a", "b", "__gran") - ), + AggregateProjectionMetadata.schemaBuilder("some_projection") + .timeColumnName("__gran") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupAndOrder("a", "b", "__gran") + .aggregators(new LongLastAggregatorFactory("atLongLast", "d", null)) + .ordering("a", "b", "__gran") + .build(), 1234 ) ); @@ -280,73 +274,49 @@ public void testMergeProjectionsUnexpectedMismatch() { List p1 = ImmutableList.of( new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "some_projection", - "__gran", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList("a", "b", "__gran"), - new AggregatorFactory[]{ - new LongLastAggregatorFactory("atLongLast", "d", null) - }, - makeOrderBy("a", "b", "__gran") - ), + AggregateProjectionMetadata.schemaBuilder("some_projection") + .timeColumnName("__gran") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupAndOrder("a", "b", "__gran") + .aggregators(new LongLastAggregatorFactory("atLongLast", "d", null)) + .ordering("a", "b", "__gran") + .build(), 654321 ) ); List p2 = ImmutableList.of( new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "some_projection", - "__gran", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList("a", "b", "_gran"), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("longSum", "d") - }, - makeOrderBy("a", "b", "__gran") - ), + AggregateProjectionMetadata.schemaBuilder("some_projection") + .timeColumnName("__gran") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupAndOrder("a", "b", "_gran") + .aggregators(new LongSumAggregatorFactory("longSum", "d")) + .ordering("a", "b", "_gran") + .build(), 1234 ) ); List p3 = ImmutableList.of( new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "some_projection", - "__gran", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList("a", "b", "__gran"), - new AggregatorFactory[]{ - new LongLastAggregatorFactory("atLongLast", "d", null) - }, - makeOrderBy("a", "b", "__gran") - ), + AggregateProjectionMetadata.schemaBuilder("some_projection") + .timeColumnName("__gran") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupAndOrder("a", "b", "__gran") + .aggregators(new LongLastAggregatorFactory("atLongLast", "d", null)) + .ordering("a", "b", "__gran") + .build(), 12121 ), new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "some_projection2", - "__gran", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.DAY, "__gran") - ), - Arrays.asList("__gran", "a"), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("longSum", "d") - }, - makeOrderBy("__gran", "a") - ), + AggregateProjectionMetadata.schemaBuilder("some_projection2") + .timeColumnName("__gran") + .virtualColumns(Granularities.toVirtualColumn(Granularities.DAY, "__gran")) + .groupAndOrder("__gran", "a") + .aggregators(new LongSumAggregatorFactory("longSum", "d")) + .ordering("__gran", "a") + .build(), 555 ) ); diff --git a/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorHolderTest.java b/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorHolderTest.java index 8ac1a26bb4f8..2df7bb0a2b05 100644 --- a/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorHolderTest.java +++ b/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorHolderTest.java @@ -35,7 +35,6 @@ import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.query.QueryContext; import org.apache.druid.query.QueryContexts; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; @@ -173,41 +172,39 @@ public void testProjectionTimeBoundaryInspector() Arrays.asList("a", "bb", 1L, 1.1, 1.1f) ) ); - IndexBuilder bob = IndexBuilder.create() - .tmpDir(tmp) - .schema( - IncrementalIndexSchema.builder() - .withDimensionsSpec(dims) - .withRollup(false) - .withMinTimestamp(startTime.getMillis()) - .withProjections( - Collections.singletonList( - new AggregateProjectionSpec( - "ab_hourly_cd_sum_time_ordered", - null, - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - "__gran" - ) - ), - Arrays.asList( - new LongDimensionSchema("__gran"), - new StringDimensionSchema("a"), - new StringDimensionSchema("b") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory( - "_c_sum", - "c" - ), - new DoubleSumAggregatorFactory("d", "d") - } - ) + IncrementalIndexSchema indexSchema = + IncrementalIndexSchema.builder() + .withDimensionsSpec(dims) + .withRollup(false) + .withMinTimestamp(startTime.getMillis()) + .withProjections( + Collections.singletonList( + AggregateProjectionSpec.builder("ab_hourly_cd_sum_time_ordered") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + "__gran" ) ) + .groupingColumns( + new LongDimensionSchema("__gran"), + new StringDimensionSchema("a"), + new StringDimensionSchema("b") + ) + .aggregators( + new LongSumAggregatorFactory( + "_c_sum", + "c" + ), + new DoubleSumAggregatorFactory("d", "d") + ) .build() - ) + ) + ) + .build(); + IndexBuilder bob = IndexBuilder.create() + .tmpDir(tmp) + .schema(indexSchema) .rows(rows); try (QueryableIndex index = bob.buildMMappedIndex()) { diff --git a/processing/src/test/java/org/apache/druid/segment/TestIndex.java b/processing/src/test/java/org/apache/druid/segment/TestIndex.java index c5bc6c7041af..3e84fc0a5b9d 100644 --- a/processing/src/test/java/org/apache/druid/segment/TestIndex.java +++ b/processing/src/test/java/org/apache/druid/segment/TestIndex.java @@ -185,25 +185,24 @@ public class TestIndex new DoubleMaxAggregatorFactory(DOUBLE_METRICS[2], VIRTUAL_COLUMNS.getVirtualColumns()[0].getOutputName()), new HyperUniquesAggregatorFactory("quality_uniques", "quality") }; - public static final ImmutableList PROJECTIONS = ImmutableList.of( - new AggregateProjectionSpec( - "daily_market_maxQuality", - null, - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.DAY, "__gran")), - List.of(new LongDimensionSchema("__gran"), new StringDimensionSchema("market")), - new AggregatorFactory[]{new LongMaxAggregatorFactory("maxQuality", "qualityLong")} - ), - new AggregateProjectionSpec( - "daily_countAndQualityCardinalityAndMaxLongNullable", - null, - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.DAY, "__gran")), - List.of(new LongDimensionSchema("__gran")), - new AggregatorFactory[]{ - new CountAggregatorFactory("count"), - QUALITY_CARDINALITY, - new LongMaxAggregatorFactory("longNullableMax", "longNumericNull") - } - ) + public static final List PROJECTIONS = List.of( + AggregateProjectionSpec.builder("daily_market_maxQuality") + .virtualColumns(Granularities.toVirtualColumn(Granularities.DAY, "__gran")) + .groupingColumns( + new LongDimensionSchema("__gran"), + new StringDimensionSchema("market") + ) + .aggregators(new LongMaxAggregatorFactory("maxQuality", "qualityLong")) + .build(), + AggregateProjectionSpec.builder("daily_countAndQualityCardinalityAndMaxLongNullable") + .virtualColumns(Granularities.toVirtualColumn(Granularities.DAY, "__gran")) + .groupingColumns(new LongDimensionSchema("__gran")) + .aggregators( + new CountAggregatorFactory("count"), + QUALITY_CARDINALITY, + new LongMaxAggregatorFactory("longNullableMax", "longNumericNull") + ) + .build() ); public static final IndexSpec INDEX_SPEC = IndexSpec.DEFAULT; diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/OnheapIncrementalIndexTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/OnheapIncrementalIndexTest.java index 02d93e0afed3..fb4881c8517c 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/OnheapIncrementalIndexTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/OnheapIncrementalIndexTest.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableList; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.AggregateProjectionSpec; @@ -38,7 +37,6 @@ import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.TestHelper; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.joda.time.DateTime; @@ -63,31 +61,28 @@ public void testSpecSerde() throws JsonProcessingException @Test public void testProjectionHappyPath() { - // arrange DimensionsSpec dimensionsSpec = DimensionsSpec.builder() - .setDimensions(ImmutableList.of( + .setDimensions(List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") )) .build(); AggregatorFactory aggregatorFactory = new DoubleSumAggregatorFactory("double", "double"); - AggregateProjectionSpec projectionSpec = new AggregateProjectionSpec( - "proj", - null, - VirtualColumns.EMPTY, - ImmutableList.of(new StringDimensionSchema("string")), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long"), - new DoubleSumAggregatorFactory("double", "double") - } - ); - // act & assert + AggregateProjectionSpec projectionSpec = + AggregateProjectionSpec.builder("proj") + .groupingColumns(new StringDimensionSchema("string")) + .aggregators( + new LongSumAggregatorFactory("sum_long", "long"), + new DoubleSumAggregatorFactory("double", "double") + ) + .build(); + IncrementalIndex index = IndexBuilder.create() .schema(IncrementalIndexSchema.builder() .withDimensionsSpec(dimensionsSpec) .withRollup(true) .withMetrics(aggregatorFactory) - .withProjections(ImmutableList.of(projectionSpec)) + .withProjections(List.of(projectionSpec)) .build()) .buildIncrementalIndex(); Assert.assertNotNull(index.getProjection("proj")); @@ -99,20 +94,7 @@ public void testProjectionDuplicatedName() // arrange DimensionsSpec dimensionsSpec = DimensionsSpec.EMPTY; AggregatorFactory aggregatorFactory = new DoubleSumAggregatorFactory("double", "double"); - AggregateProjectionSpec projectionSpec1 = new AggregateProjectionSpec( - "proj", - null, - VirtualColumns.EMPTY, - ImmutableList.of(), - new AggregatorFactory[]{new DoubleSumAggregatorFactory("double", "double")} - ); - AggregateProjectionSpec projectionSpec2 = new AggregateProjectionSpec( - "proj", - null, - VirtualColumns.EMPTY, - ImmutableList.of(), - new AggregatorFactory[]{new DoubleSumAggregatorFactory("double", "double")} - ); + AggregateProjectionSpec.Builder bob = new AggregateProjectionSpec.Builder().aggregators(aggregatorFactory); // act & assert DruidException e = Assert.assertThrows( DruidException.class, @@ -121,10 +103,12 @@ public void testProjectionDuplicatedName() .withDimensionsSpec(dimensionsSpec) .withRollup(true) .withMetrics(aggregatorFactory) - .withProjections(ImmutableList.of( - projectionSpec1, - projectionSpec2 - )) + .withProjections( + List.of( + bob.name("proj").build(), + bob.name("proj").build() + ) + ) .build()) .buildIncrementalIndex() ); @@ -152,7 +136,7 @@ public void testBadProjectionMismatchedDimensionTypes() .withDimensionsSpec( DimensionsSpec.builder() .setDimensions( - ImmutableList.of( + List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") ) @@ -160,16 +144,10 @@ public void testBadProjectionMismatchedDimensionTypes() .build() ) .withProjections( - ImmutableList.of( - new AggregateProjectionSpec( - "mismatched dims", - null, - VirtualColumns.EMPTY, - ImmutableList.of( - new LongDimensionSchema("string") - ), - null - ) + List.of( + AggregateProjectionSpec.builder("mismatched dims") + .groupingColumns(new LongDimensionSchema("string")) + .build() ) ) .build() @@ -193,7 +171,7 @@ public void testBadProjectionDimensionNoVirtualColumnOrBaseTable() .withDimensionsSpec( DimensionsSpec.builder() .setDimensions( - ImmutableList.of( + List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") ) @@ -201,24 +179,21 @@ public void testBadProjectionDimensionNoVirtualColumnOrBaseTable() .build() ) .withProjections( - ImmutableList.of( - new AggregateProjectionSpec( - "sad grouping column", - null, - VirtualColumns.create( - new ExpressionVirtualColumn( - "v0", - "cast(long, 'double')", - ColumnType.DOUBLE, - TestExprMacroTable.INSTANCE - ) - ), - ImmutableList.of( - new DoubleDimensionSchema("v0"), - new StringDimensionSchema("missing") - ), - null - ) + List.of( + AggregateProjectionSpec.builder("sad grouping column") + .virtualColumns( + new ExpressionVirtualColumn( + "v0", + "cast(long, 'double')", + ColumnType.DOUBLE, + TestExprMacroTable.INSTANCE + ) + ) + .groupingColumns( + new DoubleDimensionSchema("v0"), + new StringDimensionSchema("missing") + ) + .build() ) ) .build() @@ -242,7 +217,7 @@ public void testBadProjectionVirtualColumnNoDimension() .withDimensionsSpec( DimensionsSpec.builder() .setDimensions( - ImmutableList.of( + List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") ) @@ -250,23 +225,20 @@ public void testBadProjectionVirtualColumnNoDimension() .build() ) .withProjections( - ImmutableList.of( - new AggregateProjectionSpec( - "sad virtual column", - null, - VirtualColumns.create( - new ExpressionVirtualColumn( - "v0", - "double", - ColumnType.DOUBLE, - TestExprMacroTable.INSTANCE - ) - ), - ImmutableList.of( - new LongDimensionSchema("long") - ), - null - ) + List.of( + AggregateProjectionSpec.builder("sad virtual column") + .virtualColumns( + new ExpressionVirtualColumn( + "v0", + "double", + ColumnType.DOUBLE, + TestExprMacroTable.INSTANCE + ) + ) + .groupingColumns( + new LongDimensionSchema("long") + ) + .build() ) ) .build() @@ -290,7 +262,7 @@ public void testBadProjectionRollupMismatchedAggType() .withDimensionsSpec( DimensionsSpec.builder() .setDimensions( - ImmutableList.of( + List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") ) @@ -302,21 +274,17 @@ public void testBadProjectionRollupMismatchedAggType() new DoubleSumAggregatorFactory("sum_double", "sum_double") ) .withProjections( - ImmutableList.of( - new AggregateProjectionSpec( - "mismatched agg", - null, - VirtualColumns.EMPTY, - ImmutableList.of( - new StringDimensionSchema("string") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory( - "sum_double", - "sum_double" - ) - } - ) + List.of( + AggregateProjectionSpec.builder("mismatched agg") + .groupingColumns(new StringDimensionSchema( + "string")) + .aggregators( + new LongSumAggregatorFactory( + "sum_double", + "sum_double" + ) + ) + .build() ) ) .build() @@ -340,7 +308,7 @@ public void testBadProjectionRollupBadAggInput() .withDimensionsSpec( DimensionsSpec.builder() .setDimensions( - ImmutableList.of( + List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") ) @@ -352,19 +320,21 @@ public void testBadProjectionRollupBadAggInput() new DoubleSumAggregatorFactory("double", "double") ) .withProjections( - ImmutableList.of( - new AggregateProjectionSpec( - "renamed agg", - null, - VirtualColumns.EMPTY, - ImmutableList.of( - new StringDimensionSchema("string") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long"), - new DoubleSumAggregatorFactory("sum_double", "double") - } - ) + List.of( + AggregateProjectionSpec.builder("renamed agg") + .groupingColumns(new StringDimensionSchema( + "string")) + .aggregators( + new LongSumAggregatorFactory( + "sum_long", + "long" + ), + new DoubleSumAggregatorFactory( + "sum_double", + "double" + ) + ) + .build() ) ) .build() @@ -388,7 +358,7 @@ public void testBadProjectionVirtualColumnAggInput() .withDimensionsSpec( DimensionsSpec.builder() .setDimensions( - ImmutableList.of( + List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") ) @@ -396,25 +366,26 @@ public void testBadProjectionVirtualColumnAggInput() .build() ) .withProjections( - ImmutableList.of( - new AggregateProjectionSpec( - "sad agg virtual column", - null, - VirtualColumns.create( - new ExpressionVirtualColumn( - "v0", - "long + 100", - ColumnType.LONG, - TestExprMacroTable.INSTANCE - ) - ), - ImmutableList.of( - new LongDimensionSchema("long") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("v0_sum", "v0") - } - ) + List.of( + AggregateProjectionSpec.builder("sad agg virtual column") + .virtualColumns( + new ExpressionVirtualColumn( + "v0", + "long + 100", + ColumnType.LONG, + TestExprMacroTable.INSTANCE + ) + ) + .groupingColumns( + new LongDimensionSchema("long") + ) + .aggregators( + new LongSumAggregatorFactory( + "v0_sum", + "v0" + ) + ) + .build() ) ) .build() @@ -432,22 +403,20 @@ public void testTimestampOutOfRange() { // arrange DimensionsSpec dimensionsSpec = DimensionsSpec.builder() - .setDimensions(ImmutableList.of( + .setDimensions(List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") )) .build(); AggregatorFactory aggregatorFactory = new DoubleSumAggregatorFactory("double", "double"); - AggregateProjectionSpec projectionSpec = new AggregateProjectionSpec( - "proj", - null, - VirtualColumns.EMPTY, - ImmutableList.of(new StringDimensionSchema("string")), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long"), - new DoubleSumAggregatorFactory("double", "double") - } - ); + AggregateProjectionSpec projectionSpec = + AggregateProjectionSpec.builder("proj") + .groupingColumns(new StringDimensionSchema("string")) + .aggregators( + new LongSumAggregatorFactory("sum_long", "long"), + new DoubleSumAggregatorFactory("double", "double") + ) + .build(); final DateTime minTimestamp = DateTimes.nowUtc(); final DateTime outOfRangeTimestamp = DateTimes.nowUtc().minusDays(1); @@ -499,18 +468,15 @@ public void testTimestampOutOfRange() t.getMessage() ); - AggregateProjectionSpec projectionSpecYear = new AggregateProjectionSpec( - "proj", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.YEAR, "g") - ), - ImmutableList.of(new StringDimensionSchema("string"), new LongDimensionSchema("g")), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long"), - new DoubleSumAggregatorFactory("double", "double") - } - ); + AggregateProjectionSpec projectionSpecYear = + AggregateProjectionSpec.builder("proj") + .virtualColumns(Granularities.toVirtualColumn(Granularities.YEAR, "g")) + .groupingColumns(new StringDimensionSchema("string"), new LongDimensionSchema("g")) + .aggregators( + new LongSumAggregatorFactory("sum_long", "long"), + new DoubleSumAggregatorFactory("double", "double") + ) + .build(); IncrementalIndex index2 = IndexBuilder.create() .schema(IncrementalIndexSchema.builder() .withDimensionsSpec(dimensionsSpec) diff --git a/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java b/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java index 76f58ca26b76..12f3b4effe21 100644 --- a/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java +++ b/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java @@ -19,49 +19,51 @@ package org.apache.druid.segment.projections; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import org.apache.druid.query.OrderBy; -import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.data.input.impl.LongDimensionSchema; +import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.filter.EqualityFilter; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.LikeDimFilter; import org.apache.druid.segment.AggregateProjectionMetadata; import org.apache.druid.segment.CursorBuildSpec; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.filter.AndFilter; import org.apache.druid.segment.filter.IsBooleanFilter; import org.apache.druid.segment.filter.OrFilter; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.Set; class ProjectionsTest { @Test void testSchemaMatchSimple() { + RowSignature baseTable = RowSignature.builder() + .addTimeColumn() + .add("a", ColumnType.LONG) + .add("b", ColumnType.STRING) + .add("c", ColumnType.LONG) + .build(); AggregateProjectionMetadata spec = new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "some_projection", - null, - null, - VirtualColumns.EMPTY, - Arrays.asList("a", "b"), - new AggregatorFactory[]{new LongSumAggregatorFactory("a_projection", "a")}, - Arrays.asList(OrderBy.ascending("a"), OrderBy.ascending("b")) - ), + AggregateProjectionSpec.builder("some_projection") + .groupingColumns(new LongDimensionSchema("a"), new StringDimensionSchema("b")) + .aggregators(new LongSumAggregatorFactory("c_sum", "c")) + .build() + .toMetadataSchema(), 12345 ); CursorBuildSpec cursorBuildSpec = CursorBuildSpec.builder() - .setPreferredOrdering(ImmutableList.of()) + .setPreferredOrdering(List.of()) .setAggregators( List.of( - new LongSumAggregatorFactory("a", "a") + new LongSumAggregatorFactory("c", "c") ) ) .build(); @@ -69,15 +71,150 @@ void testSchemaMatchSimple() Projections.ProjectionMatch projectionMatch = Projections.matchAggregateProjection( spec.getSchema(), cursorBuildSpec, - (projectionName, columnName) -> true + new RowSignatureChecker(baseTable) ); Projections.ProjectionMatch expected = new Projections.ProjectionMatch( CursorBuildSpec.builder() - .setAggregators(ImmutableList.of(new LongSumAggregatorFactory("a", "a"))) - .setPhysicalColumns(ImmutableSet.of("a_projection")) - .setPreferredOrdering(ImmutableList.of()) + .setAggregators(List.of(new LongSumAggregatorFactory("c", "c"))) + .setPhysicalColumns(Set.of("c_sum")) + .setPreferredOrdering(List.of()) .build(), - ImmutableMap.of("a", "a_projection") + Map.of("c", "c_sum") + ); + Assertions.assertEquals(expected, projectionMatch); + } + + @Test + void testSchemaMatchFilter() + { + RowSignature baseTable = RowSignature.builder() + .addTimeColumn() + .add("a", ColumnType.LONG) + .add("b", ColumnType.STRING) + .add("c", ColumnType.LONG) + .build(); + AggregateProjectionMetadata spec = new AggregateProjectionMetadata( + AggregateProjectionSpec.builder("some_projection") + .filter(new EqualityFilter("b", ColumnType.STRING, "foo", null)) + .groupingColumns(new LongDimensionSchema("a")) + .aggregators(new LongSumAggregatorFactory("c_sum", "c")) + .build() + .toMetadataSchema(), + 12345 + ); + CursorBuildSpec cursorBuildSpecNoFilter = CursorBuildSpec.builder() + .setPreferredOrdering(List.of()) + .setAggregators( + List.of( + new LongSumAggregatorFactory("c", "c") + ) + ) + .build(); + + Assertions.assertNull( + Projections.matchAggregateProjection( + spec.getSchema(), + cursorBuildSpecNoFilter, + new RowSignatureChecker(baseTable) + ) + ); + CursorBuildSpec cursorBuildSpecWithFilter = CursorBuildSpec.builder() + .setPreferredOrdering(List.of()) + .setFilter( + new EqualityFilter( + "b", + ColumnType.STRING, + "foo", + null + ) + ) + .setAggregators( + List.of( + new LongSumAggregatorFactory("c", "c") + ) + ) + .build(); + Projections.ProjectionMatch projectionMatch = Projections.matchAggregateProjection( + spec.getSchema(), + cursorBuildSpecWithFilter, + new RowSignatureChecker(baseTable) + ); + Projections.ProjectionMatch expected = new Projections.ProjectionMatch( + CursorBuildSpec.builder() + .setAggregators(List.of(new LongSumAggregatorFactory("c", "c"))) + .setPhysicalColumns(Set.of("c_sum")) + .setPreferredOrdering(List.of()) + .build(), + Map.of("c", "c_sum") + ); + Assertions.assertEquals(expected, projectionMatch); + } + + @Test + void testSchemaMatchFilterIncludedInProjection() + { + RowSignature baseTable = RowSignature.builder() + .addTimeColumn() + .add("a", ColumnType.LONG) + .add("b", ColumnType.STRING) + .add("c", ColumnType.LONG) + .build(); + AggregateProjectionMetadata spec = new AggregateProjectionMetadata( + AggregateProjectionSpec.builder("some_projection") + .filter(new LikeDimFilter("b", "foo%", null, null)) + .groupingColumns(new LongDimensionSchema("a"), new StringDimensionSchema("b")) + .aggregators(new LongSumAggregatorFactory("c_sum", "c")) + .build() + .toMetadataSchema(), + 12345 + ); + CursorBuildSpec cursorBuildSpecNoFilter = CursorBuildSpec.builder() + .setPreferredOrdering(List.of()) + .setGroupingColumns(List.of("a", "b")) + .setAggregators( + List.of( + new LongSumAggregatorFactory("c", "c") + ) + ) + .build(); + + Assertions.assertNull( + Projections.matchAggregateProjection( + spec.getSchema(), + cursorBuildSpecNoFilter, + new RowSignatureChecker(baseTable) + ) + ); + CursorBuildSpec cursorBuildSpecWithFilter = CursorBuildSpec.builder() + .setGroupingColumns(List.of("a", "b")) + .setPreferredOrdering(List.of()) + .setFilter( + new LikeDimFilter( + "b", + "foo%", + null, + null + ).toFilter() + ) + .setAggregators( + List.of( + new LongSumAggregatorFactory("c", "c") + ) + ) + .build(); + Projections.ProjectionMatch projectionMatch = Projections.matchAggregateProjection( + spec.getSchema(), + cursorBuildSpecWithFilter, + new RowSignatureChecker(baseTable) + ); + Projections.ProjectionMatch expected = new Projections.ProjectionMatch( + CursorBuildSpec.builder() + .setGroupingColumns(List.of("a", "b")) + .setAggregators(List.of(new LongSumAggregatorFactory("c", "c"))) + .setPhysicalColumns(Set.of("a", "b", "c_sum")) + .setPreferredOrdering(List.of()) + .build(), + Map.of("c", "c_sum") ); Assertions.assertEquals(expected, projectionMatch); } @@ -120,5 +257,41 @@ void testRewriteFilter() new AndFilter(List.of(yeqbar, zeq123)), Projections.rewriteFilter(xeqfoo, queryFilter) ); + + queryFilter = new AndFilter( + List.of( + new EqualityFilter("a", ColumnType.STRING, "foo", null), + new EqualityFilter("b", ColumnType.STRING, "bar", null), + new EqualityFilter("c", ColumnType.STRING, "baz", null) + ) + ); + Assertions.assertEquals( + new EqualityFilter("b", ColumnType.STRING, "bar", null), + Projections.rewriteFilter( + new AndFilter( + List.of( + new EqualityFilter("a", ColumnType.STRING, "foo", null), + new EqualityFilter("c", ColumnType.STRING, "baz", null) + ) + ), + queryFilter + ) + ); + } + + private static class RowSignatureChecker implements Projections.PhysicalColumnChecker + { + private final RowSignature rowSignature; + + private RowSignatureChecker(RowSignature rowSignature) + { + this.rowSignature = rowSignature; + } + + @Override + public boolean check(String projectionName, String columnName) + { + return rowSignature.contains(columnName); + } } -} \ No newline at end of file +} diff --git a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java index 8e2fd5dde90a..64f864a26fd8 100644 --- a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java +++ b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java @@ -52,7 +52,6 @@ import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.query.filter.SelectorDimFilter; import org.apache.druid.segment.TestHelper; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.transform.ExpressionTransform; import org.apache.druid.segment.transform.TransformSpec; import org.apache.druid.testing.InitializedNullHandlingTest; @@ -636,18 +635,16 @@ void testSerde() throws Exception public void testSerdeWithProjections() throws Exception { // serialize, then deserialize of DataSchema with projections. - AggregateProjectionSpec projectionSpec = new AggregateProjectionSpec( - "ab_count_projection", - null, - null, - Arrays.asList( - new StringDimensionSchema("a"), - new LongDimensionSchema("b") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("count") - } - ); + AggregateProjectionSpec projectionSpec = + AggregateProjectionSpec.builder("ab_count_projection") + .groupingColumns( + new StringDimensionSchema("a"), + new LongDimensionSchema("b") + ) + .aggregators( + new CountAggregatorFactory("count") + ) + .build(); DataSchema original = DataSchema.builder() .withDataSource("datasource") .withTimestamp(new TimestampSpec(null, null, null)) @@ -877,20 +874,20 @@ void testInvalidProjectionDupeNames() ) .withProjections( List.of( - new AggregateProjectionSpec( - "some projection", - null, - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "g")), - List.of(new LongDimensionSchema("g")), - new AggregatorFactory[]{new CountAggregatorFactory("count")} - ), - new AggregateProjectionSpec( - "some projection", - null, - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.MINUTE, "g")), - List.of(new LongDimensionSchema("g")), - new AggregatorFactory[]{new CountAggregatorFactory("count")} - ) + AggregateProjectionSpec.builder("some projection") + .virtualColumns( + Granularities.toVirtualColumn(Granularities.HOUR, "g") + ) + .groupingColumns(new LongDimensionSchema("g")) + .aggregators(new CountAggregatorFactory("count")) + .build(), + AggregateProjectionSpec.builder("some projection") + .virtualColumns( + Granularities.toVirtualColumn(Granularities.MINUTE, "g") + ) + .groupingColumns(new LongDimensionSchema("g")) + .aggregators(new CountAggregatorFactory("count")) + .build() ) ) .build() @@ -905,6 +902,9 @@ void testInvalidProjectionDupeNames() @Test void testInvalidProjectionGranularity() { + AggregateProjectionSpec.Builder bob = AggregateProjectionSpec.builder() + .groupingColumns(new LongDimensionSchema("g")) + .aggregators(new CountAggregatorFactory("count")); Throwable t = Assertions.assertThrows( DruidException.class, () -> DataSchema.builder() @@ -919,34 +919,19 @@ void testInvalidProjectionGranularity() ) .withProjections( List.of( - new AggregateProjectionSpec( - "ok granularity", - null, - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "g")), - List.of(new LongDimensionSchema("g")), - new AggregatorFactory[]{new CountAggregatorFactory("count")} - ), - new AggregateProjectionSpec( - "acceptable granularity", - null, - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.MINUTE, "g")), - List.of(new LongDimensionSchema("g")), - new AggregatorFactory[]{new CountAggregatorFactory("count")} - ), - new AggregateProjectionSpec( - "not having a time column is ok too", - null, - VirtualColumns.EMPTY, - null, - new AggregatorFactory[]{new CountAggregatorFactory("count")} - ), - new AggregateProjectionSpec( - "bad granularity", - null, - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.DAY, "g")), - List.of(new LongDimensionSchema("g")), - new AggregatorFactory[]{new CountAggregatorFactory("count")} - ) + bob.name("ok granularity") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "g")) + .build(), + bob.name("acceptable granularity") + .virtualColumns(Granularities.toVirtualColumn(Granularities.MINUTE, "g")) + .build(), + AggregateProjectionSpec.builder() + .name("not having a time column is ok too") + .aggregators(new CountAggregatorFactory("count")) + .build(), + bob.name("bad granularity") + .virtualColumns(Granularities.toVirtualColumn(Granularities.DAY, "g")) + .build() ) ) .build() diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java index df71bf21f4ca..a8e6977dc42c 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java @@ -33,11 +33,9 @@ import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.TestDataSource; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.server.coordinator.DataSourceCompactionConfig; import org.apache.druid.server.coordinator.InlineSchemaDataSourceCompactionConfig; @@ -336,20 +334,22 @@ public void testStatusWhenProjectionsMatch() final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, 0L); final IndexSpec currentIndexSpec = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); - final AggregateProjectionSpec projection1 = new AggregateProjectionSpec( - "foo", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) - ), - List.of( - new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - new StringDimensionSchema("a") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long") - } - ); + final AggregateProjectionSpec projection1 = + AggregateProjectionSpec.builder("foo") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + new StringDimensionSchema("a") + ) + .aggregators( + new LongSumAggregatorFactory("sum_long", "long") + ) + .build(); final CompactionState lastCompactionState = new CompactionState( currentPartitionsSpec, null, @@ -384,29 +384,26 @@ public void testStatusWhenProjectionsMismatch() final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, 0L); final IndexSpec currentIndexSpec = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); - final AggregateProjectionSpec projection1 = new AggregateProjectionSpec( - "1", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) - ), - List.of( - new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - new StringDimensionSchema("a") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long") - } - ); - final AggregateProjectionSpec projection2 = new AggregateProjectionSpec( - "2", - null, - VirtualColumns.EMPTY, - Collections.emptyList(), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long") - } - ); + final AggregateProjectionSpec projection1 = + AggregateProjectionSpec.builder("1") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + new StringDimensionSchema("a") + ) + .aggregators( + new LongSumAggregatorFactory("sum_long", "long") + ) + .build(); + final AggregateProjectionSpec projection2 = + AggregateProjectionSpec.builder("2") + .aggregators(new LongSumAggregatorFactory("sum_long", "long")) + .build(); final CompactionState lastCompactionState = new CompactionState( currentPartitionsSpec, diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java b/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java index f8eff3df4b0d..e7ed9a7d038f 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java @@ -36,10 +36,8 @@ import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.math.expr.ExprMacroTable; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.expression.TestExprMacroTable; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; import org.junit.jupiter.api.Assertions; @@ -53,19 +51,17 @@ public class CatalogDataSourceCompactionConfigTest private static final ObjectMapper MAPPER; private static final MapMetadataCatalog METADATA_CATALOG; - private static final AggregateProjectionSpec TEST_PROJECTION_SPEC_1 = new AggregateProjectionSpec( - "string_sum_long_hourly", - null, - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) - ), - ImmutableList.of( - new StringDimensionSchema("string") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long") - } - ); + private static final AggregateProjectionSpec TEST_PROJECTION_SPEC_1 = + AggregateProjectionSpec.builder("string_sum_long_hourly") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns(new StringDimensionSchema("string")) + .aggregators(new LongSumAggregatorFactory("sum_long", "long")) + .build(); static { MAPPER = new DefaultObjectMapper(); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java index 0303ebd28d33..0d0022896066 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java @@ -77,7 +77,6 @@ import org.apache.druid.rpc.indexing.NoopOverlordClient; import org.apache.druid.rpc.indexing.OverlordClient; import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.incremental.OnheapIncrementalIndex; import org.apache.druid.segment.indexing.BatchIOConfig; import org.apache.druid.segment.transform.CompactionTransformSpec; @@ -942,23 +941,17 @@ public void testCompactWithProjections() final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient); final List compactionConfigs = new ArrayList<>(); final String dataSource = DATA_SOURCE_PREFIX + 0; - final List projections = ImmutableList.of( - new AggregateProjectionSpec( - dataSource + "_projection", - null, - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of( - new StringDimensionSchema("bar") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("cnt") - } - ) + final List projections = List.of( + AggregateProjectionSpec.builder(dataSource + "_projection") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns(new StringDimensionSchema("bar")) + .aggregators(new CountAggregatorFactory("cnt")) + .build() ); compactionConfigs.add( @@ -1000,22 +993,17 @@ public void testCompactWithCatalogProjections() ); final ArgumentCaptor payloadCaptor = setUpMockClient(mockClient); - final AggregateProjectionSpec projectionSpec = new AggregateProjectionSpec( - dataSource + "_projection", - null, - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of( - new StringDimensionSchema("bar") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("cnt") - } - ); + final AggregateProjectionSpec projectionSpec = + AggregateProjectionSpec.builder(dataSource + "_projection") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns(new StringDimensionSchema("bar")) + .aggregators(new CountAggregatorFactory("cnt")) + .build(); metadataCatalog.addSpec( TableId.datasource(dataSource), TableBuilder.datasource(dataSource, "P1D") From 2efc1a2ee87fe635134f4d8766ece9917d6fd1ce Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 6 Aug 2025 23:30:09 -0700 Subject: [PATCH 7/9] cleanup --- .../druid/segment/AggregateProjectionMetadata.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java index 642112cb3a12..dc0141fa7098 100644 --- a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java +++ b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java @@ -59,7 +59,7 @@ public class AggregateProjectionMetadata { private static final Interner SCHEMA_INTERNER = Interners.newWeakInterner(); - + public static final Comparator COMPARATOR = (o1, o2) -> { int rowCompare = Integer.compare(o1.numRows, o2.numRows); if (rowCompare != 0) { @@ -369,15 +369,6 @@ public String toString() public static class SchemaBuilder { - /* - @JsonProperty("name") String name, - @JsonProperty("timeColumnName") @Nullable String timeColumnName, - @JsonProperty("filter") @Nullable DimFilter filter, - @JsonProperty("virtualColumns") @Nullable VirtualColumns virtualColumns, - @JsonProperty("groupingColumns") @Nullable List groupingColumns, - @JsonProperty("aggregators") @Nullable AggregatorFactory[] aggregators, - @JsonProperty("ordering") List ordering - */ @Nullable private String name; @Nullable From ba255d9b115bb363d3383049a9e9cbc4c1a25426 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 6 Aug 2025 23:37:27 -0700 Subject: [PATCH 8/9] cleanup --- .../apache/druid/segment/projections/Projections.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java index 0c1857702801..32a30f7f7661 100644 --- a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java +++ b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java @@ -481,11 +481,8 @@ public static ProjectionMatchBuilder matchQueryPhysicalColumn( * projection doesn't contain all the rows the query would match if not using the projection. */ @Nullable - public static Filter rewriteFilter(@Nullable Filter projectionFilter, @Nullable Filter queryFilter) + public static Filter rewriteFilter(Filter projectionFilter, Filter queryFilter) { - if (projectionFilter == null || queryFilter == null) { - return queryFilter; - } if (queryFilter.equals(projectionFilter)) { return ProjectionFilterMatch.INSTANCE; } @@ -508,6 +505,10 @@ public static Filter rewriteFilter(@Nullable Filter projectionFilter, @Nullable if (projectionFilter instanceof AndFilter) { AndFilter projectionAndFilter = (AndFilter) projectionFilter; Filter rewritten = andFilter; + // calling rewriteFilter using each child of the projection AND filter as the projection filter will remove + // the child from the query AND filter if it exists (or return null if it does not exist, since it must exist + // to be a valid rewrite). The remaining AND filter of will only contain children that were not part of the + // projection AND filter for (Filter filter : projectionAndFilter.getFilters()) { rewritten = rewriteFilter(filter, rewritten); if (rewritten != null) { From 7a78c7bad755cc47c0cbebc5431610580db85813 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 7 Aug 2025 02:34:32 -0700 Subject: [PATCH 9/9] move inner classes --- .../projections/ProjectionFilterMatch.java | 37 ++++ .../segment/projections/ProjectionMatch.java | 78 ++++++++ .../projections/ProjectionMatchBuilder.java | 151 +++++++++++++++ .../segment/projections/Projections.java | 175 +----------------- .../segment/projections/ProjectionsTest.java | 14 +- 5 files changed, 274 insertions(+), 181 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/segment/projections/ProjectionFilterMatch.java create mode 100644 processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatch.java create mode 100644 processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatchBuilder.java diff --git a/processing/src/main/java/org/apache/druid/segment/projections/ProjectionFilterMatch.java b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionFilterMatch.java new file mode 100644 index 000000000000..ab4b79624bd6 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionFilterMatch.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.projections; + +import org.apache.druid.query.filter.Filter; +import org.apache.druid.segment.filter.TrueFilter; + +/** + * Terminal marker used by {@link Projections#rewriteFilter(Filter, Filter)} indicating that a {@link Filter} can be + * dropped because it matches the filter of a projection + */ +public final class ProjectionFilterMatch extends TrueFilter +{ + static final ProjectionFilterMatch INSTANCE = new ProjectionFilterMatch(); + + private ProjectionFilterMatch() + { + // no instantiation + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatch.java b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatch.java new file mode 100644 index 000000000000..82cd75982375 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatch.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.projections; + +import org.apache.druid.segment.CursorBuildSpec; + +import java.util.Map; +import java.util.Objects; + +/** + * Transformed {@link CursorBuildSpec} to run against a projection and remapping of + */ +public final class ProjectionMatch +{ + private final CursorBuildSpec cursorBuildSpec; + private final Map remapColumns; + + public ProjectionMatch(CursorBuildSpec cursorBuildSpec, Map remapColumns) + { + this.cursorBuildSpec = cursorBuildSpec; + this.remapColumns = remapColumns; + } + + public CursorBuildSpec getCursorBuildSpec() + { + return cursorBuildSpec; + } + + public Map getRemapColumns() + { + return remapColumns; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof ProjectionMatch)) { + return false; + } + ProjectionMatch that = (ProjectionMatch) o; + return Objects.equals(cursorBuildSpec, that.cursorBuildSpec) && Objects.equals(remapColumns, that.remapColumns); + } + + @Override + public int hashCode() + { + return Objects.hash(cursorBuildSpec, remapColumns); + } + + @Override + public String toString() + { + return "ProjectionMatch{" + + "cursorBuildSpec=" + cursorBuildSpec + + ", remapColumns=" + remapColumns + + '}'; + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatchBuilder.java b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatchBuilder.java new file mode 100644 index 000000000000..8a704120ae5b --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatchBuilder.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.projections; + +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.filter.Filter; +import org.apache.druid.segment.CursorBuildSpec; +import org.apache.druid.segment.VirtualColumn; +import org.apache.druid.segment.VirtualColumns; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * State holder used by {@link Projections#matchAggregateProjection} for building a {@link ProjectionMatch}. Tracks + * which all kinds of stuff to map a projection to the contents of a query {@link CursorBuildSpec}, accumulating + * details about how to transform into a new {@link CursorBuildSpec} that will run against the projection. + */ +public final class ProjectionMatchBuilder +{ + private final Set referencedPhysicalColumns; + private final Set referencedVirtualColumns; + private final Map remapColumns; + private final List combiningFactories; + private final Set matchedQueryColumns; + @Nullable + private Filter rewriteFilter; + + public ProjectionMatchBuilder() + { + this.referencedPhysicalColumns = new HashSet<>(); + this.referencedVirtualColumns = new HashSet<>(); + this.remapColumns = new HashMap<>(); + this.combiningFactories = new ArrayList<>(); + this.matchedQueryColumns = new HashSet<>(); + } + + /** + * Map a query column name to a projection column name + */ + public ProjectionMatchBuilder remapColumn(String queryColumn, String projectionColumn) + { + remapColumns.put(queryColumn, projectionColumn); + return this; + } + + @Nullable + public String getRemapValue(String queryColumn) + { + return remapColumns.get(queryColumn); + } + + /** + * Add a projection physical column, which will later be added to {@link ProjectionMatch#getCursorBuildSpec()} if + * the projection matches + */ + public ProjectionMatchBuilder addReferencedPhysicalColumn(String column) + { + referencedPhysicalColumns.add(column); + return this; + } + + /** + * Add a query virtual column that can use projection physical columns as inputs to the match builder, which will + * later be added to {@link ProjectionMatch#getCursorBuildSpec()} if the projection matches + */ + public ProjectionMatchBuilder addReferenceedVirtualColumn(VirtualColumn virtualColumn) + { + referencedVirtualColumns.add(virtualColumn); + return this; + } + + /** + * Add a query {@link AggregatorFactory#substituteCombiningFactory(AggregatorFactory)} which can combine the inputs + * of a selector created by a projection {@link AggregatorFactory} + */ + public ProjectionMatchBuilder addPreAggregatedAggregator(AggregatorFactory aggregator) + { + combiningFactories.add(aggregator); + return this; + } + + public ProjectionMatchBuilder addMatchedQueryColumn(String queryColumn) + { + matchedQueryColumns.add(queryColumn); + return this; + } + + public ProjectionMatchBuilder addMatchedQueryColumns(Collection queryColumns) + { + matchedQueryColumns.addAll(queryColumns); + return this; + } + + public ProjectionMatchBuilder rewriteFilter(Filter rewriteFilter) + { + this.rewriteFilter = rewriteFilter; + return this; + } + + public Filter getRewriteFilter() + { + return rewriteFilter; + } + + public Map getRemapColumns() + { + return remapColumns; + } + + public Set getMatchedQueryColumns() + { + return matchedQueryColumns; + } + + public ProjectionMatch build(CursorBuildSpec queryCursorBuildSpec) + { + return new ProjectionMatch( + CursorBuildSpec.builder(queryCursorBuildSpec) + .setFilter(rewriteFilter) + .setPhysicalColumns(referencedPhysicalColumns) + .setVirtualColumns(VirtualColumns.fromIterable(referencedVirtualColumns)) + .setAggregators(combiningFactories) + .build(), + remapColumns + ); + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java index 32a30f7f7661..f1971c05e7e8 100644 --- a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java +++ b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java @@ -31,18 +31,13 @@ import org.apache.druid.segment.CursorBuildSpec; import org.apache.druid.segment.CursorHolder; import org.apache.druid.segment.VirtualColumn; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.filter.AndFilter; import org.apache.druid.segment.filter.IsBooleanFilter; -import org.apache.druid.segment.filter.TrueFilter; import org.apache.druid.utils.CollectionUtils; import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -110,8 +105,7 @@ public static QueryableProjection findMatchingProjection( * @param physicalColumnChecker Helper utility which can determine if a physical column required by * queryCursorBuildSpec is available on the projection OR does not exist on the base * table either - * @return a {@link ProjectionMatch} if the {@link CursorBuildSpec} matches the projection, which contains - * information such as which + * @return a {@link ProjectionMatch} if the {@link CursorBuildSpec} matches the projection, else null */ @Nullable public static ProjectionMatch matchAggregateProjection( @@ -563,173 +557,6 @@ public static String getProjectionSmooshV9Prefix(AggregateProjectionMetadata pro return projectionSpec.getSchema().getName() + "/"; } - public static final class ProjectionMatch - { - private final CursorBuildSpec cursorBuildSpec; - private final Map remapColumns; - - public ProjectionMatch(CursorBuildSpec cursorBuildSpec, Map remapColumns) - { - this.cursorBuildSpec = cursorBuildSpec; - this.remapColumns = remapColumns; - } - - public CursorBuildSpec getCursorBuildSpec() - { - return cursorBuildSpec; - } - - public Map getRemapColumns() - { - return remapColumns; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (!(o instanceof ProjectionMatch)) { - return false; - } - ProjectionMatch that = (ProjectionMatch) o; - return Objects.equals(cursorBuildSpec, that.cursorBuildSpec) && Objects.equals(remapColumns, that.remapColumns); - } - - @Override - public int hashCode() - { - return Objects.hash(cursorBuildSpec, remapColumns); - } - - @Override - public String toString() - { - return "ProjectionMatch{" + - "cursorBuildSpec=" + cursorBuildSpec + - ", remapColumns=" + remapColumns + - '}'; - } - } - - - public static final class ProjectionMatchBuilder - { - private final Set referencedPhysicalColumns; - private final Set referencedVirtualColumns; - private final Map remapColumns; - private final List combiningFactories; - private final Set matchedQueryColumns; - @Nullable - private Filter rewriteFilter; - - public ProjectionMatchBuilder() - { - this.referencedPhysicalColumns = new HashSet<>(); - this.referencedVirtualColumns = new HashSet<>(); - this.remapColumns = new HashMap<>(); - this.combiningFactories = new ArrayList<>(); - this.matchedQueryColumns = new HashSet<>(); - } - - /** - * Map a query column name to a projection column name - */ - public ProjectionMatchBuilder remapColumn(String queryColumn, String projectionColumn) - { - remapColumns.put(queryColumn, projectionColumn); - return this; - } - - @Nullable - public String getRemapValue(String queryColumn) - { - return remapColumns.get(queryColumn); - } - - /** - * Add a projection physical column, which will later be added to {@link ProjectionMatch#getCursorBuildSpec()} if - * the projection matches - */ - public ProjectionMatchBuilder addReferencedPhysicalColumn(String column) - { - referencedPhysicalColumns.add(column); - return this; - } - - /** - * Add a query virtual column that can use projection physical columns as inputs to the match builder, which will - * later be added to {@link ProjectionMatch#getCursorBuildSpec()} if the projection matches - */ - public ProjectionMatchBuilder addReferenceedVirtualColumn(VirtualColumn virtualColumn) - { - referencedVirtualColumns.add(virtualColumn); - return this; - } - - /** - * Add a query {@link AggregatorFactory#substituteCombiningFactory(AggregatorFactory)} which can combine the inputs - * of a selector created by a projection {@link AggregatorFactory} - */ - public ProjectionMatchBuilder addPreAggregatedAggregator(AggregatorFactory aggregator) - { - combiningFactories.add(aggregator); - return this; - } - - public ProjectionMatchBuilder addMatchedQueryColumn(String queryColumn) - { - matchedQueryColumns.add(queryColumn); - return this; - } - - public ProjectionMatchBuilder addMatchedQueryColumns(Collection queryColumns) - { - matchedQueryColumns.addAll(queryColumns); - return this; - } - - public ProjectionMatchBuilder rewriteFilter(Filter rewriteFilter) - { - this.rewriteFilter = rewriteFilter; - return this; - } - - public Filter getRewriteFilter() - { - return rewriteFilter; - } - - public Map getRemapColumns() - { - return remapColumns; - } - - public Set getMatchedQueryColumns() - { - return matchedQueryColumns; - } - - public ProjectionMatch build(CursorBuildSpec queryCursorBuildSpec) - { - return new ProjectionMatch( - CursorBuildSpec.builder(queryCursorBuildSpec) - .setFilter(rewriteFilter) - .setPhysicalColumns(referencedPhysicalColumns) - .setVirtualColumns(VirtualColumns.fromIterable(referencedVirtualColumns)) - .setAggregators(combiningFactories) - .build(), - remapColumns - ); - } - } - - public static final class ProjectionFilterMatch extends TrueFilter - { - private static final ProjectionFilterMatch INSTANCE = new ProjectionFilterMatch(); - } - /** * Returns true if column is defined in {@link AggregateProjectionSpec#getGroupingColumns()} OR if the column does not * exist in the base table. Part of determining if a projection can be used for a given {@link CursorBuildSpec}, diff --git a/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java b/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java index 12f3b4effe21..3c13a0571bf2 100644 --- a/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java +++ b/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java @@ -68,12 +68,12 @@ void testSchemaMatchSimple() ) .build(); - Projections.ProjectionMatch projectionMatch = Projections.matchAggregateProjection( + ProjectionMatch projectionMatch = Projections.matchAggregateProjection( spec.getSchema(), cursorBuildSpec, new RowSignatureChecker(baseTable) ); - Projections.ProjectionMatch expected = new Projections.ProjectionMatch( + ProjectionMatch expected = new ProjectionMatch( CursorBuildSpec.builder() .setAggregators(List.of(new LongSumAggregatorFactory("c", "c"))) .setPhysicalColumns(Set.of("c_sum")) @@ -134,12 +134,12 @@ void testSchemaMatchFilter() ) ) .build(); - Projections.ProjectionMatch projectionMatch = Projections.matchAggregateProjection( + ProjectionMatch projectionMatch = Projections.matchAggregateProjection( spec.getSchema(), cursorBuildSpecWithFilter, new RowSignatureChecker(baseTable) ); - Projections.ProjectionMatch expected = new Projections.ProjectionMatch( + ProjectionMatch expected = new ProjectionMatch( CursorBuildSpec.builder() .setAggregators(List.of(new LongSumAggregatorFactory("c", "c"))) .setPhysicalColumns(Set.of("c_sum")) @@ -202,12 +202,12 @@ void testSchemaMatchFilterIncludedInProjection() ) ) .build(); - Projections.ProjectionMatch projectionMatch = Projections.matchAggregateProjection( + ProjectionMatch projectionMatch = Projections.matchAggregateProjection( spec.getSchema(), cursorBuildSpecWithFilter, new RowSignatureChecker(baseTable) ); - Projections.ProjectionMatch expected = new Projections.ProjectionMatch( + ProjectionMatch expected = new ProjectionMatch( CursorBuildSpec.builder() .setGroupingColumns(List.of("a", "b")) .setAggregators(List.of(new LongSumAggregatorFactory("c", "c"))) @@ -230,7 +230,7 @@ void testRewriteFilter() Filter queryFilter = xeqfoo2; Assertions.assertInstanceOf( - Projections.ProjectionFilterMatch.class, + ProjectionFilterMatch.class, Projections.rewriteFilter(xeqfoo, queryFilter) );