diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java index 7b6b48802f34..9084c9fbaa17 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java @@ -35,7 +35,6 @@ import org.apache.druid.query.aggregation.datasketches.theta.SketchMergeAggregatorFactory; import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.segment.AutoTypeColumnSchema; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.generator.GeneratorBasicSchemas; import org.apache.druid.segment.generator.GeneratorSchemaInfo; import org.apache.druid.segment.transform.ExpressionTransform; @@ -162,34 +161,47 @@ public class SqlBenchmarkDatasets makeDimensionsSpec(expressionsSchema), expressionsSchema.getAggsArray(), Arrays.asList( - new AggregateProjectionSpec( - "string2_hourly_sums_hll", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList( - new StringDimensionSchema("string2"), - new LongDimensionSchema("__gran") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("long4_sum", "long4"), - new DoubleSumAggregatorFactory("double2_sum", "double2"), - new HllSketchBuildAggregatorFactory("hll_string5", "string5", null, null, null, false, true) - } - ), - new AggregateProjectionSpec( - "string2_long2_sums", - VirtualColumns.EMPTY, - Arrays.asList( - new StringDimensionSchema("string2"), - new LongDimensionSchema("long2") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("long4_sum", "long4"), - new DoubleSumAggregatorFactory("double2_sum", "double2"), - new HllSketchBuildAggregatorFactory("hll_string5", "string5", null, null, null, false, true) - } - ) + AggregateProjectionSpec.builder("string2_hourly_sums_hll") + .virtualColumns( + Granularities.toVirtualColumn(Granularities.HOUR, "__gran") + ) + .groupingColumns( + new StringDimensionSchema("string2"), + new LongDimensionSchema("__gran") + ) + .aggregators( + new LongSumAggregatorFactory("long4_sum", "long4"), + new DoubleSumAggregatorFactory("double2_sum", "double2"), + new HllSketchBuildAggregatorFactory( + "hll_string5", + "string5", + null, + null, + null, + false, + true + ) + ) + .build(), + AggregateProjectionSpec.builder("string2_long2_sums") + .groupingColumns( + new StringDimensionSchema("string2"), + new LongDimensionSchema("long2") + ) + .aggregators( + new LongSumAggregatorFactory("long4_sum", "long4"), + new DoubleSumAggregatorFactory("double2_sum", "double2"), + new HllSketchBuildAggregatorFactory( + "hll_string5", + "string5", + null, + null, + null, + false, + true + ) + ) + .build() ), Granularities.NONE ) @@ -407,15 +419,17 @@ public BenchmarkSchema asAutoDimensions() ), aggregators, projections.stream() - .map(projection -> new AggregateProjectionSpec( - projection.getName(), - projection.getVirtualColumns(), - projection.getGroupingColumns() - .stream() - .map(dim -> new AutoTypeColumnSchema(dim.getName(), null)) - .collect(Collectors.toList()), - projection.getAggregators() - )).collect(Collectors.toList()), + .map( + projection -> + AggregateProjectionSpec.builder(projection) + .groupingColumns( + projection.getGroupingColumns() + .stream() + .map(dim -> new AutoTypeColumnSchema(dim.getName(), null)) + .collect(Collectors.toList()) + ) + .build() + ).collect(Collectors.toList()), queryGranularity ); } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/segment/DatasketchesProjectionTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/segment/DatasketchesProjectionTest.java index e0548babab3e..6073b0623032 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/segment/DatasketchesProjectionTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/segment/DatasketchesProjectionTest.java @@ -47,7 +47,6 @@ import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.query.DruidProcessingConfig; import org.apache.druid.query.QueryContexts; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.datasketches.hll.HllSketchBuildAggregatorFactory; import org.apache.druid.query.aggregation.datasketches.hll.HllSketchHolder; import org.apache.druid.query.aggregation.datasketches.hll.HllSketchModule; @@ -98,36 +97,43 @@ public class DatasketchesProjectionTest extends InitializedNullHandlingTest private static final Closer CLOSER = Closer.create(); private static final List PROJECTIONS = Collections.singletonList( - new AggregateProjectionSpec( - "a_projection", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList( - new LongDimensionSchema("__gran"), - new StringDimensionSchema("a") - ), - new AggregatorFactory[]{ - new HllSketchBuildAggregatorFactory("_b_hll", "b", null, null, null, null, false), - new SketchMergeAggregatorFactory("_b_theta", "b", null, null, false, null), - new DoublesSketchAggregatorFactory("_d_doubles", "d", null), - new ArrayOfDoublesSketchAggregatorFactory("_bcd_aod", "b", null, Arrays.asList("c", "d"), null), - new KllDoublesSketchAggregatorFactory("_d_kll", "d", null, null) - } - ) + AggregateProjectionSpec.builder("a_projection") + .virtualColumns( + Granularities.toVirtualColumn(Granularities.HOUR, "__gran") + ) + .groupingColumns( + new LongDimensionSchema("__gran"), + new StringDimensionSchema("a") + ) + .aggregators( + new HllSketchBuildAggregatorFactory("_b_hll", "b", null, null, null, null, false), + new SketchMergeAggregatorFactory("_b_theta", "b", null, null, false, null), + new DoublesSketchAggregatorFactory("_d_doubles", "d", null), + new ArrayOfDoublesSketchAggregatorFactory( + "_bcd_aod", + "b", + null, + Arrays.asList("c", "d"), + null + ), + new KllDoublesSketchAggregatorFactory("_d_kll", "d", null, null) + ) + .build() ); - private static final List AUTO_PROJECTIONS = PROJECTIONS.stream().map(projection -> { - return new AggregateProjectionSpec( - projection.getName(), - projection.getVirtualColumns(), - projection.getGroupingColumns() - .stream() - .map(x -> new AutoTypeColumnSchema(x.getName(), null)) - .collect(Collectors.toList()), - projection.getAggregators() - ); - }).collect(Collectors.toList()); + private static final List AUTO_PROJECTIONS = + PROJECTIONS.stream() + .map( + projection -> + AggregateProjectionSpec.builder(projection) + .groupingColumns( + projection.getGroupingColumns() + .stream() + .map(x -> new AutoTypeColumnSchema(x.getName(), null)) + .collect(Collectors.toList()) + ) + .build() + ).collect(Collectors.toList()); @Parameterized.Parameters(name = "name: {0}, sortByDim: {3}, autoSchema: {4}") public static Collection constructorFeeder() diff --git a/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java b/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java index 0967cde0bf41..8947a161558e 100644 --- a/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java +++ b/extensions-core/druid-catalog/src/test/java/org/apache/druid/catalog/storage/TableManagerTest.java @@ -40,9 +40,7 @@ import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.metadata.TestDerbyConnector; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; -import org.apache.druid.segment.VirtualColumns; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -94,21 +92,16 @@ public void tearDown() public void testCreate() throws DuplicateKeyException, NotFoundException { final DatasourceProjectionMetadata projectionMetadata = new DatasourceProjectionMetadata( - new AggregateProjectionSpec( - "projection", - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of( - new StringDimensionSchema("dim") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("count") - } - ) + AggregateProjectionSpec.builder("projection") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns(new StringDimensionSchema("dim")) + .aggregators(new CountAggregatorFactory("count")) + .build() ); List jsonProjectionSpec = JSON_MAPPER.convertValue(ImmutableList.of(projectionMetadata), List.class); diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java index 719f096a4e93..4cfd1dcd363d 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java @@ -57,9 +57,7 @@ import org.apache.druid.msq.test.CounterSnapshotMatcher; import org.apache.druid.msq.test.MSQTestBase; import org.apache.druid.msq.util.MultiStageQueryContext; -import org.apache.druid.query.OrderBy; import org.apache.druid.query.QueryContexts; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; @@ -69,7 +67,6 @@ import org.apache.druid.query.groupby.orderby.OrderByColumnSpec; import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; import org.apache.druid.segment.AggregateProjectionMetadata; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.ValueType; @@ -196,40 +193,38 @@ protected CatalogResolver createMockCatalogResolver() DatasourceDefn.PROJECTIONS_KEYS_PROPERTY, ImmutableList.of( new DatasourceProjectionMetadata( - new AggregateProjectionSpec( - "channel_added_hourly", - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of( - new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - new StringDimensionSchema("channel") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_added", "added") - } - ) + AggregateProjectionSpec.builder("channel_added_hourly") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + new StringDimensionSchema("channel") + ) + .aggregators( + new LongSumAggregatorFactory("sum_added", "added") + ) + .build() ), new DatasourceProjectionMetadata( - new AggregateProjectionSpec( - "channel_delta_daily", - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.DAY, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of( - new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - new StringDimensionSchema("channel") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_delta", "delta") - } - ) + AggregateProjectionSpec.builder("channel_delta_daily") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.DAY, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + new StringDimensionSchema("channel") + ) + .aggregators( + new LongSumAggregatorFactory("sum_delta", "delta") + ) + .build() ) ) ) @@ -550,45 +545,31 @@ public void testInsertOnExternalDataSourceWithCatalogProjections(String contextN .add("delta", ColumnType.LONG) .build(); AggregateProjectionMetadata expectedProjection = new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "channel_added_hourly", - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel"), - new AggregatorFactory[] { - new LongSumAggregatorFactory("sum_added", "added") - }, - ImmutableList.of( - OrderBy.ascending(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - OrderBy.ascending("channel") - ) - ), + AggregateProjectionMetadata.schemaBuilder("channel_added_hourly") + .timeColumnName(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupAndOrder(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel") + .aggregators(new LongSumAggregatorFactory("sum_added", "added")) + .build(), 16 ); AggregateProjectionMetadata expectedProjection2 = new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "channel_delta_daily", - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.DAY, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel"), - new AggregatorFactory[] { - new LongSumAggregatorFactory("sum_delta", "delta") - }, - ImmutableList.of( - OrderBy.ascending(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - OrderBy.ascending("channel") - ) - ), + AggregateProjectionMetadata.schemaBuilder("channel_delta_daily") + .timeColumnName(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.DAY, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupAndOrder(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel") + .aggregators(new LongSumAggregatorFactory("sum_delta", "delta")) + .build(), 11 ); diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java index 0d737321e32c..03fec193603f 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java @@ -64,7 +64,6 @@ import org.apache.druid.segment.AutoTypeColumnSchema; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.NestedDataColumnSchema; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.data.CompressionFactory; @@ -81,7 +80,6 @@ import org.junit.Test; import javax.annotation.Nullable; - import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -119,19 +117,19 @@ public class MSQCompactionRunnerTest NESTED_DIMENSION, AUTO_DIMENSION ); - private static final AggregateProjectionSpec PROJECTION_SPEC = new AggregateProjectionSpec( - "projection", - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of(STRING_DIMENSION), - new AggregatorFactory[]{ - new LongSumAggregatorFactory(LONG_DIMENSION.getName(), LONG_DIMENSION.getName()) - } - ); + private static final AggregateProjectionSpec PROJECTION_SPEC = + AggregateProjectionSpec.builder("projection") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns(STRING_DIMENSION) + .aggregators( + new LongSumAggregatorFactory(LONG_DIMENSION.getName(), LONG_DIMENSION.getName()) + ) + .build(); private static final Map INTERVAL_DATASCHEMAS = ImmutableMap.of( COMPACTION_INTERVAL, new CombinedDataSchema( diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java index 5d19bc4fc547..14160c5ff7a0 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/destination/DataSourceMSQDestinationTest.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.data.input.impl.AggregateProjectionSpec; @@ -29,10 +28,8 @@ import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; -import org.apache.druid.segment.VirtualColumns; import org.junit.Assert; import org.junit.Test; @@ -68,36 +65,29 @@ public void testEquals() ) .withPrefabValues( List.class, - ImmutableList.of( - new AggregateProjectionSpec( - "projection1", - VirtualColumns.EMPTY, - ImmutableList.of( - new StringDimensionSchema("region") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("count") - } - ) + List.of( + AggregateProjectionSpec.builder("projection1") + .groupingColumns(new StringDimensionSchema("region")) + .aggregators(new CountAggregatorFactory("count")) + .build() ), - ImmutableList.of( - new AggregateProjectionSpec( - "projection2", - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of( - new StringDimensionSchema("language"), - new StringDimensionSchema("region") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("count"), - new LongSumAggregatorFactory("sum_added", "added") - } - ) + List.of( + AggregateProjectionSpec.builder("projection2") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new StringDimensionSchema("language"), + new StringDimensionSchema("region") + ) + .aggregators( + new CountAggregatorFactory("count"), + new LongSumAggregatorFactory("sum_added", "added") + ) + .build() ) ) .usingGetClass() diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java index de2f068d73bc..2a6457e36349 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java @@ -1178,12 +1178,13 @@ private void processProjections(final QueryableIndex index) } projections.put( schema.getName(), - new AggregateProjectionSpec( - schema.getName(), - schema.getVirtualColumns(), - columnSchemas, - schema.getAggregators() - ) + AggregateProjectionSpec.builder() + .name(schema.getName()) + .virtualColumns(schema.getVirtualColumns()) + .filter(schema.getFilter()) + .groupingColumns(columnSchemas) + .aggregators(schema.getAggregators()) + .build() ); } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java index fcb3a1cf5d6d..e78085b0b104 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java @@ -63,7 +63,6 @@ import org.apache.druid.query.filter.SelectorDimFilter; import org.apache.druid.segment.DataSegmentsWithSchemas; import org.apache.druid.segment.SegmentUtils; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.loading.NoopSegmentCacheManager; import org.apache.druid.segment.transform.CompactionTransformSpec; @@ -118,22 +117,20 @@ public static Iterable constructorFeeder() private static final String DATA_SOURCE = "test"; private static final Interval INTERVAL_TO_INDEX = Intervals.of("2014-01-01/2014-01-02"); - private static final AggregateProjectionSpec PROJECTION_SPEC = new AggregateProjectionSpec( - "projection1", - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of( - new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - new StringDimensionSchema("dim", DimensionSchema.MultiValueHandling.ARRAY, null) - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("val", "val") - } - ); + private static final AggregateProjectionSpec PROJECTION_SPEC = + AggregateProjectionSpec.builder("projection1") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + new StringDimensionSchema("dim", DimensionSchema.MultiValueHandling.ARRAY, null) + ) + .aggregators(new LongSumAggregatorFactory("val", "val")) + .build(); private final LockGranularity lockGranularity; @@ -957,14 +954,10 @@ public void testRunParallelAddProjections() DATA_SOURCE, getSegmentCacheManagerFactory() ); - final AggregateProjectionSpec addProjection = new AggregateProjectionSpec( - "projection2", - VirtualColumns.EMPTY, - null, - new AggregatorFactory[]{ - new LongSumAggregatorFactory("val", "val") - } - ); + final AggregateProjectionSpec addProjection = + AggregateProjectionSpec.builder("projection2") + .aggregators(new LongSumAggregatorFactory("val", "val")) + .build(); final CompactionTask compactionTask = builder .inputSpec(new CompactionIntervalSpec(INTERVAL_TO_INDEX, null)) .tuningConfig(AbstractParallelIndexSupervisorTaskTest.DEFAULT_TUNING_CONFIG_FOR_PARALLEL_INDEXING) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java index 783214d77dd7..b14ae4c09e80 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java @@ -105,7 +105,6 @@ import org.apache.druid.segment.SegmentUtils; import org.apache.druid.segment.SimpleQueryableIndex; import org.apache.druid.segment.TestIndex; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.BaseColumn; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnCapabilitiesImpl; @@ -573,20 +572,18 @@ public void testSerdeWithProjections() throws IOException DATA_SOURCE, segmentCacheManagerFactory ); - final List projections = ImmutableList.of( - new AggregateProjectionSpec( - "test", - VirtualColumns.EMPTY, - ImmutableList.of( - new StringDimensionSchema("dim1"), - new StringDimensionSchema("dim2"), - new StringDimensionSchema("dim3") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("count"), - new LongSumAggregatorFactory("sum_long_dim_1", "long_dim_1") - } - ) + final List projections = List.of( + AggregateProjectionSpec.builder("test") + .groupingColumns( + new StringDimensionSchema("dim1"), + new StringDimensionSchema("dim2"), + new StringDimensionSchema("dim3") + ) + .aggregators( + new CountAggregatorFactory("count"), + new LongSumAggregatorFactory("sum_long_dim_1", "long_dim_1") + ) + .build() ); final CompactionTask task = builder diff --git a/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java b/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java index 925fd26f4fe1..d9e09cc6fba7 100644 --- a/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java +++ b/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java @@ -31,6 +31,7 @@ import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.OrderBy; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.filter.DimFilter; import org.apache.druid.segment.AggregateProjectionMetadata; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.VirtualColumns; @@ -56,9 +57,30 @@ public class AggregateProjectionSpec { public static final String TYPE_NAME = "aggregate"; + public static Builder builder() + { + return new Builder(); + } + + public static Builder builder(String name) + { + return new Builder().name(name); + } + + public static Builder builder(AggregateProjectionSpec spec) + { + return new Builder().name(spec.getName()) + .virtualColumns(spec.getVirtualColumns()) + .filter(spec.getFilter()) + .groupingColumns(spec.getGroupingColumns()) + .aggregators(spec.getAggregators()); + } + private final String name; - private final List groupingColumns; + @Nullable + private final DimFilter filter; private final VirtualColumns virtualColumns; + private final List groupingColumns; private final AggregatorFactory[] aggregators; private final List ordering; @Nullable @@ -67,6 +89,7 @@ public class AggregateProjectionSpec @JsonCreator public AggregateProjectionSpec( @JsonProperty("name") String name, + @JsonProperty("filter") @Nullable DimFilter filter, @JsonProperty("virtualColumns") @Nullable VirtualColumns virtualColumns, @JsonProperty("groupingColumns") @Nullable List groupingColumns, @JsonProperty("aggregators") @Nullable AggregatorFactory[] aggregators @@ -77,10 +100,14 @@ public AggregateProjectionSpec( } this.name = name; if (CollectionUtils.isNullOrEmpty(groupingColumns) && (aggregators == null || aggregators.length == 0)) { - throw InvalidInput.exception("projection[%s] groupingColumns and aggregators must not both be null or empty", name); + throw InvalidInput.exception( + "projection[%s] groupingColumns and aggregators must not both be null or empty", + name + ); } - this.groupingColumns = groupingColumns == null ? Collections.emptyList() : groupingColumns; + this.filter = filter; this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns; + this.groupingColumns = groupingColumns == null ? Collections.emptyList() : groupingColumns; // in the future this should be expanded to support user specified ordering, but for now we compute it based on // the grouping columns, which is consistent with how rollup ordering works for incremental index base table final ProjectionOrdering ordering = computeOrdering(this.virtualColumns, this.groupingColumns); @@ -95,6 +122,14 @@ public String getName() return name; } + @Nullable + @JsonProperty + @JsonInclude(JsonInclude.Include.NON_NULL) + public DimFilter getFilter() + { + return filter; + } + @JsonProperty @JsonInclude(JsonInclude.Include.NON_DEFAULT) public VirtualColumns getVirtualColumns() @@ -128,6 +163,7 @@ public AggregateProjectionMetadata.Schema toMetadataSchema() return new AggregateProjectionMetadata.Schema( name, timeColumnName, + filter, virtualColumns, groupingColumns.stream().map(DimensionSchema::getName).collect(Collectors.toList()), aggregators, @@ -146,6 +182,7 @@ public boolean equals(Object o) } AggregateProjectionSpec that = (AggregateProjectionSpec) o; return Objects.equals(name, that.name) + && Objects.equals(filter, that.filter) && Objects.equals(groupingColumns, that.groupingColumns) && Objects.equals(virtualColumns, that.virtualColumns) && Objects.deepEquals(aggregators, that.aggregators) @@ -155,7 +192,7 @@ public boolean equals(Object o) @Override public int hashCode() { - return Objects.hash(name, groupingColumns, virtualColumns, Arrays.hashCode(aggregators), ordering); + return Objects.hash(name, filter, virtualColumns, groupingColumns, Arrays.hashCode(aggregators), ordering); } @Override @@ -163,14 +200,14 @@ public String toString() { return "AggregateProjectionSpec{" + "name='" + name + '\'' + - ", groupingColumns=" + groupingColumns + + ", filter=" + filter + ", virtualColumns=" + virtualColumns + + ", groupingColumns=" + groupingColumns + ", aggregators=" + Arrays.toString(aggregators) + ", ordering=" + ordering + '}'; } - private static ProjectionOrdering computeOrdering(VirtualColumns virtualColumns, List groupingColumns) { if (groupingColumns.isEmpty()) { @@ -217,4 +254,66 @@ private ProjectionOrdering(List ordering, @Nullable String timeColumnNa this.timeColumnName = timeColumnName; } } + + public static final class Builder + { + private String name; + private DimFilter filter; + private VirtualColumns virtualColumns = VirtualColumns.EMPTY; + private List groupingColumns; + private AggregatorFactory[] aggregators; + + public Builder name(String name) + { + this.name = name; + return this; + } + + public Builder filter(@Nullable DimFilter filter) + { + this.filter = filter; + return this; + } + + public Builder virtualColumns(@Nullable VirtualColumns virtualColumns) + { + this.virtualColumns = virtualColumns; + return this; + } + + public Builder virtualColumns(VirtualColumn... virtualColumns) + { + this.virtualColumns = VirtualColumns.create(virtualColumns); + return this; + } + + public Builder groupingColumns(@Nullable List groupingColumns) + { + this.groupingColumns = groupingColumns; + return this; + } + + public Builder groupingColumns(DimensionSchema... groupingColumns) + { + this.groupingColumns = Arrays.asList(groupingColumns); + return this; + } + + public Builder aggregators(@Nullable AggregatorFactory... aggregators) + { + this.aggregators = aggregators; + return this; + } + + public AggregateProjectionSpec build() + { + return new AggregateProjectionSpec( + name, + filter, + virtualColumns, + groupingColumns, + aggregators + ); + } + } } diff --git a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java index 83b762b629dc..dc0141fa7098 100644 --- a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java +++ b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java @@ -34,6 +34,7 @@ import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.OrderBy; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.filter.DimFilter; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.projections.Projections; import org.apache.druid.utils.CollectionUtils; @@ -44,7 +45,7 @@ import java.util.Comparator; import java.util.List; import java.util.Objects; -import java.util.Set; +import java.util.stream.Collectors; /** * Aggregate projection schema and row count information to store in {@link Metadata} which itself is stored inside a @@ -67,6 +68,11 @@ public class AggregateProjectionMetadata return Schema.COMPARATOR.compare(o1.getSchema(), o2.getSchema()); }; + public static SchemaBuilder schemaBuilder(String name) + { + return new SchemaBuilder().name(name); + } + private final Schema schema; private final int numRows; @@ -130,10 +136,10 @@ public static class Schema */ public static final Comparator COMPARATOR = (o1, o2) -> { // coarsest granularity first - if (o1.getGranularity().isFinerThan(o2.getGranularity())) { + if (o1.getEffectiveGranularity().isFinerThan(o2.getEffectiveGranularity())) { return 1; } - if (o2.getGranularity().isFinerThan(o1.getGranularity())) { + if (o2.getEffectiveGranularity().isFinerThan(o1.getEffectiveGranularity())) { return -1; } // fewer dimensions first @@ -163,6 +169,8 @@ public static class Schema private final String name; @Nullable private final String timeColumnName; + @Nullable + private final DimFilter filter; private final VirtualColumns virtualColumns; private final List groupingColumns; private final AggregatorFactory[] aggregators; @@ -171,12 +179,13 @@ public static class Schema // computed fields private final int timeColumnPosition; - private final Granularity granularity; + private final Granularity effectiveGranularity; @JsonCreator public Schema( @JsonProperty("name") String name, @JsonProperty("timeColumnName") @Nullable String timeColumnName, + @JsonProperty("filter") @Nullable DimFilter filter, @JsonProperty("virtualColumns") @Nullable VirtualColumns virtualColumns, @JsonProperty("groupingColumns") @Nullable List groupingColumns, @JsonProperty("aggregators") @Nullable AggregatorFactory[] aggregators, @@ -188,11 +197,15 @@ public Schema( } this.name = name; if (CollectionUtils.isNullOrEmpty(groupingColumns) && (aggregators == null || aggregators.length == 0)) { - throw DruidException.defensive("projection schema[%s] groupingColumns and aggregators must not both be null or empty", name); + throw DruidException.defensive( + "projection schema[%s] groupingColumns and aggregators must not both be null or empty", + name + ); } if (ordering == null) { throw DruidException.defensive("projection schema[%s] ordering must not be null", name); } + this.filter = filter; this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns; this.groupingColumns = groupingColumns == null ? Collections.emptyList() : groupingColumns; this.aggregators = aggregators == null ? new AggregatorFactory[0] : aggregators; @@ -219,7 +232,7 @@ public Schema( } this.timeColumnName = timeColumnName; this.timeColumnPosition = foundTimePosition; - this.granularity = granularity == null ? Granularities.ALL : granularity; + this.effectiveGranularity = granularity == null ? Granularities.ALL : granularity; } @JsonProperty @@ -235,6 +248,13 @@ public String getTimeColumnName() return timeColumnName; } + @JsonProperty + @Nullable + public DimFilter getFilter() + { + return filter; + } + @JsonProperty @JsonInclude(JsonInclude.Include.NON_DEFAULT) public VirtualColumns getVirtualColumns() @@ -275,236 +295,21 @@ public int getTimeColumnPosition() } @JsonIgnore - public Granularity getGranularity() + public Granularity getEffectiveGranularity() { - return granularity; - } - - /** - * Check if this projection "matches" a {@link CursorBuildSpec} for a query to see if we can use a projection - * instead. For a projection to match, all grouping columns of the build spec must match, virtual columns of the - * build spec must either be available as a physical column on the projection, or the inputs to the virtual column - * must be available on the projection, and all aggregators must be compatible with pre-aggregated columns of the - * projection per {@link AggregatorFactory#substituteCombiningFactory(AggregatorFactory)}. If the projection - * matches, this method returns a {@link Projections.ProjectionMatch} which contains an updated - * {@link CursorBuildSpec} which has the remaining virtual columns from the original build spec which must still be - * computed and the 'combining' aggregator factories to process the pre-aggregated data from the projection, as well - * as a mapping of query column names to projection column names. - * - * @param queryCursorBuildSpec the {@link CursorBuildSpec} that contains the required inputs to build a - * {@link CursorHolder} for a query - * @param physicalColumnChecker Helper utility which can determine if a physical column required by - * queryCursorBuildSpec is available on the projection OR does not exist on the base - * table either - * @return a {@link Projections.ProjectionMatch} if the {@link CursorBuildSpec} matches the projection, which - * contains information such as which - */ - @Nullable - public Projections.ProjectionMatch matches( - CursorBuildSpec queryCursorBuildSpec, - Projections.PhysicalColumnChecker physicalColumnChecker - ) - { - if (!queryCursorBuildSpec.isCompatibleOrdering(orderingWithTimeSubstitution)) { - return null; - } - Projections.ProjectionMatchBuilder matchBuilder = new Projections.ProjectionMatchBuilder(); - - final List queryGrouping = queryCursorBuildSpec.getGroupingColumns(); - if (queryGrouping != null) { - for (String queryColumn : queryGrouping) { - matchBuilder = matchRequiredColumn( - matchBuilder, - queryColumn, - queryCursorBuildSpec.getVirtualColumns(), - physicalColumnChecker - ); - if (matchBuilder == null) { - return null; - } - // a query grouping column must also be defined as a projection grouping column - if (isInvalidGrouping(queryColumn) || isInvalidGrouping(matchBuilder.getRemapValue(queryColumn))) { - return null; - } - } - } - if (queryCursorBuildSpec.getFilter() != null) { - for (String queryColumn : queryCursorBuildSpec.getFilter().getRequiredColumns()) { - matchBuilder = matchRequiredColumn( - matchBuilder, - queryColumn, - queryCursorBuildSpec.getVirtualColumns(), - physicalColumnChecker - ); - if (matchBuilder == null) { - return null; - } - } - } - if (!CollectionUtils.isNullOrEmpty(queryCursorBuildSpec.getAggregators())) { - boolean allMatch = true; - for (AggregatorFactory queryAgg : queryCursorBuildSpec.getAggregators()) { - boolean foundMatch = false; - for (AggregatorFactory projectionAgg : aggregators) { - final AggregatorFactory combining = queryAgg.substituteCombiningFactory(projectionAgg); - if (combining != null) { - matchBuilder.remapColumn(queryAgg.getName(), projectionAgg.getName()) - .addReferencedPhysicalColumn(projectionAgg.getName()) - .addPreAggregatedAggregator(combining) - .addMatchedQueryColumns(queryAgg.requiredFields()); - foundMatch = true; - break; - } - } - allMatch = allMatch && foundMatch; - } - if (!allMatch) { - return null; - } - } - // validate physical and virtual columns have all been accounted for - final Set matchedQueryColumns = matchBuilder.getMatchedQueryColumns(); - if (queryCursorBuildSpec.getPhysicalColumns() != null) { - for (String queryColumn : queryCursorBuildSpec.getPhysicalColumns()) { - // time is special handled - if (ColumnHolder.TIME_COLUMN_NAME.equals(queryColumn)) { - continue; - } - if (!matchedQueryColumns.contains(queryColumn)) { - matchBuilder = matchRequiredColumn( - matchBuilder, - queryColumn, - queryCursorBuildSpec.getVirtualColumns(), - physicalColumnChecker - ); - if (matchBuilder == null) { - return null; - } - } - } - for (VirtualColumn vc : queryCursorBuildSpec.getVirtualColumns().getVirtualColumns()) { - if (!matchedQueryColumns.contains(vc.getOutputName())) { - matchBuilder = matchRequiredColumn( - matchBuilder, - vc.getOutputName(), - queryCursorBuildSpec.getVirtualColumns(), - physicalColumnChecker - ); - if (matchBuilder == null) { - return null; - } - } - } - } - return matchBuilder.build(queryCursorBuildSpec); - } - - /** - * Ensure that the projection has the specified column required by a {@link CursorBuildSpec} in one form or another. - * If the column is a {@link VirtualColumn} on the build spec, ensure that the projection has an equivalent virtual - * column, or has the required inputs to compute the virtual column. If an equivalent virtual column exists, its - * name will be added to {@link Projections.ProjectionMatchBuilder#remapColumn(String, String)} so the query - * virtual column name can be mapped to the projection physical column name. If no equivalent virtual column exists, - * but the inputs are available on the projection to compute it, it will be added to - * {@link Projections.ProjectionMatchBuilder#addReferenceedVirtualColumn(VirtualColumn)}. - *

- * Finally, if the column is not a virtual column in the query, it is checked with - * {@link Projections.PhysicalColumnChecker} which true if the column is present on the projection OR if the column - * is NOT present on the base table (meaning missing columns that do not exist anywhere do not disqualify a - * projection from being used). - * - * @param matchBuilder match state to add mappings of query virtual columns to projection physical columns - * and query virtual columns which still must be computed from projection physical - * columns - * @param column Column name to check - * @param queryVirtualColumns {@link VirtualColumns} from the {@link CursorBuildSpec} required by the query - * @param physicalColumnChecker Helper to check if the physical column exists on a projection, or does not exist on - * the base table - * @return {@link Projections.ProjectionMatchBuilder} with updated state per the rules described above, or null - * if the column cannot be matched - */ - @Nullable - private Projections.ProjectionMatchBuilder matchRequiredColumn( - Projections.ProjectionMatchBuilder matchBuilder, - String column, - VirtualColumns queryVirtualColumns, - Projections.PhysicalColumnChecker physicalColumnChecker - ) - { - final VirtualColumn queryVirtualColumn = queryVirtualColumns.getVirtualColumn(column); - if (queryVirtualColumn != null) { - matchBuilder.addMatchedQueryColumn(column) - .addMatchedQueryColumns(queryVirtualColumn.requiredColumns()); - // check to see if we have an equivalent virtual column defined in the projection, if so we can - final VirtualColumn projectionEquivalent = virtualColumns.findEquivalent(queryVirtualColumn); - if (projectionEquivalent != null) { - final String remapColumnName; - if (Objects.equals(projectionEquivalent.getOutputName(), timeColumnName)) { - remapColumnName = ColumnHolder.TIME_COLUMN_NAME; - } else { - remapColumnName = projectionEquivalent.getOutputName(); - } - if (!queryVirtualColumn.getOutputName().equals(remapColumnName)) { - matchBuilder.remapColumn(queryVirtualColumn.getOutputName(), remapColumnName); - } - return matchBuilder.addReferencedPhysicalColumn(remapColumnName); - } - - matchBuilder.addReferenceedVirtualColumn(queryVirtualColumn); - final List requiredInputs = queryVirtualColumn.requiredColumns(); - if (requiredInputs.size() == 1 && ColumnHolder.TIME_COLUMN_NAME.equals(requiredInputs.get(0))) { - // special handle time granularity. in the future this should be reworked to push this concept into the - // virtual column and underlying expression itself, but this will do for now - final Granularity virtualGranularity = Granularities.fromVirtualColumn(queryVirtualColumn); - if (virtualGranularity != null) { - if (virtualGranularity.isFinerThan(granularity)) { - return null; - } - // same granularity, replace virtual column directly by remapping it to the physical column - if (granularity.equals(virtualGranularity)) { - return matchBuilder.remapColumn(column, ColumnHolder.TIME_COLUMN_NAME) - .addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); - } - return matchBuilder.addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); - } else { - // anything else with __time requires none granularity - if (Granularities.NONE.equals(granularity)) { - return matchBuilder.addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); - } - return null; - } - } else { - for (String required : requiredInputs) { - matchBuilder = matchRequiredColumn( - matchBuilder, - required, - queryVirtualColumns, - physicalColumnChecker - ); - if (matchBuilder == null) { - return null; - } - } - return matchBuilder; - } - } else { - if (physicalColumnChecker.check(name, column)) { - return matchBuilder.addMatchedQueryColumn(column) - .addReferencedPhysicalColumn(column); - } - return null; - } + return effectiveGranularity; } /** * Check if a column is either part of {@link #groupingColumns}, or at least is not present in - * {@link #virtualColumns}. Naively we would just check that grouping column contains the column in question, - * however we can also use a projection when a column is truly missing. {@link #matchRequiredColumn} returns a - * match builder if the column is present as either a physical column, or a virtual column, but a virtual column + * {@link #virtualColumns}. Naively, we would just check that grouping column contains the column in question, + * however, we can also use a projection when a column is truly missing. + * {@link Projections#matchAggregateProjection(Schema, CursorBuildSpec, Projections.PhysicalColumnChecker)} returns + * a match builder if the column is present as either a physical column, or a virtual column, but a virtual column * could also be present for an aggregator input, so we must further check that a column not in the grouping list * is also not a virtual column, the implication being that it is a missing column. */ - private boolean isInvalidGrouping(@Nullable String columnName) + public boolean isInvalidGrouping(@Nullable String columnName) { if (columnName == null) { return false; @@ -524,6 +329,7 @@ public boolean equals(Object o) Schema schema = (Schema) o; return Objects.equals(name, schema.name) && Objects.equals(timeColumnName, schema.timeColumnName) + && Objects.equals(filter, schema.filter) && Objects.equals(virtualColumns, schema.virtualColumns) && Objects.equals(groupingColumns, schema.groupingColumns) && Objects.deepEquals(aggregators, schema.aggregators) @@ -536,6 +342,7 @@ public int hashCode() return Objects.hash( name, timeColumnName, + filter, virtualColumns, groupingColumns, Arrays.hashCode(aggregators), @@ -554,9 +361,90 @@ public String toString() ", aggregators=" + Arrays.toString(aggregators) + ", ordering=" + ordering + ", timeColumnPosition=" + timeColumnPosition + - ", granularity=" + granularity + + ", effectiveGranularity=" + effectiveGranularity + ", orderingWithTimeSubstitution=" + orderingWithTimeSubstitution + '}'; } } + + public static class SchemaBuilder + { + @Nullable + private String name; + @Nullable + private String timeColumnName; + private VirtualColumns virtualColumns = VirtualColumns.EMPTY; + @Nullable + private DimFilter filter; + private List groupingColumns; + private AggregatorFactory[] aggregators; + private List ordering; + + public SchemaBuilder name(@Nullable String name) + { + this.name = name; + return this; + } + + public SchemaBuilder timeColumnName(@Nullable String timeColumnName) + { + this.timeColumnName = timeColumnName; + return this; + } + + public SchemaBuilder virtualColumns(VirtualColumns virtualColumns) + { + this.virtualColumns = virtualColumns; + return this; + } + + public SchemaBuilder virtualColumns(VirtualColumn... virtualColumns) + { + this.virtualColumns = VirtualColumns.create(virtualColumns); + return this; + } + + public SchemaBuilder filter(@Nullable DimFilter filter) + { + this.filter = filter; + return this; + } + + public SchemaBuilder groupAndOrder(String... groupingColumns) + { + this.groupingColumns = Arrays.asList(groupingColumns); + return ordering(groupingColumns); + } + + public SchemaBuilder aggregators(@Nullable AggregatorFactory... aggregators) + { + this.aggregators = aggregators; + return this; + } + + public SchemaBuilder ordering(final String... columnNames) + { + this.ordering = Arrays.stream(columnNames).map(OrderBy::ascending).collect(Collectors.toList()); + return this; + } + + public SchemaBuilder ordering(List ordering) + { + this.ordering = ordering; + return this; + } + + public Schema build() + { + return new Schema( + name, + timeColumnName, + filter, + virtualColumns, + groupingColumns, + aggregators, + ordering + ); + } + } } diff --git a/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java b/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java index aca3743cf6c9..35e847818e30 100644 --- a/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java +++ b/processing/src/main/java/org/apache/druid/segment/CursorBuildSpec.java @@ -278,6 +278,22 @@ public int hashCode() ); } + @Override + public String toString() + { + return "CursorBuildSpec{" + + "filter=" + filter + + ", interval=" + interval + + ", groupingColumns=" + groupingColumns + + ", virtualColumns=" + virtualColumns + + ", aggregators=" + aggregators + + ", preferredOrdering=" + preferredOrdering + + ", queryContext=" + queryContext + + ", isAggregate=" + isAggregate + + ", physicalColumns=" + physicalColumns + + '}'; + } + public static class CursorBuildSpecBuilder { @Nullable diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index f1faed65da14..95d39521908b 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -65,6 +65,7 @@ import org.apache.druid.segment.data.IndexedIterable; import org.apache.druid.segment.data.ListIndexed; import org.apache.druid.segment.data.VSizeColumnarMultiInts; +import org.apache.druid.segment.projections.ConstantTimeColumn; import org.apache.druid.segment.projections.Projections; import org.apache.druid.segment.serde.ComplexColumnPartSupplier; import org.apache.druid.segment.serde.FloatNumericColumnSupplier; @@ -742,7 +743,7 @@ private Map> readProjectionColumns( if (projectionSpec.getSchema().getTimeColumnName() == null) { projectionColumns.put( ColumnHolder.TIME_COLUMN_NAME, - Projections.makeConstantTimeSupplier(projectionSpec.getNumRows(), dataInterval.getStartMillis()) + ConstantTimeColumn.makeConstantTimeSupplier(projectionSpec.getNumRows(), dataInterval.getStartMillis()) ); } return projectionColumns; diff --git a/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java index 813d4ac2f5ae..792b305cccb7 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java @@ -62,6 +62,16 @@ public IsBooleanFilter(Filter baseFilter, boolean isTrue) this.isTrue = isTrue; } + public boolean isTrue() + { + return isTrue; + } + + public Filter getBaseFilter() + { + return baseFilter; + } + @Nullable @Override public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector) diff --git a/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java index 3dcf59ee47eb..69bad38af069 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/TrueFilter.java @@ -46,7 +46,7 @@ public static TrueFilter instance() return INSTANCE; } - private TrueFilter() + protected TrueFilter() { } diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/OnHeapAggregateProjection.java b/processing/src/main/java/org/apache/druid/segment/incremental/OnHeapAggregateProjection.java index ed104ef1ddfc..5ac8f4a71748 100644 --- a/processing/src/main/java/org/apache/druid/segment/incremental/OnHeapAggregateProjection.java +++ b/processing/src/main/java/org/apache/druid/segment/incremental/OnHeapAggregateProjection.java @@ -30,11 +30,14 @@ import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorAndSize; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.segment.AggregateProjectionMetadata; import org.apache.druid.segment.AutoTypeColumnIndexer; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.EncodedKeyComponent; +import org.apache.druid.segment.RowAdapters; +import org.apache.druid.segment.RowBasedColumnSelectorFactory; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.CapabilitiesBasedFormat; @@ -43,6 +46,7 @@ import org.apache.druid.segment.column.ColumnFormat; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; @@ -78,6 +82,8 @@ public class OnHeapAggregateProjection implements IncrementalIndexRowSelector private final long minTimestamp; private final AtomicInteger rowCounter = new AtomicInteger(0); private final AtomicInteger numEntries = new AtomicInteger(0); + @Nullable + private final ValueMatcher valueMatcher; public OnHeapAggregateProjection( AggregateProjectionSpec projectionSpec, @@ -121,6 +127,33 @@ public OnHeapAggregateProjection( this.aggregatorsMap = new LinkedHashMap<>(); this.aggregatorFactories = new AggregatorFactory[projectionSchema.getAggregators().length]; initializeAndValidateAggregators(projectionSpec, getBaseTableDimensionDesc, getBaseTableAggregatorFactory); + + if (projectionSpec.getFilter() != null) { + RowSignature.Builder bob = RowSignature.builder(); + if (projectionSchema.getTimeColumnPosition() < 0) { + bob.addTimeColumn(); + } + for (String groupingColumn : projectionSchema.getGroupingColumns()) { + if (projectionSchema.getTimeColumnName().equals(groupingColumn)) { + bob.addTimeColumn(); + } else { + bob.add(groupingColumn, dimensionsMap.get(groupingColumn).getCapabilities().toColumnType()); + } + } + valueMatcher = projectionSchema.getFilter() + .toFilter() + .makeMatcher( + RowBasedColumnSelectorFactory.create( + RowAdapters.standardRow(), + inputRowHolder::getRow, + bob.build(), + false, + false + ) + ); + } else { + valueMatcher = null; + } } /** @@ -134,6 +167,9 @@ public void addToFacts( ) { inputRowHolder.set(inputRow); + if (valueMatcher != null && !valueMatcher.matches(false)) { + return; + } final Object[] projectionDims = new Object[dimensions.size()]; for (int i = 0; i < projectionDims.length; i++) { int parentDimIndex = parentDimensionIndex[i]; @@ -153,7 +189,7 @@ public void addToFacts( final long timestamp; if (projectionSchema.getTimeColumnName() != null) { - timestamp = projectionSchema.getGranularity().bucketStart(DateTimes.utc(key.getTimestamp())).getMillis(); + timestamp = projectionSchema.getEffectiveGranularity().bucketStart(DateTimes.utc(key.getTimestamp())).getMillis(); if (timestamp < minTimestamp) { throw DruidException.defensive( "Cannot add row[%s] to projection[%s] because projection effective timestamp[%s] is below the minTimestamp[%s]", diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarNestedCommonFormatColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarNestedCommonFormatColumnSerializer.java index dc18970ad799..271d9c9b33f2 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarNestedCommonFormatColumnSerializer.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarNestedCommonFormatColumnSerializer.java @@ -212,23 +212,29 @@ public void writeTo( bitmapIndexWriter.setObjectsNotSorted(); final MutableBitmap[] bitmaps; bitmaps = new MutableBitmap[getCardinality()]; - for (int i = 0; i < bitmaps.length; i++) { - bitmaps[i] = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap(); - } final IntIterator rows = intermediateValueWriter.getIterator(); int rowCount = 0; while (rows.hasNext()) { final int dictId = rows.nextInt(); encodedValueSerializer.addValue(dictId); - bitmaps[dictId].add(rowCount++); + MutableBitmap b = bitmaps[dictId]; + if (b == null) { + b = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap(); + bitmaps[dictId] = b; + } + b.add(rowCount++); } for (int i = 0; i < bitmaps.length; i++) { final MutableBitmap bitmap = bitmaps[i]; - bitmapIndexWriter.write( - indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeImmutableBitmap(bitmap) - ); + if (bitmap == null) { + bitmapIndexWriter.write(null); + } else { + bitmapIndexWriter.write( + indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeImmutableBitmap(bitmap) + ); + } bitmaps[i] = null; // Reclaim memory } diff --git a/processing/src/main/java/org/apache/druid/segment/projections/ConstantTimeColumn.java b/processing/src/main/java/org/apache/druid/segment/projections/ConstantTimeColumn.java new file mode 100644 index 000000000000..6485cb9578de --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/projections/ConstantTimeColumn.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.projections; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.ConstantExprEvalSelector; +import org.apache.druid.segment.column.CapabilitiesBasedFormat; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; +import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.NumericColumn; +import org.apache.druid.segment.data.ReadableOffset; +import org.apache.druid.segment.vector.ConstantVectorSelectors; +import org.apache.druid.segment.vector.ReadableVectorOffset; +import org.apache.druid.segment.vector.VectorValueSelector; + +public class ConstantTimeColumn implements NumericColumn +{ + public static Supplier makeConstantTimeSupplier(int numRows, long constant) + { + return Suppliers.memoize( + () -> new ColumnBuilder().setNumericColumnSupplier(() -> new ConstantTimeColumn(numRows, constant)) + .setColumnFormat( + new CapabilitiesBasedFormat( + ColumnCapabilitiesImpl.createDefault().setType(ColumnType.LONG) + ) + ) + .setType(ColumnType.LONG) + .setHasNulls(false) + .build() + ); + } + + private final int numRows; + private final long constant; + + public ConstantTimeColumn(int numRows, long constant) + { + this.numRows = numRows; + this.constant = constant; + } + + @Override + public int length() + { + return numRows; + } + + @Override + public long getLongSingleValueRow(int rowNum) + { + return constant; + } + + @Override + public void close() + { + // nothing to close + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + + } + + @Override + public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + { + return new ConstantExprEvalSelector(ExprEval.ofLong(constant)); + } + + @Override + public VectorValueSelector makeVectorValueSelector(ReadableVectorOffset offset) + { + return ConstantVectorSelectors.vectorValueSelector(offset, constant); + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/projections/ProjectionFilterMatch.java b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionFilterMatch.java new file mode 100644 index 000000000000..ab4b79624bd6 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionFilterMatch.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.projections; + +import org.apache.druid.query.filter.Filter; +import org.apache.druid.segment.filter.TrueFilter; + +/** + * Terminal marker used by {@link Projections#rewriteFilter(Filter, Filter)} indicating that a {@link Filter} can be + * dropped because it matches the filter of a projection + */ +public final class ProjectionFilterMatch extends TrueFilter +{ + static final ProjectionFilterMatch INSTANCE = new ProjectionFilterMatch(); + + private ProjectionFilterMatch() + { + // no instantiation + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatch.java b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatch.java new file mode 100644 index 000000000000..82cd75982375 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatch.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.projections; + +import org.apache.druid.segment.CursorBuildSpec; + +import java.util.Map; +import java.util.Objects; + +/** + * Transformed {@link CursorBuildSpec} to run against a projection and remapping of + */ +public final class ProjectionMatch +{ + private final CursorBuildSpec cursorBuildSpec; + private final Map remapColumns; + + public ProjectionMatch(CursorBuildSpec cursorBuildSpec, Map remapColumns) + { + this.cursorBuildSpec = cursorBuildSpec; + this.remapColumns = remapColumns; + } + + public CursorBuildSpec getCursorBuildSpec() + { + return cursorBuildSpec; + } + + public Map getRemapColumns() + { + return remapColumns; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof ProjectionMatch)) { + return false; + } + ProjectionMatch that = (ProjectionMatch) o; + return Objects.equals(cursorBuildSpec, that.cursorBuildSpec) && Objects.equals(remapColumns, that.remapColumns); + } + + @Override + public int hashCode() + { + return Objects.hash(cursorBuildSpec, remapColumns); + } + + @Override + public String toString() + { + return "ProjectionMatch{" + + "cursorBuildSpec=" + cursorBuildSpec + + ", remapColumns=" + remapColumns + + '}'; + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatchBuilder.java b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatchBuilder.java new file mode 100644 index 000000000000..8a704120ae5b --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionMatchBuilder.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.projections; + +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.filter.Filter; +import org.apache.druid.segment.CursorBuildSpec; +import org.apache.druid.segment.VirtualColumn; +import org.apache.druid.segment.VirtualColumns; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * State holder used by {@link Projections#matchAggregateProjection} for building a {@link ProjectionMatch}. Tracks + * which all kinds of stuff to map a projection to the contents of a query {@link CursorBuildSpec}, accumulating + * details about how to transform into a new {@link CursorBuildSpec} that will run against the projection. + */ +public final class ProjectionMatchBuilder +{ + private final Set referencedPhysicalColumns; + private final Set referencedVirtualColumns; + private final Map remapColumns; + private final List combiningFactories; + private final Set matchedQueryColumns; + @Nullable + private Filter rewriteFilter; + + public ProjectionMatchBuilder() + { + this.referencedPhysicalColumns = new HashSet<>(); + this.referencedVirtualColumns = new HashSet<>(); + this.remapColumns = new HashMap<>(); + this.combiningFactories = new ArrayList<>(); + this.matchedQueryColumns = new HashSet<>(); + } + + /** + * Map a query column name to a projection column name + */ + public ProjectionMatchBuilder remapColumn(String queryColumn, String projectionColumn) + { + remapColumns.put(queryColumn, projectionColumn); + return this; + } + + @Nullable + public String getRemapValue(String queryColumn) + { + return remapColumns.get(queryColumn); + } + + /** + * Add a projection physical column, which will later be added to {@link ProjectionMatch#getCursorBuildSpec()} if + * the projection matches + */ + public ProjectionMatchBuilder addReferencedPhysicalColumn(String column) + { + referencedPhysicalColumns.add(column); + return this; + } + + /** + * Add a query virtual column that can use projection physical columns as inputs to the match builder, which will + * later be added to {@link ProjectionMatch#getCursorBuildSpec()} if the projection matches + */ + public ProjectionMatchBuilder addReferenceedVirtualColumn(VirtualColumn virtualColumn) + { + referencedVirtualColumns.add(virtualColumn); + return this; + } + + /** + * Add a query {@link AggregatorFactory#substituteCombiningFactory(AggregatorFactory)} which can combine the inputs + * of a selector created by a projection {@link AggregatorFactory} + */ + public ProjectionMatchBuilder addPreAggregatedAggregator(AggregatorFactory aggregator) + { + combiningFactories.add(aggregator); + return this; + } + + public ProjectionMatchBuilder addMatchedQueryColumn(String queryColumn) + { + matchedQueryColumns.add(queryColumn); + return this; + } + + public ProjectionMatchBuilder addMatchedQueryColumns(Collection queryColumns) + { + matchedQueryColumns.addAll(queryColumns); + return this; + } + + public ProjectionMatchBuilder rewriteFilter(Filter rewriteFilter) + { + this.rewriteFilter = rewriteFilter; + return this; + } + + public Filter getRewriteFilter() + { + return rewriteFilter; + } + + public Map getRemapColumns() + { + return remapColumns; + } + + public Set getMatchedQueryColumns() + { + return matchedQueryColumns; + } + + public ProjectionMatch build(CursorBuildSpec queryCursorBuildSpec) + { + return new ProjectionMatch( + CursorBuildSpec.builder(queryCursorBuildSpec) + .setFilter(rewriteFilter) + .setPhysicalColumns(referencedPhysicalColumns) + .setVirtualColumns(VirtualColumns.fromIterable(referencedVirtualColumns)) + .setAggregators(combiningFactories) + .build(), + remapColumns + ); + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java index db067bde85b7..f1971c05e7e8 100644 --- a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java +++ b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java @@ -19,36 +19,25 @@ package org.apache.druid.segment.projections; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; +import com.google.common.collect.Lists; import org.apache.druid.data.input.impl.AggregateProjectionSpec; import org.apache.druid.error.InvalidInput; -import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.aggregation.AggregatorFactory; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import org.apache.druid.query.filter.Filter; import org.apache.druid.segment.AggregateProjectionMetadata; -import org.apache.druid.segment.ColumnValueSelector; -import org.apache.druid.segment.ConstantExprEvalSelector; import org.apache.druid.segment.CursorBuildSpec; +import org.apache.druid.segment.CursorHolder; import org.apache.druid.segment.VirtualColumn; -import org.apache.druid.segment.VirtualColumns; -import org.apache.druid.segment.column.CapabilitiesBasedFormat; -import org.apache.druid.segment.column.ColumnBuilder; -import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnHolder; -import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.NumericColumn; -import org.apache.druid.segment.data.ReadableOffset; -import org.apache.druid.segment.vector.ConstantVectorSelectors; -import org.apache.druid.segment.vector.ReadableVectorOffset; -import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.segment.filter.AndFilter; +import org.apache.druid.segment.filter.IsBooleanFilter; +import org.apache.druid.utils.CollectionUtils; import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -58,21 +47,6 @@ public class Projections { - public static Supplier makeConstantTimeSupplier(int numRows, long constant) - { - return Suppliers.memoize( - () -> new ColumnBuilder().setNumericColumnSupplier(() -> new ConstantTimeColumn(numRows, constant)) - .setColumnFormat( - new CapabilitiesBasedFormat( - ColumnCapabilitiesImpl.createDefault().setType(ColumnType.LONG) - ) - ) - .setType(ColumnType.LONG) - .setHasNulls(false) - .build() - ); - } - @Nullable public static QueryableProjection findMatchingProjection( CursorBuildSpec cursorBuildSpec, @@ -91,7 +65,7 @@ public static QueryableProjection findMatchingProjection( if (name != null && !name.equals(spec.getSchema().getName())) { continue; } - final ProjectionMatch match = spec.getSchema().matches(cursorBuildSpec, physicalChecker); + final ProjectionMatch match = matchAggregateProjection(spec.getSchema(), cursorBuildSpec, physicalChecker); if (match != null) { if (cursorBuildSpec.getQueryMetrics() != null) { cursorBuildSpec.getQueryMetrics().projection(spec.getSchema().getName()); @@ -113,206 +87,490 @@ public static QueryableProjection findMatchingProjection( return null; } - public static String getProjectionSmooshV9FileName(AggregateProjectionMetadata projectionSpec, String columnName) - { - return getProjectionSmooshV9Prefix(projectionSpec) + columnName; - } - - public static String getProjectionSmooshV9Prefix(AggregateProjectionMetadata projectionSpec) - { - return projectionSpec.getSchema().getName() + "/"; - } /** - * Returns true if column is defined in {@link AggregateProjectionSpec#getGroupingColumns()} OR if the column does not - * exist in the base table. Part of determining if a projection can be used for a given {@link CursorBuildSpec}, + * Check if this projection "matches" a {@link CursorBuildSpec} for a query to see if we can use a projection + * instead. For a projection to match, all grouping columns of the build spec must match, virtual columns of the + * build spec must either be available as a physical column on the projection, or the inputs to the virtual column + * must be available on the projection, and all aggregators must be compatible with pre-aggregated columns of the + * projection per {@link AggregatorFactory#substituteCombiningFactory(AggregatorFactory)}. If the projection + * matches, this method returns a {@link ProjectionMatch} which contains an updated {@link CursorBuildSpec} which has + * the remaining virtual columns from the original build spec which must still be computed and the 'combining' + * aggregator factories to process the pre-aggregated data from the projection, as well as a mapping of query column + * names to projection column names. * - * @see AggregateProjectionMetadata.Schema#matches(CursorBuildSpec, PhysicalColumnChecker) + * @param projection the {@link AggregateProjectionMetadata.Schema} to check for match + * @param queryCursorBuildSpec the {@link CursorBuildSpec} that contains the required inputs to build a + * {@link CursorHolder} for a query + * @param physicalColumnChecker Helper utility which can determine if a physical column required by + * queryCursorBuildSpec is available on the projection OR does not exist on the base + * table either + * @return a {@link ProjectionMatch} if the {@link CursorBuildSpec} matches the projection, else null */ - @FunctionalInterface - public interface PhysicalColumnChecker - { - boolean check(String projectionName, String columnName); - } - - public static final class ProjectionMatch + @Nullable + public static ProjectionMatch matchAggregateProjection( + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + PhysicalColumnChecker physicalColumnChecker + ) { - private final CursorBuildSpec cursorBuildSpec; - private final Map remapColumns; + if (!queryCursorBuildSpec.isCompatibleOrdering(projection.getOrderingWithTimeColumnSubstitution())) { + return null; + } + ProjectionMatchBuilder matchBuilder = new ProjectionMatchBuilder(); - public ProjectionMatch(CursorBuildSpec cursorBuildSpec, Map remapColumns) - { - this.cursorBuildSpec = cursorBuildSpec; - this.remapColumns = remapColumns; + // match virtual columns first, which will populate the 'remapColumns' of the match builder + matchBuilder = matchQueryVirtualColumns(projection, queryCursorBuildSpec, physicalColumnChecker, matchBuilder); + if (matchBuilder == null) { + return null; } - public CursorBuildSpec getCursorBuildSpec() - { - return cursorBuildSpec; + matchBuilder = matchFilter(projection, queryCursorBuildSpec, physicalColumnChecker, matchBuilder); + if (matchBuilder == null) { + return null; } - public Map getRemapColumns() - { - return remapColumns; + matchBuilder = matchGrouping(projection, queryCursorBuildSpec, physicalColumnChecker, matchBuilder); + if (matchBuilder == null) { + return null; } - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (!(o instanceof ProjectionMatch)) { - return false; - } - ProjectionMatch that = (ProjectionMatch) o; - return Objects.equals(cursorBuildSpec, that.cursorBuildSpec) && Objects.equals(remapColumns, that.remapColumns); + matchBuilder = matchAggregators(projection, queryCursorBuildSpec, matchBuilder); + if (matchBuilder == null) { + return null; } - @Override - public int hashCode() - { - return Objects.hash(cursorBuildSpec, remapColumns); + matchBuilder = matchRemainingPhysicalColumns(projection, queryCursorBuildSpec, physicalColumnChecker, matchBuilder); + if (matchBuilder == null) { + return null; } + + return matchBuilder.build(queryCursorBuildSpec); } - public static final class ProjectionMatchBuilder + @Nullable + public static ProjectionMatchBuilder matchQueryVirtualColumns( + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + PhysicalColumnChecker physicalColumnChecker, + ProjectionMatchBuilder matchBuilder + ) { - private final Set referencedPhysicalColumns; - private final Set referencedVirtualColumns; - private final Map remapColumns; - private final List combiningFactories; - private final Set matchedQueryColumns; - - public ProjectionMatchBuilder() - { - this.referencedPhysicalColumns = new HashSet<>(); - this.referencedVirtualColumns = new HashSet<>(); - this.remapColumns = new HashMap<>(); - this.combiningFactories = new ArrayList<>(); - this.matchedQueryColumns = new HashSet<>(); + for (VirtualColumn vc : queryCursorBuildSpec.getVirtualColumns().getVirtualColumns()) { + matchBuilder = matchQueryVirtualColumn( + vc, + projection, + queryCursorBuildSpec, + physicalColumnChecker, + matchBuilder + ); + if (matchBuilder == null) { + return null; + } } + return matchBuilder; + } - /** - * Map a query column name to a projection column name - */ - public ProjectionMatchBuilder remapColumn(String queryColumn, String projectionColumn) - { - remapColumns.put(queryColumn, projectionColumn); - return this; - } + @Nullable + public static ProjectionMatchBuilder matchFilter( + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + PhysicalColumnChecker physicalColumnChecker, + ProjectionMatchBuilder matchBuilder + ) + { + // if the projection has a filter, the query must contain this filter match + if (projection.getFilter() != null) { + final Filter queryFilter = queryCursorBuildSpec.getFilter(); + if (queryFilter != null) { + // try to rewrite the query filter into a projection filter, if the rewrite is valid, we can proceed + final Filter projectionFilter = projection.getFilter().toOptimizedFilter(false); + final Map filterRewrites = new HashMap<>(); + // start with identity + for (String required : queryFilter.getRequiredColumns()) { + filterRewrites.put(required, required); + } + // overlay projection rewrites + filterRewrites.putAll(matchBuilder.getRemapColumns()); - @Nullable - public String getRemapValue(String queryColumn) - { - return remapColumns.get(queryColumn); - } + final Filter remappedQueryFilter = queryFilter.rewriteRequiredColumns(filterRewrites); - /** - * Add a projection physical column, which will later be added to {@link ProjectionMatch#getCursorBuildSpec()} if - * the projection matches - */ - public ProjectionMatchBuilder addReferencedPhysicalColumn(String column) - { - referencedPhysicalColumns.add(column); - return this; + final Filter rewritten = rewriteFilter(projectionFilter, remappedQueryFilter); + // if the filter does not contain the projection filter, we cannot match this projection + if (rewritten == null) { + return null; + } + //noinspection ObjectEquality + if (rewritten == ProjectionFilterMatch.INSTANCE) { + // we can remove the whole thing since the query filter exactly matches the projection filter + matchBuilder.rewriteFilter(null); + } else { + // otherwise, we partially rewrote the query filter to eliminate the projection filter since it is baked in + matchBuilder.rewriteFilter(rewritten); + } + } else { + // projection has a filter, but the query doesn't, no good + return null; + } + } else { + // projection doesn't have a filter, retain the original + matchBuilder.rewriteFilter(queryCursorBuildSpec.getFilter()); } - /** - * Add a query virtual column that can use projection physical columns as inputs to the match builder, which will - * later be added to {@link ProjectionMatch#getCursorBuildSpec()} if the projection matches - */ - public ProjectionMatchBuilder addReferenceedVirtualColumn(VirtualColumn virtualColumn) - { - referencedVirtualColumns.add(virtualColumn); - return this; + // now that filter has been possibly rewritten, make sure the projection actually has all the required columns. + if (matchBuilder.getRewriteFilter() != null) { + for (String queryColumn : matchBuilder.getRewriteFilter().getRequiredColumns()) { + matchBuilder = matchRequiredColumn( + queryColumn, + projection, + queryCursorBuildSpec, + physicalColumnChecker, + matchBuilder + ); + if (matchBuilder == null) { + return null; + } + } } - /** - * Add a query {@link AggregatorFactory#substituteCombiningFactory(AggregatorFactory)} which can combine the inputs - * of a selector created by a projection {@link AggregatorFactory} - */ - public ProjectionMatchBuilder addPreAggregatedAggregator(AggregatorFactory aggregator) - { - combiningFactories.add(aggregator); - return this; - } + return matchBuilder; + } - public ProjectionMatchBuilder addMatchedQueryColumn(String queryColumn) - { - matchedQueryColumns.add(queryColumn); - return this; + @Nullable + public static ProjectionMatchBuilder matchGrouping( + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + PhysicalColumnChecker physicalColumnChecker, + ProjectionMatchBuilder matchBuilder + ) + { + final List queryGrouping = queryCursorBuildSpec.getGroupingColumns(); + if (queryGrouping != null) { + for (String queryColumn : queryGrouping) { + matchBuilder = matchRequiredColumn( + queryColumn, + projection, + queryCursorBuildSpec, + physicalColumnChecker, + matchBuilder + ); + if (matchBuilder == null) { + return null; + } + // a query grouping column must also be defined as a projection grouping column + if (projection.isInvalidGrouping(queryColumn)) { + return null; + } + // even if remapped + if (projection.isInvalidGrouping(matchBuilder.getRemapValue(queryColumn))) { + return null; + } + } } + return matchBuilder; + } - public ProjectionMatchBuilder addMatchedQueryColumns(Collection queryColumns) - { - matchedQueryColumns.addAll(queryColumns); - return this; + @Nullable + public static ProjectionMatchBuilder matchAggregators( + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + ProjectionMatchBuilder matchBuilder + ) + { + if (CollectionUtils.isNullOrEmpty(queryCursorBuildSpec.getAggregators())) { + return matchBuilder; + } + boolean allMatch = true; + for (AggregatorFactory queryAgg : queryCursorBuildSpec.getAggregators()) { + boolean foundMatch = false; + for (AggregatorFactory projectionAgg : projection.getAggregators()) { + final AggregatorFactory combining = queryAgg.substituteCombiningFactory(projectionAgg); + if (combining != null) { + matchBuilder.remapColumn(queryAgg.getName(), projectionAgg.getName()) + .addReferencedPhysicalColumn(projectionAgg.getName()) + .addPreAggregatedAggregator(combining) + .addMatchedQueryColumns(queryAgg.requiredFields()); + foundMatch = true; + break; + } + } + allMatch = allMatch && foundMatch; + } + if (allMatch) { + return matchBuilder; } + return null; + } - public Set getMatchedQueryColumns() - { - return matchedQueryColumns; + @Nullable + public static ProjectionMatchBuilder matchRemainingPhysicalColumns( + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + PhysicalColumnChecker physicalColumnChecker, + ProjectionMatchBuilder matchBuilder + ) + { + // validate physical and virtual columns have all been accounted for + final Set matchedQueryColumns = matchBuilder.getMatchedQueryColumns(); + if (queryCursorBuildSpec.getPhysicalColumns() != null) { + for (String queryColumn : queryCursorBuildSpec.getPhysicalColumns()) { + // a projection always has a __time column, it just might be a constant of the segment interval start if the + // projection itself did not transform the base table __time column + if (ColumnHolder.TIME_COLUMN_NAME.equals(queryColumn)) { + continue; + } + if (!matchedQueryColumns.contains(queryColumn)) { + matchBuilder = matchQueryPhysicalColumn( + queryColumn, + projection, + physicalColumnChecker, + matchBuilder + ); + if (matchBuilder == null) { + return null; + } + } + } } + return matchBuilder; + } - public ProjectionMatch build(CursorBuildSpec queryCursorBuildSpec) - { - return new ProjectionMatch( - CursorBuildSpec.builder(queryCursorBuildSpec) - .setPhysicalColumns(referencedPhysicalColumns) - .setVirtualColumns(VirtualColumns.fromIterable(referencedVirtualColumns)) - .setAggregators(combiningFactories) - .build(), - remapColumns + /** + * Ensure that the projection has the specified column required by a {@link CursorBuildSpec} in one form or another. + * If the column is a {@link VirtualColumn} on the build spec, ensure that the projection has an equivalent virtual + * column, or has the required inputs to compute the virtual column. If an equivalent virtual column exists, its + * name will be added to {@link ProjectionMatchBuilder#remapColumn(String, String)} so the query virtual column name\ + * can be mapped to the projection physical column name. If no equivalent virtual column exists, but the inputs are + * available on the projection to compute it, it will be added to + * {@link ProjectionMatchBuilder#addReferenceedVirtualColumn(VirtualColumn)}. + *

+ * Finally, if the column is not a virtual column in the query, it is checked with {@link PhysicalColumnChecker} + * which true if the column is present on the projection OR if the column is NOT present on the base table (meaning + * missing columns that do not exist anywhere do not disqualify a projection from being used). + * + * @param column Column name to check + * @param projection {@link AggregateProjectionMetadata.Schema} to match against + * @param queryCursorBuildSpec the {@link CursorBuildSpec} required by the query + * @param physicalColumnChecker Helper to check if the physical column exists on a projection, or does not exist on + * the base table + * @param matchBuilder match state to add mappings of query virtual columns to projection physical columns + * and query virtual columns which still must be computed from projection physical + * columns + * @return {@link ProjectionMatchBuilder} with updated state per the rules described above, or null if the column + * cannot be matched + */ + @Nullable + public static ProjectionMatchBuilder matchRequiredColumn( + String column, + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + PhysicalColumnChecker physicalColumnChecker, + ProjectionMatchBuilder matchBuilder + ) + { + if (matchBuilder.getMatchedQueryColumns().contains(column)) { + return matchBuilder; + } + final VirtualColumn virtualColumn = queryCursorBuildSpec.getVirtualColumns().getVirtualColumn(column); + if (virtualColumn != null) { + return matchQueryVirtualColumn( + virtualColumn, + projection, + queryCursorBuildSpec, + physicalColumnChecker, + matchBuilder ); } + + return matchQueryPhysicalColumn(column, projection, physicalColumnChecker, matchBuilder); } - private static class ConstantTimeColumn implements NumericColumn + @Nullable + public static ProjectionMatchBuilder matchQueryVirtualColumn( + VirtualColumn queryVirtualColumn, + AggregateProjectionMetadata.Schema projection, + CursorBuildSpec queryCursorBuildSpec, + PhysicalColumnChecker physicalColumnChecker, + ProjectionMatchBuilder matchBuilder + ) { - private final int numRows; - private final long constant; + // check to see if we have an equivalent virtual column defined in the projection, if so we can + final VirtualColumn projectionEquivalent = projection.getVirtualColumns().findEquivalent(queryVirtualColumn); + if (projectionEquivalent != null) { + final String remapColumnName; + if (Objects.equals(projectionEquivalent.getOutputName(), projection.getTimeColumnName())) { + remapColumnName = ColumnHolder.TIME_COLUMN_NAME; + } else { + remapColumnName = projectionEquivalent.getOutputName(); + } + if (!queryVirtualColumn.getOutputName().equals(remapColumnName)) { + matchBuilder.remapColumn(queryVirtualColumn.getOutputName(), remapColumnName); + } + return matchBuilder.addMatchedQueryColumn(queryVirtualColumn.getOutputName()) + .addMatchedQueryColumns(queryVirtualColumn.requiredColumns()) + .addReferencedPhysicalColumn(remapColumnName); + } - private ConstantTimeColumn(int numRows, long constant) - { - this.numRows = numRows; - this.constant = constant; + matchBuilder.addMatchedQueryColumn(queryVirtualColumn.getOutputName()) + .addReferenceedVirtualColumn(queryVirtualColumn); + final List requiredInputs = queryVirtualColumn.requiredColumns(); + if (requiredInputs.size() == 1 && ColumnHolder.TIME_COLUMN_NAME.equals(requiredInputs.get(0))) { + // special handle time granularity. in the future this should be reworked to push this concept into the + // virtual column and underlying expression itself, but this will do for now + final Granularity virtualGranularity = Granularities.fromVirtualColumn(queryVirtualColumn); + if (virtualGranularity != null) { + if (virtualGranularity.isFinerThan(projection.getEffectiveGranularity())) { + return null; + } + // same granularity, replace virtual column directly by remapping it to the physical column + if (projection.getEffectiveGranularity().equals(virtualGranularity)) { + return matchBuilder.remapColumn(queryVirtualColumn.getOutputName(), ColumnHolder.TIME_COLUMN_NAME) + .addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); + } + return matchBuilder.addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); + } else { + // anything else with __time requires none granularity + if (Granularities.NONE.equals(projection.getEffectiveGranularity())) { + return matchBuilder.addReferencedPhysicalColumn(ColumnHolder.TIME_COLUMN_NAME); + } + return null; + } + } else { + for (String required : requiredInputs) { + matchBuilder = matchRequiredColumn( + required, + projection, + queryCursorBuildSpec, + physicalColumnChecker, + matchBuilder + ); + if (matchBuilder == null) { + return null; + } + } + return matchBuilder; } + } - @Override - public int length() - { - return numRows; + @Nullable + public static ProjectionMatchBuilder matchQueryPhysicalColumn( + String column, + AggregateProjectionMetadata.Schema projection, + PhysicalColumnChecker physicalColumnChecker, + ProjectionMatchBuilder matchBuilder + ) + { + if (physicalColumnChecker.check(projection.getName(), column)) { + return matchBuilder.addMatchedQueryColumn(column) + .addReferencedPhysicalColumn(column); } + return null; + } - @Override - public long getLongSingleValueRow(int rowNum) - { - return constant; + /** + * Rewrites a query {@link Filter} if possible, removing the {@link Filter} of a projection. To match a projection + * filter, the query filter must be equal to the projection filter, or must contain the projection filter as the child + * of an AND filter. This method returns null + * indicating that a rewrite is impossible with the implication that the query cannot use the projection because the + * projection doesn't contain all the rows the query would match if not using the projection. + */ + @Nullable + public static Filter rewriteFilter(Filter projectionFilter, Filter queryFilter) + { + if (queryFilter.equals(projectionFilter)) { + return ProjectionFilterMatch.INSTANCE; } + if (queryFilter instanceof IsBooleanFilter && ((IsBooleanFilter) queryFilter).isTrue()) { + final IsBooleanFilter isTrueFilter = (IsBooleanFilter) queryFilter; + final Filter rewritten = rewriteFilter(projectionFilter, isTrueFilter.getBaseFilter()); + if (rewritten == null) { + return null; + } + //noinspection ObjectEquality + if (rewritten == ProjectionFilterMatch.INSTANCE) { + return ProjectionFilterMatch.INSTANCE; + } + return new IsBooleanFilter(rewritten, true); + } + if (queryFilter instanceof AndFilter) { + AndFilter andFilter = (AndFilter) queryFilter; + + // if both and filters, check to see if the query and filter contains all of the clauses of the projection and filter + if (projectionFilter instanceof AndFilter) { + AndFilter projectionAndFilter = (AndFilter) projectionFilter; + Filter rewritten = andFilter; + // calling rewriteFilter using each child of the projection AND filter as the projection filter will remove + // the child from the query AND filter if it exists (or return null if it does not exist, since it must exist + // to be a valid rewrite). The remaining AND filter of will only contain children that were not part of the + // projection AND filter + for (Filter filter : projectionAndFilter.getFilters()) { + rewritten = rewriteFilter(filter, rewritten); + if (rewritten != null) { + if (rewritten == ProjectionFilterMatch.INSTANCE) { + return ProjectionFilterMatch.INSTANCE; + } + } + } + if (rewritten != null) { + return rewritten; + } + return null; + } - @Override - public void close() - { - // nothing to close + // else check to see if any clause of the query AND filter is the projection filter + List newChildren = Lists.newArrayListWithExpectedSize(andFilter.getFilters().size()); + boolean childRewritten = false; + for (Filter filter : andFilter.getFilters()) { + Filter rewritten = rewriteFilter(projectionFilter, filter); + //noinspection ObjectEquality + if (rewritten == ProjectionFilterMatch.INSTANCE) { + childRewritten = true; + } else { + if (rewritten != null) { + newChildren.add(rewritten); + childRewritten = true; + } else { + newChildren.add(filter); + } + } + } + // at least one child must have been rewritten to rewrite the AND + if (childRewritten) { + if (newChildren.size() > 1) { + return new AndFilter(newChildren); + } else { + return newChildren.get(0); + } + } + return null; } + return null; + } - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { + public static String getProjectionSmooshV9FileName(AggregateProjectionMetadata projectionSpec, String columnName) + { + return getProjectionSmooshV9Prefix(projectionSpec) + columnName; + } - } + public static String getProjectionSmooshV9Prefix(AggregateProjectionMetadata projectionSpec) + { + return projectionSpec.getSchema().getName() + "/"; + } - @Override - public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) - { - return new ConstantExprEvalSelector(ExprEval.ofLong(constant)); - } + /** + * Returns true if column is defined in {@link AggregateProjectionSpec#getGroupingColumns()} OR if the column does not + * exist in the base table. Part of determining if a projection can be used for a given {@link CursorBuildSpec}, + * + * @see #matchAggregateProjection(AggregateProjectionMetadata.Schema, CursorBuildSpec, PhysicalColumnChecker) + */ + @FunctionalInterface + public interface PhysicalColumnChecker + { + boolean check(String projectionName, String columnName); + } - @Override - public VectorValueSelector makeVectorValueSelector(ReadableVectorOffset offset) - { - return ConstantVectorSelectors.vectorValueSelector(offset, constant); - } + private Projections() + { + // no instantiation } } diff --git a/processing/src/test/java/org/apache/druid/data/input/impl/AggregateProjectionSpecTest.java b/processing/src/test/java/org/apache/druid/data/input/impl/AggregateProjectionSpecTest.java index dcb47416321b..7c43e8cf4adf 100644 --- a/processing/src/test/java/org/apache/druid/data/input/impl/AggregateProjectionSpecTest.java +++ b/processing/src/test/java/org/apache/druid/data/input/impl/AggregateProjectionSpecTest.java @@ -27,8 +27,10 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.filter.EqualityFilter; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.VirtualColumns; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -46,6 +48,7 @@ void testSerde() throws JsonProcessingException { AggregateProjectionSpec spec = new AggregateProjectionSpec( "some_projection", + new EqualityFilter("a", ColumnType.STRING, "a", null), VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, "time") ), @@ -69,6 +72,7 @@ void testComputeOrdering_noOrdering() { AggregateProjectionSpec spec = new AggregateProjectionSpec( "some_projection", + null, VirtualColumns.EMPTY, List.of(), new AggregatorFactory[] { @@ -85,6 +89,7 @@ void testMissingName() Throwable t = Assertions.assertThrows( DruidException.class, () -> new AggregateProjectionSpec( + null, null, VirtualColumns.EMPTY, List.of(new StringDimensionSchema("string")), @@ -98,6 +103,7 @@ void testMissingName() DruidException.class, () -> new AggregateProjectionSpec( "", + null, VirtualColumns.EMPTY, List.of(new StringDimensionSchema("string")), null @@ -115,6 +121,7 @@ void testInvalidGrouping() "other_projection", null, null, + null, null ) ); @@ -125,6 +132,7 @@ void testInvalidGrouping() () -> new AggregateProjectionSpec( "other_projection", null, + null, Collections.emptyList(), null ) diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java index 56800c4c3c41..df3abe0e53c3 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java @@ -26,15 +26,12 @@ import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.OrderBy; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.segment.AggregateProjectionMetadata; import org.apache.druid.segment.TestHelper; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; import org.junit.Assert; import org.junit.Test; @@ -83,24 +80,17 @@ public void testSerde() throws Exception 2, ImmutableMap.of("cnt", new CountAggregatorFactory("cnt")), ImmutableMap.of("channel_added_hourly", new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "channel_added_hourly", - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel"), - new AggregatorFactory[] { - new LongSumAggregatorFactory("sum_added", "added") - }, - ImmutableList.of( - OrderBy.ascending(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - OrderBy.ascending("channel") - ) - ), + AggregateProjectionMetadata.schemaBuilder("channel_added_hourly") + .timeColumnName(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupAndOrder(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel") + .aggregators(new LongSumAggregatorFactory("sum_added", "added")) + .build(), 16 )), new TimestampSpec(null, null, null), diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java index 69be9165ac9d..14a6b8765cb0 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java @@ -31,10 +31,8 @@ import org.apache.druid.query.CacheStrategy; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; -import org.apache.druid.query.OrderBy; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.UnionDataSource; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.DoubleMaxAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.query.aggregation.LongMaxAggregatorFactory; @@ -45,7 +43,6 @@ import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; import org.apache.druid.query.spec.LegacySegmentSpec; import org.apache.druid.segment.AggregateProjectionMetadata; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.ValueType; import org.apache.druid.timeline.LogicalSegment; @@ -70,38 +67,31 @@ public class SegmentMetadataQueryQueryToolChestTest private static final SegmentId TEST_SEGMENT_ID1 = SegmentId.of(TEST_DATASOURCE.toString(), INTERVAL_2020, "test", 0); private static final SegmentId TEST_SEGMENT_ID2 = SegmentId.of(TEST_DATASOURCE.toString(), INTERVAL_2021, "test", 0); - private static final AggregateProjectionMetadata.Schema PROJECTION_CHANNEL_ADDED_HOURLY = new AggregateProjectionMetadata.Schema( - "name1-does-not-matter", - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, - VirtualColumns.create(Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - )), - ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel"), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("channel_sum", "channel") - }, - ImmutableList.of( - OrderBy.ascending(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - OrderBy.ascending("channel") - ) - ); - private static final AggregateProjectionMetadata.Schema PROJECTION_CHANNEL_ADDED_DAILY = new AggregateProjectionMetadata.Schema( - "name2-does-not-matter", - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, - VirtualColumns.create(Granularities.toVirtualColumn( - Granularities.DAY, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - )), - ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel"), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("channel_sum", "channel") - }, - ImmutableList.of( - OrderBy.ascending(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - OrderBy.ascending("channel") - ) - ); + private static final AggregateProjectionMetadata.Schema PROJECTION_CHANNEL_ADDED_HOURLY = + AggregateProjectionMetadata.schemaBuilder("name1-does-not-matter") + .timeColumnName(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupAndOrder(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel") + .aggregators(new LongSumAggregatorFactory("channel_sum", "channel")) + .build(); + + private static final AggregateProjectionMetadata.Schema PROJECTION_CHANNEL_ADDED_DAILY = + AggregateProjectionMetadata.schemaBuilder("name2-does-not-matter") + .timeColumnName(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.DAY, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupAndOrder(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel") + .aggregators(new LongSumAggregatorFactory("channel_sum", "channel")) + .build(); @Test public void testCacheStrategy() throws Exception diff --git a/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java b/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java index 8e025600686b..fc6dade3fc3c 100644 --- a/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java +++ b/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java @@ -21,9 +21,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import it.unimi.dsi.fastutil.objects.ObjectAVLTreeSet; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.error.DruidException; @@ -32,8 +29,9 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.filter.EqualityFilter; import org.apache.druid.segment.column.ColumnHolder; -import org.apache.druid.segment.projections.Projections; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -54,6 +52,7 @@ void testSerde() throws JsonProcessingException new AggregateProjectionMetadata.Schema( "some_projection", "time", + new EqualityFilter("a", ColumnType.STRING, "a", null), VirtualColumns.create( Granularities.toVirtualColumn(Granularities.HOUR, "time") ), @@ -87,6 +86,7 @@ void testComparator() new AggregateProjectionMetadata.Schema( "good", "theTime", + null, VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "theTime")), Arrays.asList("theTime", "a", "b", "c"), new AggregatorFactory[]{new CountAggregatorFactory("chocula")}, @@ -104,6 +104,7 @@ void testComparator() new AggregateProjectionMetadata.Schema( "betterLessGroupingColumns", "theTime", + null, VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "theTime")), Arrays.asList("c", "d", "theTime"), new AggregatorFactory[]{new CountAggregatorFactory("chocula")}, @@ -120,6 +121,7 @@ void testComparator() new AggregateProjectionMetadata.Schema( "evenBetterMoreAggs", "theTime", + null, VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "theTime")), Arrays.asList("c", "d", "theTime"), new AggregatorFactory[]{ @@ -139,6 +141,7 @@ void testComparator() new AggregateProjectionMetadata.Schema( "best", null, + null, VirtualColumns.EMPTY, Arrays.asList("f", "g"), new AggregatorFactory[0], @@ -168,6 +171,7 @@ void testInvalidName() null, null, null, + null, new AggregatorFactory[]{new CountAggregatorFactory("count")}, List.of(OrderBy.ascending(ColumnHolder.TIME_COLUMN_NAME), OrderBy.ascending("count")) ), @@ -187,6 +191,7 @@ void testInvalidName() null, null, null, + null, new AggregatorFactory[]{new CountAggregatorFactory("count")}, List.of(OrderBy.ascending(ColumnHolder.TIME_COLUMN_NAME), OrderBy.ascending("count")) ), @@ -211,6 +216,7 @@ void testInvalidGrouping() null, null, null, + null, null ), 0 @@ -228,6 +234,7 @@ void testInvalidGrouping() "other_projection", null, null, + null, Collections.emptyList(), null, null @@ -252,6 +259,7 @@ void testInvalidOrdering() null, null, null, + null, new AggregatorFactory[]{new CountAggregatorFactory("count")}, null ), @@ -271,6 +279,7 @@ void testInvalidOrdering() null, null, null, + null, new AggregatorFactory[]{new CountAggregatorFactory("count")}, List.of(OrderBy.ascending(ColumnHolder.TIME_COLUMN_NAME), OrderBy.ascending("count")) ), @@ -293,44 +302,8 @@ void testEqualsAndHashcode() void testEqualsAndHashcodeSchema() { EqualsVerifier.forClass(AggregateProjectionMetadata.Schema.class) - .withIgnoredFields("orderingWithTimeSubstitution", "timeColumnPosition", "granularity") + .withIgnoredFields("orderingWithTimeSubstitution", "timeColumnPosition", "effectiveGranularity") .usingGetClass() .verify(); } - - @Test - public void testSchemaMatchSimple() - { - // arrange - AggregateProjectionMetadata spec = new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "some_projection", - null, - VirtualColumns.EMPTY, - Arrays.asList("a", "b"), - new AggregatorFactory[]{new LongSumAggregatorFactory("a_projection", "a")}, - Arrays.asList(OrderBy.ascending("a"), OrderBy.ascending("b")) - ), - 12345 - ); - CursorBuildSpec cursorBuildSpec = CursorBuildSpec.builder() - .setPreferredOrdering(ImmutableList.of()) - .setAggregators(ImmutableList.of(new LongSumAggregatorFactory( - "a", - "a" - ))) - .build(); - // act & assert - Projections.ProjectionMatch projectionMatch = spec.getSchema() - .matches(cursorBuildSpec, (projectionName, columnName) -> true); - Projections.ProjectionMatch expected = new Projections.ProjectionMatch( - CursorBuildSpec.builder() - .setAggregators(ImmutableList.of(new LongSumAggregatorFactory("a", "a"))) - .setPhysicalColumns(ImmutableSet.of("a_projection")) - .setPreferredOrdering(ImmutableList.of()) - .build(), - ImmutableMap.of("a", "a_projection") - ); - Assertions.assertEquals(expected, projectionMatch); - } } diff --git a/processing/src/test/java/org/apache/druid/segment/CursorFactoryProjectionTest.java b/processing/src/test/java/org/apache/druid/segment/CursorFactoryProjectionTest.java index aaef9cc89308..555afa1ae813 100644 --- a/processing/src/test/java/org/apache/druid/segment/CursorFactoryProjectionTest.java +++ b/processing/src/test/java/org/apache/druid/segment/CursorFactoryProjectionTest.java @@ -91,7 +91,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -170,190 +169,159 @@ public static List makeRows(List dimensions) static final List ROLLUP_ROWS = makeRows(ImmutableList.of("a", "b")); private static final List PROJECTIONS = Arrays.asList( - new AggregateProjectionSpec( - "ab_hourly_cd_sum", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList( - new StringDimensionSchema("a"), - new StringDimensionSchema("b"), - new LongDimensionSchema("__gran") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("_c_sum", "c"), - new DoubleSumAggregatorFactory("d", "d") - } - ), - new AggregateProjectionSpec( - "a_hourly_c_sum_with_count_latest", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList( - new LongDimensionSchema("__gran"), - new StringDimensionSchema("a") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("chocula"), - new LongSumAggregatorFactory("_c_sum", "c"), - new LongLastAggregatorFactory("_c_last", "c", null) - } - ), - new AggregateProjectionSpec( - "b_hourly_c_sum_non_time_ordered", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList( - new StringDimensionSchema("b"), - new LongDimensionSchema("__gran") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("chocula"), - new LongSumAggregatorFactory("_c_sum", "c"), - new LongLastAggregatorFactory("_c_last", "c", null) - } - ), - new AggregateProjectionSpec( - "bf_daily_c_sum", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.DAY, "__gran") - ), - Arrays.asList( - new LongDimensionSchema("__gran"), - new StringDimensionSchema("b"), - new FloatDimensionSchema("e") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("_c_sum", "c") - } - ), - new AggregateProjectionSpec( - "b_c_sum", - VirtualColumns.EMPTY, - List.of(new StringDimensionSchema("b")), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("_c_sum", "c") - } - ), - new AggregateProjectionSpec( - "ab", - null, - Arrays.asList( - new StringDimensionSchema("a"), - new StringDimensionSchema("b") - ), - null - ), - new AggregateProjectionSpec( - "abfoo", - VirtualColumns.create( - new ExpressionVirtualColumn( - "bfoo", - "concat(b, 'foo')", - ColumnType.STRING, - TestExprMacroTable.INSTANCE - ) - ), - Arrays.asList( - new StringDimensionSchema("a"), - new StringDimensionSchema("bfoo") - ), - null - ), - new AggregateProjectionSpec( - "c_sum_daily", - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.DAY, "__gran")), - Collections.singletonList(new LongDimensionSchema("__gran")), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("_c_sum", "c") - } - ), - new AggregateProjectionSpec( - "c_sum", - VirtualColumns.EMPTY, - Collections.emptyList(), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("_c_sum", "c") - } - ), - new AggregateProjectionSpec( - "missing_column", - VirtualColumns.EMPTY, - List.of(new StringDimensionSchema("missing")), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("csum", "c") - } - ), - new AggregateProjectionSpec( - "json", - VirtualColumns.EMPTY, - List.of(new AutoTypeColumnSchema("f", null)), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("_c_sum", "c") - } - ) + AggregateProjectionSpec.builder("ab_hourly_cd_sum") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupingColumns( + new StringDimensionSchema("a"), + new StringDimensionSchema("b"), + new LongDimensionSchema("__gran") + ) + .aggregators( + new LongSumAggregatorFactory("_c_sum", "c"), + new DoubleSumAggregatorFactory("d", "d") + ) + .build(), + AggregateProjectionSpec.builder("a_hourly_c_sum_with_count_latest") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupingColumns( + new LongDimensionSchema("__gran"), + new StringDimensionSchema("a") + ) + .aggregators( + new CountAggregatorFactory("chocula"), + new LongSumAggregatorFactory("_c_sum", "c"), + new LongLastAggregatorFactory("_c_last", "c", null) + ) + .build(), + AggregateProjectionSpec.builder("b_hourly_c_sum_non_time_ordered") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupingColumns( + new StringDimensionSchema("b"), + new LongDimensionSchema("__gran") + ) + .aggregators( + new CountAggregatorFactory("chocula"), + new LongSumAggregatorFactory("_c_sum", "c"), + new LongLastAggregatorFactory("_c_last", "c", null) + ) + .build(), + AggregateProjectionSpec.builder("bf_daily_c_sum") + .virtualColumns(Granularities.toVirtualColumn(Granularities.DAY, "__gran")) + .groupingColumns( + new LongDimensionSchema("__gran"), + new StringDimensionSchema("b"), + new FloatDimensionSchema("e") + ) + .aggregators(new LongSumAggregatorFactory("_c_sum", "c")) + .build(), + AggregateProjectionSpec.builder("b_c_sum") + .groupingColumns(new StringDimensionSchema("b")) + .aggregators(new LongSumAggregatorFactory("_c_sum", "c")) + .build(), + AggregateProjectionSpec.builder("ab") + .groupingColumns( + new StringDimensionSchema("a"), + new StringDimensionSchema("b") + ) + .build(), + AggregateProjectionSpec.builder("abfoo") + .virtualColumns( + new ExpressionVirtualColumn( + "bfoo", + "concat(b, 'foo')", + ColumnType.STRING, + TestExprMacroTable.INSTANCE + ) + ) + .groupingColumns( + new StringDimensionSchema("a"), + new StringDimensionSchema("bfoo") + ) + .build(), + AggregateProjectionSpec.builder("c_sum_daily") + .virtualColumns(Granularities.toVirtualColumn(Granularities.DAY, "__gran")) + .groupingColumns(new LongDimensionSchema("__gran")) + .aggregators(new LongSumAggregatorFactory("_c_sum", "c")) + .build(), + AggregateProjectionSpec.builder("c_sum") + .aggregators(new LongSumAggregatorFactory("_c_sum", "c")) + .build(), + AggregateProjectionSpec.builder("missing_column") + .groupingColumns(new StringDimensionSchema("missing")) + .aggregators(new LongSumAggregatorFactory("csum", "c")) + .build(), + AggregateProjectionSpec.builder("json") + .groupingColumns(new AutoTypeColumnSchema("f", null)) + .aggregators(new LongSumAggregatorFactory("_c_sum", "c")) + .build(), + AggregateProjectionSpec.builder("a_filter_b_aaonly_hourly_cd_sum") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .filter(new EqualityFilter("b", ColumnType.STRING, "aa", null)) + .groupingColumns( + new StringDimensionSchema("a"), + new LongDimensionSchema("__gran") + ) + .aggregators( + new LongSumAggregatorFactory("_c_sum", "c"), + new DoubleSumAggregatorFactory("d", "d") + ) + .build() ); private static final List ROLLUP_PROJECTIONS = Arrays.asList( - new AggregateProjectionSpec( - "a_hourly_c_sum_with_count", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList( - new LongDimensionSchema("__gran"), - new StringDimensionSchema("a") - ), - new AggregatorFactory[]{ + AggregateProjectionSpec.builder("a_hourly_c_sum_with_count") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupingColumns( + new LongDimensionSchema("__gran"), + new StringDimensionSchema("a") + ) + .aggregators( new CountAggregatorFactory("chocula"), new LongSumAggregatorFactory("sum_c", "sum_c") - } - ), - new AggregateProjectionSpec( - "afoo", - VirtualColumns.create( - new ExpressionVirtualColumn( - "afoo", - "concat(a, 'foo')", - ColumnType.STRING, - TestExprMacroTable.INSTANCE - ) - ), - List.of( - new StringDimensionSchema("afoo") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_c", "sum_c") - } - ) + ) + .build(), + AggregateProjectionSpec.builder("afoo") + .virtualColumns( + new ExpressionVirtualColumn( + "afoo", + "concat(a, 'foo')", + ColumnType.STRING, + TestExprMacroTable.INSTANCE + ) + ) + .groupingColumns(new StringDimensionSchema("afoo")) + .aggregators(new LongSumAggregatorFactory("sum_c", "sum_c")) + .build() ); private static final List AUTO_PROJECTIONS = PROJECTIONS.stream() - .map(projection -> new AggregateProjectionSpec( - projection.getName(), - projection.getVirtualColumns(), - projection.getGroupingColumns() - .stream() - .map(x -> new AutoTypeColumnSchema(x.getName(), null)) - .collect(Collectors.toList()), - projection.getAggregators() - )) + .map( + projection -> + AggregateProjectionSpec.builder(projection) + .groupingColumns( + projection.getGroupingColumns() + .stream() + .map(x -> new AutoTypeColumnSchema(x.getName(), null)) + .collect(Collectors.toList()) + ) + .build() + ) .collect(Collectors.toList()); private static final List AUTO_ROLLUP_PROJECTIONS = ROLLUP_PROJECTIONS.stream() - .map(projection -> new AggregateProjectionSpec( - projection.getName(), - projection.getVirtualColumns(), - projection.getGroupingColumns() - .stream() - .map(x -> new AutoTypeColumnSchema(x.getName(), null)) - .collect(Collectors.toList()), - projection.getAggregators() - )) + .map( + projection -> + AggregateProjectionSpec.builder(projection) + .groupingColumns( + projection.getGroupingColumns() + .stream() + .map(x -> new AutoTypeColumnSchema(x.getName(), null)) + .collect(Collectors.toList()) + ) + .build() + ) .collect(Collectors.toList()); @Parameterized.Parameters(name = "name: {0}, segmentTimeOrdered: {5}, autoSchema: {6}") @@ -1769,6 +1737,46 @@ public void testProjectionJson() Assert.assertArrayEquals(new Object[]{"b", 12L}, resultsNoProjection.get(1).getArray()); } + + @Test + public void testProjectionFilter() + { + // since d isn't present on the smaller projection, cant use it, but can still use the larger projection + final GroupByQuery query = + GroupByQuery.builder() + .setDataSource("test") + .setGranularity(Granularities.ALL) + .setInterval(Intervals.ETERNITY) + .addDimension("a") + .setDimFilter(new EqualityFilter("b", ColumnType.STRING, "aa", null)) + .addAggregator(new LongSumAggregatorFactory("c_sum", "c")) + .addAggregator(new DoubleSumAggregatorFactory("d_sum", "d")) + .build(); + final CursorBuildSpec buildSpec = GroupingEngine.makeCursorBuildSpec(query, null); + try (final CursorHolder cursorHolder = projectionsCursorFactory.makeCursorHolder(buildSpec)) { + final Cursor cursor = cursorHolder.asCursor(); + int rowCount = 0; + while (!cursor.isDone()) { + rowCount++; + cursor.advance(); + } + Assert.assertEquals(3, rowCount); + } + + final Sequence resultRows = groupingEngine.process( + query, + projectionsCursorFactory, + projectionsTimeBoundaryInspector, + nonBlockingPool, + null + ); + + final List results = resultRows.toList(); + Assert.assertEquals(2, results.size()); + Assert.assertArrayEquals(new Object[]{"a", 2L, 2.1}, results.get(0).getArray()); + Assert.assertArrayEquals(new Object[]{"b", 7L, 7.7}, results.get(1).getArray()); + } + private static IndexBuilder makeBuilder(DimensionsSpec dimensionsSpec, boolean autoSchema, boolean writeNullColumns) { File tmp = FileUtils.createTempDir(); diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java index 87959a63b6b2..b6a1dfa03909 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerTestBase.java @@ -3011,29 +3011,18 @@ public void testMergeProjections() throws IOException ); List projections = Arrays.asList( - new AggregateProjectionSpec( - "a_hourly_c_sum", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList( - new StringDimensionSchema("a"), - new LongDimensionSchema("__gran") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("c_sum", "c") - } - ), - new AggregateProjectionSpec( - "a_c_sum", - VirtualColumns.EMPTY, - Collections.singletonList( - new StringDimensionSchema("a") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("c_sum", "c") - } - ) + AggregateProjectionSpec.builder("a_hourly_c_sum") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupingColumns( + new StringDimensionSchema("a"), + new LongDimensionSchema("__gran") + ) + .aggregators(new LongSumAggregatorFactory("c_sum", "c")) + .build(), + AggregateProjectionSpec.builder("a_c_sum") + .groupingColumns(new StringDimensionSchema("a")) + .aggregators(new LongSumAggregatorFactory("c_sum", "c")) + .build() ); IndexBuilder bob = IndexBuilder.create() diff --git a/processing/src/test/java/org/apache/druid/segment/MetadataTest.java b/processing/src/test/java/org/apache/druid/segment/MetadataTest.java index 23318406bb47..b889d58eca3d 100644 --- a/processing/src/test/java/org/apache/druid/segment/MetadataTest.java +++ b/processing/src/test/java/org/apache/druid/segment/MetadataTest.java @@ -99,20 +99,15 @@ public void testMerge() AggregatorFactory[] aggs = new AggregatorFactory[]{ new LongMaxAggregatorFactory("n", "f") }; - List projectionSpecs = ImmutableList.of( + List projectionSpecs = List.of( new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "some_projection", - "__gran", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList("a", "b", "__gran"), - new AggregatorFactory[]{ - new LongLastAggregatorFactory("atLongLast", "d", null) - }, - makeOrderBy("a", "b", "__gran") - ), + AggregateProjectionMetadata.schemaBuilder("some_projection") + .timeColumnName("__gran") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupAndOrder("a", "b", "__gran") + .aggregators(new LongLastAggregatorFactory("atLongLast", "d", null)) + .ordering("a", "b", "__gran") + .build(), 1234 ) ); @@ -279,69 +274,49 @@ public void testMergeProjectionsUnexpectedMismatch() { List p1 = ImmutableList.of( new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "some_projection", - "__gran", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList("a", "b", "__gran"), - new AggregatorFactory[]{ - new LongLastAggregatorFactory("atLongLast", "d", null) - }, - makeOrderBy("a", "b", "__gran") - ), + AggregateProjectionMetadata.schemaBuilder("some_projection") + .timeColumnName("__gran") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupAndOrder("a", "b", "__gran") + .aggregators(new LongLastAggregatorFactory("atLongLast", "d", null)) + .ordering("a", "b", "__gran") + .build(), 654321 ) ); List p2 = ImmutableList.of( new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "some_projection", - "__gran", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList("a", "b", "_gran"), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("longSum", "d") - }, - makeOrderBy("a", "b", "__gran") - ), + AggregateProjectionMetadata.schemaBuilder("some_projection") + .timeColumnName("__gran") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupAndOrder("a", "b", "_gran") + .aggregators(new LongSumAggregatorFactory("longSum", "d")) + .ordering("a", "b", "_gran") + .build(), 1234 ) ); List p3 = ImmutableList.of( new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "some_projection", - "__gran", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, "__gran") - ), - Arrays.asList("a", "b", "__gran"), - new AggregatorFactory[]{ - new LongLastAggregatorFactory("atLongLast", "d", null) - }, - makeOrderBy("a", "b", "__gran") - ), + AggregateProjectionMetadata.schemaBuilder("some_projection") + .timeColumnName("__gran") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "__gran")) + .groupAndOrder("a", "b", "__gran") + .aggregators(new LongLastAggregatorFactory("atLongLast", "d", null)) + .ordering("a", "b", "__gran") + .build(), 12121 ), new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "some_projection2", - "__gran", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.DAY, "__gran") - ), - Arrays.asList("__gran", "a"), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("longSum", "d") - }, - makeOrderBy("__gran", "a") - ), + AggregateProjectionMetadata.schemaBuilder("some_projection2") + .timeColumnName("__gran") + .virtualColumns(Granularities.toVirtualColumn(Granularities.DAY, "__gran")) + .groupAndOrder("__gran", "a") + .aggregators(new LongSumAggregatorFactory("longSum", "d")) + .ordering("__gran", "a") + .build(), 555 ) ); diff --git a/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorHolderTest.java b/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorHolderTest.java index d0be594844dd..2df7bb0a2b05 100644 --- a/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorHolderTest.java +++ b/processing/src/test/java/org/apache/druid/segment/QueryableIndexCursorHolderTest.java @@ -35,7 +35,6 @@ import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.query.QueryContext; import org.apache.druid.query.QueryContexts; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; @@ -173,40 +172,39 @@ public void testProjectionTimeBoundaryInspector() Arrays.asList("a", "bb", 1L, 1.1, 1.1f) ) ); - IndexBuilder bob = IndexBuilder.create() - .tmpDir(tmp) - .schema( - IncrementalIndexSchema.builder() - .withDimensionsSpec(dims) - .withRollup(false) - .withMinTimestamp(startTime.getMillis()) - .withProjections( - Collections.singletonList( - new AggregateProjectionSpec( - "ab_hourly_cd_sum_time_ordered", - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - "__gran" - ) - ), - Arrays.asList( - new LongDimensionSchema("__gran"), - new StringDimensionSchema("a"), - new StringDimensionSchema("b") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory( - "_c_sum", - "c" - ), - new DoubleSumAggregatorFactory("d", "d") - } - ) + IncrementalIndexSchema indexSchema = + IncrementalIndexSchema.builder() + .withDimensionsSpec(dims) + .withRollup(false) + .withMinTimestamp(startTime.getMillis()) + .withProjections( + Collections.singletonList( + AggregateProjectionSpec.builder("ab_hourly_cd_sum_time_ordered") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + "__gran" ) ) + .groupingColumns( + new LongDimensionSchema("__gran"), + new StringDimensionSchema("a"), + new StringDimensionSchema("b") + ) + .aggregators( + new LongSumAggregatorFactory( + "_c_sum", + "c" + ), + new DoubleSumAggregatorFactory("d", "d") + ) .build() - ) + ) + ) + .build(); + IndexBuilder bob = IndexBuilder.create() + .tmpDir(tmp) + .schema(indexSchema) .rows(rows); try (QueryableIndex index = bob.buildMMappedIndex()) { diff --git a/processing/src/test/java/org/apache/druid/segment/TestIndex.java b/processing/src/test/java/org/apache/druid/segment/TestIndex.java index 712d5d950027..3e84fc0a5b9d 100644 --- a/processing/src/test/java/org/apache/druid/segment/TestIndex.java +++ b/processing/src/test/java/org/apache/druid/segment/TestIndex.java @@ -185,23 +185,24 @@ public class TestIndex new DoubleMaxAggregatorFactory(DOUBLE_METRICS[2], VIRTUAL_COLUMNS.getVirtualColumns()[0].getOutputName()), new HyperUniquesAggregatorFactory("quality_uniques", "quality") }; - public static final ImmutableList PROJECTIONS = ImmutableList.of( - new AggregateProjectionSpec( - "daily_market_maxQuality", - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.DAY, "__gran")), - List.of(new LongDimensionSchema("__gran"), new StringDimensionSchema("market")), - new AggregatorFactory[]{new LongMaxAggregatorFactory("maxQuality", "qualityLong")} - ), - new AggregateProjectionSpec( - "daily_countAndQualityCardinalityAndMaxLongNullable", - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.DAY, "__gran")), - List.of(new LongDimensionSchema("__gran")), - new AggregatorFactory[]{ - new CountAggregatorFactory("count"), - QUALITY_CARDINALITY, - new LongMaxAggregatorFactory("longNullableMax", "longNumericNull") - } - ) + public static final List PROJECTIONS = List.of( + AggregateProjectionSpec.builder("daily_market_maxQuality") + .virtualColumns(Granularities.toVirtualColumn(Granularities.DAY, "__gran")) + .groupingColumns( + new LongDimensionSchema("__gran"), + new StringDimensionSchema("market") + ) + .aggregators(new LongMaxAggregatorFactory("maxQuality", "qualityLong")) + .build(), + AggregateProjectionSpec.builder("daily_countAndQualityCardinalityAndMaxLongNullable") + .virtualColumns(Granularities.toVirtualColumn(Granularities.DAY, "__gran")) + .groupingColumns(new LongDimensionSchema("__gran")) + .aggregators( + new CountAggregatorFactory("count"), + QUALITY_CARDINALITY, + new LongMaxAggregatorFactory("longNullableMax", "longNumericNull") + ) + .build() ); public static final IndexSpec INDEX_SPEC = IndexSpec.DEFAULT; diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/OnheapIncrementalIndexTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/OnheapIncrementalIndexTest.java index 3ec43db9054c..fb4881c8517c 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/OnheapIncrementalIndexTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/OnheapIncrementalIndexTest.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableList; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.AggregateProjectionSpec; @@ -38,7 +37,6 @@ import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.TestHelper; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.joda.time.DateTime; @@ -63,30 +61,28 @@ public void testSpecSerde() throws JsonProcessingException @Test public void testProjectionHappyPath() { - // arrange DimensionsSpec dimensionsSpec = DimensionsSpec.builder() - .setDimensions(ImmutableList.of( + .setDimensions(List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") )) .build(); AggregatorFactory aggregatorFactory = new DoubleSumAggregatorFactory("double", "double"); - AggregateProjectionSpec projectionSpec = new AggregateProjectionSpec( - "proj", - VirtualColumns.EMPTY, - ImmutableList.of(new StringDimensionSchema("string")), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long"), - new DoubleSumAggregatorFactory("double", "double") - } - ); - // act & assert + AggregateProjectionSpec projectionSpec = + AggregateProjectionSpec.builder("proj") + .groupingColumns(new StringDimensionSchema("string")) + .aggregators( + new LongSumAggregatorFactory("sum_long", "long"), + new DoubleSumAggregatorFactory("double", "double") + ) + .build(); + IncrementalIndex index = IndexBuilder.create() .schema(IncrementalIndexSchema.builder() .withDimensionsSpec(dimensionsSpec) .withRollup(true) .withMetrics(aggregatorFactory) - .withProjections(ImmutableList.of(projectionSpec)) + .withProjections(List.of(projectionSpec)) .build()) .buildIncrementalIndex(); Assert.assertNotNull(index.getProjection("proj")); @@ -98,18 +94,7 @@ public void testProjectionDuplicatedName() // arrange DimensionsSpec dimensionsSpec = DimensionsSpec.EMPTY; AggregatorFactory aggregatorFactory = new DoubleSumAggregatorFactory("double", "double"); - AggregateProjectionSpec projectionSpec1 = new AggregateProjectionSpec( - "proj", - VirtualColumns.EMPTY, - ImmutableList.of(), - new AggregatorFactory[]{new DoubleSumAggregatorFactory("double", "double")} - ); - AggregateProjectionSpec projectionSpec2 = new AggregateProjectionSpec( - "proj", - VirtualColumns.EMPTY, - ImmutableList.of(), - new AggregatorFactory[]{new DoubleSumAggregatorFactory("double", "double")} - ); + AggregateProjectionSpec.Builder bob = new AggregateProjectionSpec.Builder().aggregators(aggregatorFactory); // act & assert DruidException e = Assert.assertThrows( DruidException.class, @@ -118,10 +103,12 @@ public void testProjectionDuplicatedName() .withDimensionsSpec(dimensionsSpec) .withRollup(true) .withMetrics(aggregatorFactory) - .withProjections(ImmutableList.of( - projectionSpec1, - projectionSpec2 - )) + .withProjections( + List.of( + bob.name("proj").build(), + bob.name("proj").build() + ) + ) .build()) .buildIncrementalIndex() ); @@ -149,7 +136,7 @@ public void testBadProjectionMismatchedDimensionTypes() .withDimensionsSpec( DimensionsSpec.builder() .setDimensions( - ImmutableList.of( + List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") ) @@ -157,15 +144,10 @@ public void testBadProjectionMismatchedDimensionTypes() .build() ) .withProjections( - ImmutableList.of( - new AggregateProjectionSpec( - "mismatched dims", - VirtualColumns.EMPTY, - ImmutableList.of( - new LongDimensionSchema("string") - ), - null - ) + List.of( + AggregateProjectionSpec.builder("mismatched dims") + .groupingColumns(new LongDimensionSchema("string")) + .build() ) ) .build() @@ -189,7 +171,7 @@ public void testBadProjectionDimensionNoVirtualColumnOrBaseTable() .withDimensionsSpec( DimensionsSpec.builder() .setDimensions( - ImmutableList.of( + List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") ) @@ -197,23 +179,21 @@ public void testBadProjectionDimensionNoVirtualColumnOrBaseTable() .build() ) .withProjections( - ImmutableList.of( - new AggregateProjectionSpec( - "sad grouping column", - VirtualColumns.create( - new ExpressionVirtualColumn( - "v0", - "cast(long, 'double')", - ColumnType.DOUBLE, - TestExprMacroTable.INSTANCE - ) - ), - ImmutableList.of( - new DoubleDimensionSchema("v0"), - new StringDimensionSchema("missing") - ), - null - ) + List.of( + AggregateProjectionSpec.builder("sad grouping column") + .virtualColumns( + new ExpressionVirtualColumn( + "v0", + "cast(long, 'double')", + ColumnType.DOUBLE, + TestExprMacroTable.INSTANCE + ) + ) + .groupingColumns( + new DoubleDimensionSchema("v0"), + new StringDimensionSchema("missing") + ) + .build() ) ) .build() @@ -237,7 +217,7 @@ public void testBadProjectionVirtualColumnNoDimension() .withDimensionsSpec( DimensionsSpec.builder() .setDimensions( - ImmutableList.of( + List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") ) @@ -245,22 +225,20 @@ public void testBadProjectionVirtualColumnNoDimension() .build() ) .withProjections( - ImmutableList.of( - new AggregateProjectionSpec( - "sad virtual column", - VirtualColumns.create( - new ExpressionVirtualColumn( - "v0", - "double", - ColumnType.DOUBLE, - TestExprMacroTable.INSTANCE - ) - ), - ImmutableList.of( - new LongDimensionSchema("long") - ), - null - ) + List.of( + AggregateProjectionSpec.builder("sad virtual column") + .virtualColumns( + new ExpressionVirtualColumn( + "v0", + "double", + ColumnType.DOUBLE, + TestExprMacroTable.INSTANCE + ) + ) + .groupingColumns( + new LongDimensionSchema("long") + ) + .build() ) ) .build() @@ -284,7 +262,7 @@ public void testBadProjectionRollupMismatchedAggType() .withDimensionsSpec( DimensionsSpec.builder() .setDimensions( - ImmutableList.of( + List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") ) @@ -296,20 +274,17 @@ public void testBadProjectionRollupMismatchedAggType() new DoubleSumAggregatorFactory("sum_double", "sum_double") ) .withProjections( - ImmutableList.of( - new AggregateProjectionSpec( - "mismatched agg", - VirtualColumns.EMPTY, - ImmutableList.of( - new StringDimensionSchema("string") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory( - "sum_double", - "sum_double" - ) - } - ) + List.of( + AggregateProjectionSpec.builder("mismatched agg") + .groupingColumns(new StringDimensionSchema( + "string")) + .aggregators( + new LongSumAggregatorFactory( + "sum_double", + "sum_double" + ) + ) + .build() ) ) .build() @@ -333,7 +308,7 @@ public void testBadProjectionRollupBadAggInput() .withDimensionsSpec( DimensionsSpec.builder() .setDimensions( - ImmutableList.of( + List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") ) @@ -345,18 +320,21 @@ public void testBadProjectionRollupBadAggInput() new DoubleSumAggregatorFactory("double", "double") ) .withProjections( - ImmutableList.of( - new AggregateProjectionSpec( - "renamed agg", - VirtualColumns.EMPTY, - ImmutableList.of( - new StringDimensionSchema("string") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long"), - new DoubleSumAggregatorFactory("sum_double", "double") - } - ) + List.of( + AggregateProjectionSpec.builder("renamed agg") + .groupingColumns(new StringDimensionSchema( + "string")) + .aggregators( + new LongSumAggregatorFactory( + "sum_long", + "long" + ), + new DoubleSumAggregatorFactory( + "sum_double", + "double" + ) + ) + .build() ) ) .build() @@ -380,7 +358,7 @@ public void testBadProjectionVirtualColumnAggInput() .withDimensionsSpec( DimensionsSpec.builder() .setDimensions( - ImmutableList.of( + List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") ) @@ -388,24 +366,26 @@ public void testBadProjectionVirtualColumnAggInput() .build() ) .withProjections( - ImmutableList.of( - new AggregateProjectionSpec( - "sad agg virtual column", - VirtualColumns.create( - new ExpressionVirtualColumn( - "v0", - "long + 100", - ColumnType.LONG, - TestExprMacroTable.INSTANCE - ) - ), - ImmutableList.of( - new LongDimensionSchema("long") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("v0_sum", "v0") - } - ) + List.of( + AggregateProjectionSpec.builder("sad agg virtual column") + .virtualColumns( + new ExpressionVirtualColumn( + "v0", + "long + 100", + ColumnType.LONG, + TestExprMacroTable.INSTANCE + ) + ) + .groupingColumns( + new LongDimensionSchema("long") + ) + .aggregators( + new LongSumAggregatorFactory( + "v0_sum", + "v0" + ) + ) + .build() ) ) .build() @@ -423,21 +403,20 @@ public void testTimestampOutOfRange() { // arrange DimensionsSpec dimensionsSpec = DimensionsSpec.builder() - .setDimensions(ImmutableList.of( + .setDimensions(List.of( new StringDimensionSchema("string"), new LongDimensionSchema("long") )) .build(); AggregatorFactory aggregatorFactory = new DoubleSumAggregatorFactory("double", "double"); - AggregateProjectionSpec projectionSpec = new AggregateProjectionSpec( - "proj", - VirtualColumns.EMPTY, - ImmutableList.of(new StringDimensionSchema("string")), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long"), - new DoubleSumAggregatorFactory("double", "double") - } - ); + AggregateProjectionSpec projectionSpec = + AggregateProjectionSpec.builder("proj") + .groupingColumns(new StringDimensionSchema("string")) + .aggregators( + new LongSumAggregatorFactory("sum_long", "long"), + new DoubleSumAggregatorFactory("double", "double") + ) + .build(); final DateTime minTimestamp = DateTimes.nowUtc(); final DateTime outOfRangeTimestamp = DateTimes.nowUtc().minusDays(1); @@ -489,17 +468,15 @@ public void testTimestampOutOfRange() t.getMessage() ); - AggregateProjectionSpec projectionSpecYear = new AggregateProjectionSpec( - "proj", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.YEAR, "g") - ), - ImmutableList.of(new StringDimensionSchema("string"), new LongDimensionSchema("g")), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long"), - new DoubleSumAggregatorFactory("double", "double") - } - ); + AggregateProjectionSpec projectionSpecYear = + AggregateProjectionSpec.builder("proj") + .virtualColumns(Granularities.toVirtualColumn(Granularities.YEAR, "g")) + .groupingColumns(new StringDimensionSchema("string"), new LongDimensionSchema("g")) + .aggregators( + new LongSumAggregatorFactory("sum_long", "long"), + new DoubleSumAggregatorFactory("double", "double") + ) + .build(); IncrementalIndex index2 = IndexBuilder.create() .schema(IncrementalIndexSchema.builder() .withDimensionsSpec(dimensionsSpec) diff --git a/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java b/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java new file mode 100644 index 000000000000..3c13a0571bf2 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.projections; + +import org.apache.druid.data.input.impl.AggregateProjectionSpec; +import org.apache.druid.data.input.impl.LongDimensionSchema; +import org.apache.druid.data.input.impl.StringDimensionSchema; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.filter.EqualityFilter; +import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.LikeDimFilter; +import org.apache.druid.segment.AggregateProjectionMetadata; +import org.apache.druid.segment.CursorBuildSpec; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.filter.AndFilter; +import org.apache.druid.segment.filter.IsBooleanFilter; +import org.apache.druid.segment.filter.OrFilter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +class ProjectionsTest +{ + @Test + void testSchemaMatchSimple() + { + RowSignature baseTable = RowSignature.builder() + .addTimeColumn() + .add("a", ColumnType.LONG) + .add("b", ColumnType.STRING) + .add("c", ColumnType.LONG) + .build(); + AggregateProjectionMetadata spec = new AggregateProjectionMetadata( + AggregateProjectionSpec.builder("some_projection") + .groupingColumns(new LongDimensionSchema("a"), new StringDimensionSchema("b")) + .aggregators(new LongSumAggregatorFactory("c_sum", "c")) + .build() + .toMetadataSchema(), + 12345 + ); + CursorBuildSpec cursorBuildSpec = CursorBuildSpec.builder() + .setPreferredOrdering(List.of()) + .setAggregators( + List.of( + new LongSumAggregatorFactory("c", "c") + ) + ) + .build(); + + ProjectionMatch projectionMatch = Projections.matchAggregateProjection( + spec.getSchema(), + cursorBuildSpec, + new RowSignatureChecker(baseTable) + ); + ProjectionMatch expected = new ProjectionMatch( + CursorBuildSpec.builder() + .setAggregators(List.of(new LongSumAggregatorFactory("c", "c"))) + .setPhysicalColumns(Set.of("c_sum")) + .setPreferredOrdering(List.of()) + .build(), + Map.of("c", "c_sum") + ); + Assertions.assertEquals(expected, projectionMatch); + } + + @Test + void testSchemaMatchFilter() + { + RowSignature baseTable = RowSignature.builder() + .addTimeColumn() + .add("a", ColumnType.LONG) + .add("b", ColumnType.STRING) + .add("c", ColumnType.LONG) + .build(); + AggregateProjectionMetadata spec = new AggregateProjectionMetadata( + AggregateProjectionSpec.builder("some_projection") + .filter(new EqualityFilter("b", ColumnType.STRING, "foo", null)) + .groupingColumns(new LongDimensionSchema("a")) + .aggregators(new LongSumAggregatorFactory("c_sum", "c")) + .build() + .toMetadataSchema(), + 12345 + ); + CursorBuildSpec cursorBuildSpecNoFilter = CursorBuildSpec.builder() + .setPreferredOrdering(List.of()) + .setAggregators( + List.of( + new LongSumAggregatorFactory("c", "c") + ) + ) + .build(); + + Assertions.assertNull( + Projections.matchAggregateProjection( + spec.getSchema(), + cursorBuildSpecNoFilter, + new RowSignatureChecker(baseTable) + ) + ); + CursorBuildSpec cursorBuildSpecWithFilter = CursorBuildSpec.builder() + .setPreferredOrdering(List.of()) + .setFilter( + new EqualityFilter( + "b", + ColumnType.STRING, + "foo", + null + ) + ) + .setAggregators( + List.of( + new LongSumAggregatorFactory("c", "c") + ) + ) + .build(); + ProjectionMatch projectionMatch = Projections.matchAggregateProjection( + spec.getSchema(), + cursorBuildSpecWithFilter, + new RowSignatureChecker(baseTable) + ); + ProjectionMatch expected = new ProjectionMatch( + CursorBuildSpec.builder() + .setAggregators(List.of(new LongSumAggregatorFactory("c", "c"))) + .setPhysicalColumns(Set.of("c_sum")) + .setPreferredOrdering(List.of()) + .build(), + Map.of("c", "c_sum") + ); + Assertions.assertEquals(expected, projectionMatch); + } + + @Test + void testSchemaMatchFilterIncludedInProjection() + { + RowSignature baseTable = RowSignature.builder() + .addTimeColumn() + .add("a", ColumnType.LONG) + .add("b", ColumnType.STRING) + .add("c", ColumnType.LONG) + .build(); + AggregateProjectionMetadata spec = new AggregateProjectionMetadata( + AggregateProjectionSpec.builder("some_projection") + .filter(new LikeDimFilter("b", "foo%", null, null)) + .groupingColumns(new LongDimensionSchema("a"), new StringDimensionSchema("b")) + .aggregators(new LongSumAggregatorFactory("c_sum", "c")) + .build() + .toMetadataSchema(), + 12345 + ); + CursorBuildSpec cursorBuildSpecNoFilter = CursorBuildSpec.builder() + .setPreferredOrdering(List.of()) + .setGroupingColumns(List.of("a", "b")) + .setAggregators( + List.of( + new LongSumAggregatorFactory("c", "c") + ) + ) + .build(); + + Assertions.assertNull( + Projections.matchAggregateProjection( + spec.getSchema(), + cursorBuildSpecNoFilter, + new RowSignatureChecker(baseTable) + ) + ); + CursorBuildSpec cursorBuildSpecWithFilter = CursorBuildSpec.builder() + .setGroupingColumns(List.of("a", "b")) + .setPreferredOrdering(List.of()) + .setFilter( + new LikeDimFilter( + "b", + "foo%", + null, + null + ).toFilter() + ) + .setAggregators( + List.of( + new LongSumAggregatorFactory("c", "c") + ) + ) + .build(); + ProjectionMatch projectionMatch = Projections.matchAggregateProjection( + spec.getSchema(), + cursorBuildSpecWithFilter, + new RowSignatureChecker(baseTable) + ); + ProjectionMatch expected = new ProjectionMatch( + CursorBuildSpec.builder() + .setGroupingColumns(List.of("a", "b")) + .setAggregators(List.of(new LongSumAggregatorFactory("c", "c"))) + .setPhysicalColumns(Set.of("a", "b", "c_sum")) + .setPreferredOrdering(List.of()) + .build(), + Map.of("c", "c_sum") + ); + Assertions.assertEquals(expected, projectionMatch); + } + + @Test + void testRewriteFilter() + { + Filter xeqfoo = new EqualityFilter("x", ColumnType.STRING, "foo", null); + Filter xeqfoo2 = new EqualityFilter("x", ColumnType.STRING, "foo", null); + Filter xeqbar = new EqualityFilter("x", ColumnType.STRING, "bar", null); + Filter yeqbar = new EqualityFilter("y", ColumnType.STRING, "bar", null); + Filter zeq123 = new EqualityFilter("z", ColumnType.LONG, 123L, null); + + Filter queryFilter = xeqfoo2; + Assertions.assertInstanceOf( + ProjectionFilterMatch.class, + Projections.rewriteFilter(xeqfoo, queryFilter) + ); + + queryFilter = yeqbar; + Assertions.assertNull(Projections.rewriteFilter(xeqfoo, queryFilter)); + + queryFilter = new AndFilter(List.of(xeqfoo, yeqbar)); + Assertions.assertEquals( + yeqbar, + Projections.rewriteFilter(xeqfoo, queryFilter) + ); + + queryFilter = new AndFilter(List.of(new OrFilter(List.of(xeqfoo, xeqbar)), yeqbar)); + Assertions.assertNull(Projections.rewriteFilter(xeqfoo, queryFilter)); + + queryFilter = new AndFilter(List.of(new IsBooleanFilter(xeqfoo, true), yeqbar)); + Assertions.assertEquals(yeqbar, Projections.rewriteFilter(xeqfoo, queryFilter)); + + queryFilter = new AndFilter(List.of(new IsBooleanFilter(xeqfoo, false), yeqbar)); + Assertions.assertNull(Projections.rewriteFilter(xeqfoo, queryFilter)); + + queryFilter = new AndFilter(List.of(new AndFilter(List.of(xeqfoo, yeqbar)), zeq123)); + Assertions.assertEquals( + new AndFilter(List.of(yeqbar, zeq123)), + Projections.rewriteFilter(xeqfoo, queryFilter) + ); + + queryFilter = new AndFilter( + List.of( + new EqualityFilter("a", ColumnType.STRING, "foo", null), + new EqualityFilter("b", ColumnType.STRING, "bar", null), + new EqualityFilter("c", ColumnType.STRING, "baz", null) + ) + ); + Assertions.assertEquals( + new EqualityFilter("b", ColumnType.STRING, "bar", null), + Projections.rewriteFilter( + new AndFilter( + List.of( + new EqualityFilter("a", ColumnType.STRING, "foo", null), + new EqualityFilter("c", ColumnType.STRING, "baz", null) + ) + ), + queryFilter + ) + ); + } + + private static class RowSignatureChecker implements Projections.PhysicalColumnChecker + { + private final RowSignature rowSignature; + + private RowSignatureChecker(RowSignature rowSignature) + { + this.rowSignature = rowSignature; + } + + @Override + public boolean check(String projectionName, String columnName) + { + return rowSignature.contains(columnName); + } + } +} diff --git a/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java b/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java index 169d5a309495..782ed8580b71 100644 --- a/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java +++ b/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java @@ -464,7 +464,7 @@ public static void validateProjections( if (schema.getTimeColumnName() == null) { continue; } - final Granularity projectionGranularity = schema.getGranularity(); + final Granularity projectionGranularity = schema.getEffectiveGranularity(); if (segmentGranularity != null) { if (segmentGranularity.isFinerThan(projectionGranularity)) { throw InvalidInput.exception( diff --git a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java index 360fcf455fa3..64f864a26fd8 100644 --- a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java +++ b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java @@ -52,7 +52,6 @@ import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.query.filter.SelectorDimFilter; import org.apache.druid.segment.TestHelper; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.transform.ExpressionTransform; import org.apache.druid.segment.transform.TransformSpec; import org.apache.druid.testing.InitializedNullHandlingTest; @@ -636,17 +635,16 @@ void testSerde() throws Exception public void testSerdeWithProjections() throws Exception { // serialize, then deserialize of DataSchema with projections. - AggregateProjectionSpec projectionSpec = new AggregateProjectionSpec( - "ab_count_projection", - null, - Arrays.asList( - new StringDimensionSchema("a"), - new LongDimensionSchema("b") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("count") - } - ); + AggregateProjectionSpec projectionSpec = + AggregateProjectionSpec.builder("ab_count_projection") + .groupingColumns( + new StringDimensionSchema("a"), + new LongDimensionSchema("b") + ) + .aggregators( + new CountAggregatorFactory("count") + ) + .build(); DataSchema original = DataSchema.builder() .withDataSource("datasource") .withTimestamp(new TimestampSpec(null, null, null)) @@ -876,18 +874,20 @@ void testInvalidProjectionDupeNames() ) .withProjections( List.of( - new AggregateProjectionSpec( - "some projection", - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "g")), - List.of(new LongDimensionSchema("g")), - new AggregatorFactory[]{new CountAggregatorFactory("count")} - ), - new AggregateProjectionSpec( - "some projection", - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.MINUTE, "g")), - List.of(new LongDimensionSchema("g")), - new AggregatorFactory[]{new CountAggregatorFactory("count")} - ) + AggregateProjectionSpec.builder("some projection") + .virtualColumns( + Granularities.toVirtualColumn(Granularities.HOUR, "g") + ) + .groupingColumns(new LongDimensionSchema("g")) + .aggregators(new CountAggregatorFactory("count")) + .build(), + AggregateProjectionSpec.builder("some projection") + .virtualColumns( + Granularities.toVirtualColumn(Granularities.MINUTE, "g") + ) + .groupingColumns(new LongDimensionSchema("g")) + .aggregators(new CountAggregatorFactory("count")) + .build() ) ) .build() @@ -902,6 +902,9 @@ void testInvalidProjectionDupeNames() @Test void testInvalidProjectionGranularity() { + AggregateProjectionSpec.Builder bob = AggregateProjectionSpec.builder() + .groupingColumns(new LongDimensionSchema("g")) + .aggregators(new CountAggregatorFactory("count")); Throwable t = Assertions.assertThrows( DruidException.class, () -> DataSchema.builder() @@ -916,30 +919,19 @@ void testInvalidProjectionGranularity() ) .withProjections( List.of( - new AggregateProjectionSpec( - "ok granularity", - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "g")), - List.of(new LongDimensionSchema("g")), - new AggregatorFactory[]{new CountAggregatorFactory("count")} - ), - new AggregateProjectionSpec( - "acceptable granularity", - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.MINUTE, "g")), - List.of(new LongDimensionSchema("g")), - new AggregatorFactory[]{new CountAggregatorFactory("count")} - ), - new AggregateProjectionSpec( - "not having a time column is ok too", - VirtualColumns.EMPTY, - null, - new AggregatorFactory[]{new CountAggregatorFactory("count")} - ), - new AggregateProjectionSpec( - "bad granularity", - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.DAY, "g")), - List.of(new LongDimensionSchema("g")), - new AggregatorFactory[]{new CountAggregatorFactory("count")} - ) + bob.name("ok granularity") + .virtualColumns(Granularities.toVirtualColumn(Granularities.HOUR, "g")) + .build(), + bob.name("acceptable granularity") + .virtualColumns(Granularities.toVirtualColumn(Granularities.MINUTE, "g")) + .build(), + AggregateProjectionSpec.builder() + .name("not having a time column is ok too") + .aggregators(new CountAggregatorFactory("count")) + .build(), + bob.name("bad granularity") + .virtualColumns(Granularities.toVirtualColumn(Granularities.DAY, "g")) + .build() ) ) .build() diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java index 41d9a8d3d5fb..a8e6977dc42c 100644 --- a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java +++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTest.java @@ -33,11 +33,9 @@ import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.TestDataSource; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.server.coordinator.DataSourceCompactionConfig; import org.apache.druid.server.coordinator.InlineSchemaDataSourceCompactionConfig; @@ -336,19 +334,22 @@ public void testStatusWhenProjectionsMatch() final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, 0L); final IndexSpec currentIndexSpec = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); - final AggregateProjectionSpec projection1 = new AggregateProjectionSpec( - "foo", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) - ), - List.of( - new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - new StringDimensionSchema("a") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long") - } - ); + final AggregateProjectionSpec projection1 = + AggregateProjectionSpec.builder("foo") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + new StringDimensionSchema("a") + ) + .aggregators( + new LongSumAggregatorFactory("sum_long", "long") + ) + .build(); final CompactionState lastCompactionState = new CompactionState( currentPartitionsSpec, null, @@ -383,27 +384,26 @@ public void testStatusWhenProjectionsMismatch() final PartitionsSpec currentPartitionsSpec = new DynamicPartitionsSpec(100, 0L); final IndexSpec currentIndexSpec = IndexSpec.builder().withDimensionCompression(CompressionStrategy.ZSTD).build(); - final AggregateProjectionSpec projection1 = new AggregateProjectionSpec( - "1", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) - ), - List.of( - new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), - new StringDimensionSchema("a") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long") - } - ); - final AggregateProjectionSpec projection2 = new AggregateProjectionSpec( - "2", - VirtualColumns.EMPTY, - Collections.emptyList(), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long") - } - ); + final AggregateProjectionSpec projection1 = + AggregateProjectionSpec.builder("1") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns( + new LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + new StringDimensionSchema("a") + ) + .aggregators( + new LongSumAggregatorFactory("sum_long", "long") + ) + .build(); + final AggregateProjectionSpec projection2 = + AggregateProjectionSpec.builder("2") + .aggregators(new LongSumAggregatorFactory("sum_long", "long")) + .build(); final CompactionState lastCompactionState = new CompactionState( currentPartitionsSpec, diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java b/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java index fdd606344607..e7ed9a7d038f 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CatalogDataSourceCompactionConfigTest.java @@ -36,10 +36,8 @@ import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.math.expr.ExprMacroTable; -import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.expression.TestExprMacroTable; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; import org.junit.jupiter.api.Assertions; @@ -53,18 +51,17 @@ public class CatalogDataSourceCompactionConfigTest private static final ObjectMapper MAPPER; private static final MapMetadataCatalog METADATA_CATALOG; - private static final AggregateProjectionSpec TEST_PROJECTION_SPEC_1 = new AggregateProjectionSpec( - "string_sum_long_hourly", - VirtualColumns.create( - Granularities.toVirtualColumn(Granularities.HOUR, Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME) - ), - ImmutableList.of( - new StringDimensionSchema("string") - ), - new AggregatorFactory[]{ - new LongSumAggregatorFactory("sum_long", "long") - } - ); + private static final AggregateProjectionSpec TEST_PROJECTION_SPEC_1 = + AggregateProjectionSpec.builder("string_sum_long_hourly") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns(new StringDimensionSchema("string")) + .aggregators(new LongSumAggregatorFactory("sum_long", "long")) + .build(); static { MAPPER = new DefaultObjectMapper(); diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java index 228229e7b737..0d0022896066 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java @@ -77,7 +77,6 @@ import org.apache.druid.rpc.indexing.NoopOverlordClient; import org.apache.druid.rpc.indexing.OverlordClient; import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.incremental.OnheapIncrementalIndex; import org.apache.druid.segment.indexing.BatchIOConfig; import org.apache.druid.segment.transform.CompactionTransformSpec; @@ -942,22 +941,17 @@ public void testCompactWithProjections() final CompactSegments compactSegments = new CompactSegments(statusTracker, mockClient); final List compactionConfigs = new ArrayList<>(); final String dataSource = DATA_SOURCE_PREFIX + 0; - final List projections = ImmutableList.of( - new AggregateProjectionSpec( - dataSource + "_projection", - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of( - new StringDimensionSchema("bar") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("cnt") - } - ) + final List projections = List.of( + AggregateProjectionSpec.builder(dataSource + "_projection") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns(new StringDimensionSchema("bar")) + .aggregators(new CountAggregatorFactory("cnt")) + .build() ); compactionConfigs.add( @@ -999,21 +993,17 @@ public void testCompactWithCatalogProjections() ); final ArgumentCaptor payloadCaptor = setUpMockClient(mockClient); - final AggregateProjectionSpec projectionSpec = new AggregateProjectionSpec( - dataSource + "_projection", - VirtualColumns.create( - Granularities.toVirtualColumn( - Granularities.HOUR, - Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME - ) - ), - ImmutableList.of( - new StringDimensionSchema("bar") - ), - new AggregatorFactory[]{ - new CountAggregatorFactory("cnt") - } - ); + final AggregateProjectionSpec projectionSpec = + AggregateProjectionSpec.builder(dataSource + "_projection") + .virtualColumns( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ) + .groupingColumns(new StringDimensionSchema("bar")) + .aggregators(new CountAggregatorFactory("cnt")) + .build(); metadataCatalog.addSpec( TableId.datasource(dataSource), TableBuilder.datasource(dataSource, "P1D")