From a8557a198880472a896820caeca931a113cde5bc Mon Sep 17 00:00:00 2001 From: cecemei Date: Tue, 10 Jun 2025 21:35:11 -0700 Subject: [PATCH 1/9] Add projection data to SegmentAnalysis --- ...ruidSchemaInternRowSignatureBenchmark.java | 1 + processing/pom.xml | 5 + .../query/aggregation/AggregatorFactory.java | 10 +- .../SegmentMetadataQueryQueryToolChest.java | 12 +- .../SegmentMetadataQueryRunnerFactory.java | 19 + .../metadata/metadata/SegmentAnalysis.java | 25 +- .../metadata/SegmentMetadataQuery.java | 8 +- .../segment/AggregateProjectionMetadata.java | 31 + .../apache/druid/segment/IndexMergerV9.java | 101 +- .../apache/druid/query/DoubleStorageTest.java | 2 + .../metadata/SegmentAnalysisBuilder.java | 131 ++ .../query/metadata/SegmentAnalysisTest.java | 36 + ...egmentMetadataQueryQueryToolChestTest.java | 1604 +++++------------ .../metadata/SegmentMetadataQueryTest.java | 21 +- .../SegmentMetadataUnionQueryTest.java | 1 + .../AggregateProjectionMetadataTest.java | 135 +- .../apache/druid/segment/MetadataTest.java | 12 +- .../org/apache/druid/segment/TestIndex.java | 21 +- .../CoordinatorSegmentMetadataCacheTest.java | 2 + 19 files changed, 904 insertions(+), 1273 deletions(-) create mode 100644 processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisBuilder.java diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index 42017bcced1e..f08d618ff6c3 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -143,6 +143,7 @@ public Sequence runSegmentMetadataQuery(Iterable seg null, null, null, + null, false ) ) diff --git a/processing/pom.xml b/processing/pom.xml index 6c5b76ca1d72..8d73e536c76d 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -453,6 +453,11 @@ system-rules test + + org.junit.jupiter + junit-jupiter-params + test + diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java index 3d34500b7109..b46330ebd3d8 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java @@ -443,20 +443,20 @@ public static AggregatorFactory[] mergeAggregators(List agg mergedAggregators.put(name, aggregator.getMergingFactory(other)); } catch (AggregatorFactoryNotMergeableException ex) { + // Aggregator with the same name can't be merged, log it and return null early log.warn(ex, "failed to merge aggregator factories"); - mergedAggregators = null; - break; + return null; } } else { mergedAggregators.put(name, aggregator); } } } else { - mergedAggregators = null; - break; + // one of the segments being merged has unknown aggregators, return null early + return null; } } - return mergedAggregators == null ? null : mergedAggregators.values().toArray(new AggregatorFactory[0]); + return mergedAggregators.values().toArray(new AggregatorFactory[0]); } } diff --git a/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java index 23e583214c84..fde69d13a9c7 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java @@ -30,10 +30,10 @@ import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import com.google.inject.Inject; +import org.apache.commons.collections.MapUtils; import org.apache.druid.common.guava.CombiningSequence; import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.error.DruidException; -import org.apache.druid.error.InvalidInput; import org.apache.druid.java.util.common.JodaUtils; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.guava.Comparators; @@ -57,6 +57,7 @@ import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; +import org.apache.druid.segment.AggregateProjectionMetadata; import org.apache.druid.timeline.LogicalSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.utils.CollectionUtils; @@ -281,7 +282,7 @@ public static SegmentAnalysis mergeAnalyses( // This is a defensive check since SegementMetadata query instantiation guarantees this if (CollectionUtils.isNullOrEmpty(dataSources)) { - throw InvalidInput.exception("SegementMetadata queries require at least one datasource."); + throw DruidException.defensive("SegementMetadata queries require at least one datasource."); } SegmentId mergedSegmentId = null; @@ -437,6 +438,11 @@ public static SegmentAnalysis mergeAnalyses( rollup = null; } + Map projections = + (arg1.getProjections() == null || arg2.getProjections() == null) + ? null + : AggregateProjectionMetadata.merge(arg1.getProjections(), arg2.getProjections()); + return new SegmentAnalysis( mergedId, newIntervals, @@ -444,6 +450,7 @@ public static SegmentAnalysis mergeAnalyses( arg1.getSize() + arg2.getSize(), arg1.getNumRows() + arg2.getNumRows(), aggregators.isEmpty() ? null : aggregators, + MapUtils.isEmpty(projections) ? null : projections, timestampSpec, queryGranularity, rollup @@ -460,6 +467,7 @@ public static SegmentAnalysis finalizeAnalysis(SegmentAnalysis analysis) analysis.getSize(), analysis.getNumRows(), analysis.getAggregators(), + analysis.getProjections(), analysis.getTimestampSpec(), analysis.getQueryGranularity(), analysis.isRollup() diff --git a/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java index 664b5d32bdce..ab4305d776dd 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java @@ -46,6 +46,7 @@ import org.apache.druid.query.metadata.metadata.ColumnIncluderator; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; +import org.apache.druid.segment.AggregateProjectionMetadata; import org.apache.druid.segment.Metadata; import org.apache.druid.segment.PhysicalSegmentInspector; import org.apache.druid.segment.Segment; @@ -57,10 +58,12 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; public class SegmentMetadataQueryRunnerFactory implements QueryRunnerFactory { @@ -132,6 +135,21 @@ public Sequence run(QueryPlus inQ, ResponseCon aggregators = null; } + final Map projectionsMap; + + if (updatedQuery.hasProjections() + && ((metadata = Objects.isNull(metadata) ? getMetadata(segment) : metadata)) != null + && metadata.getProjections() != null) { + projectionsMap = metadata.getProjections() + .stream() + .collect(Collectors.toUnmodifiableMap( + projectionMetadata -> projectionMetadata.getSchema().getName(), + p -> p + )); + } else { + projectionsMap = null; + } + final TimestampSpec timestampSpec; if (updatedQuery.hasTimestampSpec()) { if (metadata == null) { @@ -174,6 +192,7 @@ public Sequence run(QueryPlus inQ, ResponseCon totalSize, numRows, aggregators, + projectionsMap, timestampSpec, queryGranularity, rollup diff --git a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java index 13576a6a11f0..5b642d1b1f98 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java @@ -24,6 +24,7 @@ import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.segment.AggregateProjectionMetadata; import org.joda.time.Interval; import java.util.LinkedHashMap; @@ -52,6 +53,7 @@ public class SegmentAnalysis implements Comparable private final long size; private final long numRows; private final Map aggregators; + private final Map projections; private final TimestampSpec timestampSpec; private final Granularity queryGranularity; private final Boolean rollup; @@ -64,6 +66,7 @@ public SegmentAnalysis( @JsonProperty("size") long size, @JsonProperty("numRows") long numRows, @JsonProperty("aggregators") Map aggregators, + @JsonProperty("projections") Map projections, @JsonProperty("timestampSpec") TimestampSpec timestampSpec, @JsonProperty("queryGranularity") Granularity queryGranularity, @JsonProperty("rollup") Boolean rollup @@ -75,6 +78,7 @@ public SegmentAnalysis( this.size = size; this.numRows = numRows; this.aggregators = aggregators; + this.projections = projections; this.timestampSpec = timestampSpec; this.queryGranularity = queryGranularity; this.rollup = rollup; @@ -134,6 +138,12 @@ public Map getAggregators() return aggregators; } + @JsonProperty + public Map getProjections() + { + return projections; + } + @Override public String toString() { @@ -144,6 +154,7 @@ public String toString() ", size=" + size + ", numRows=" + numRows + ", aggregators=" + aggregators + + ", projections=" + projections + ", timestampSpec=" + timestampSpec + ", queryGranularity=" + queryGranularity + ", rollup=" + rollup + @@ -170,6 +181,7 @@ public boolean equals(Object o) Objects.equals(interval, that.interval) && Objects.equals(columns, that.columns) && Objects.equals(aggregators, that.aggregators) && + Objects.equals(projections, that.projections) && Objects.equals(timestampSpec, that.timestampSpec) && Objects.equals(queryGranularity, that.queryGranularity); } @@ -181,7 +193,18 @@ public boolean equals(Object o) @Override public int hashCode() { - return Objects.hash(id, interval, columns, size, numRows, aggregators, timestampSpec, queryGranularity, rollup); + return Objects.hash( + id, + interval, + columns, + size, + numRows, + aggregators, + projections, + timestampSpec, + queryGranularity, + rollup + ); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java index 5458fc65de73..bf407e0cc9ac 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java @@ -59,7 +59,8 @@ public enum AnalysisType implements Cacheable MINMAX, TIMESTAMPSPEC, QUERYGRANULARITY, - ROLLUP; + ROLLUP, + PROJECTIONS; @JsonValue @Override @@ -192,6 +193,11 @@ public boolean hasAggregators() return analysisTypes.contains(AnalysisType.AGGREGATORS); } + public boolean hasProjections() + { + return analysisTypes.contains(AnalysisType.PROJECTIONS); + } + public boolean hasTimestampSpec() { return analysisTypes.contains(AnalysisType.TIMESTAMPSPEC); diff --git a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java index 7e9aec5e8081..4cb0cec6b3b5 100644 --- a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java +++ b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java @@ -28,6 +28,7 @@ import com.google.common.collect.Interner; import com.google.common.collect.Interners; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.apache.druid.data.input.impl.AggregateProjectionSpec; import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.granularity.Granularities; @@ -42,7 +43,9 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; /** @@ -119,6 +122,34 @@ public String toString() '}'; } + /** + * Merge two maps of {@link AggregateProjectionMetadata} objects, returning a new map with the same keys and merged + * metadata. If the schemas do not match, the metadata is not merged and the key is not included in the result. + * + * @param projections1 first map of projections to merge + * @param projections2 second map of projections to merge + * @return merged map of projections + */ + public static Map merge( + Map projections1, + Map projections2 + ) + { + final Map projections = new HashMap<>(); + for (String name : Sets.intersection(projections1.keySet(), projections2.keySet())) { + AggregateProjectionMetadata spec1 = projections1.get(name); + AggregateProjectionMetadata spec2 = projections2.get(name); + if (spec1.getSchema().equals(spec2.getSchema())) { + // If the schemas are equal, we can merge the metadata + projections.put( + name, + new AggregateProjectionMetadata(spec1.getSchema(), spec1.getNumRows() + spec2.getNumRows()) + ); + } + } + return projections; + } + public static class Schema { /** diff --git a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java index 4afd7afdc0e3..846ed5cec9f2 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java @@ -130,7 +130,7 @@ public IndexMergerV9( private File makeIndexFiles( final List adapters, - final @Nullable AggregatorFactory[] metricAggs, + final @Nullable Metadata segmentMetadata, final File outDir, final ProgressIndicator progress, final List mergedDimensionsWithTime, // has both explicit and implicit dimensions, as well as __time @@ -144,43 +144,11 @@ private File makeIndexFiles( progress.start(); progress.progress(); - List metadataList = Lists.transform(adapters, IndexableAdapter::getMetadata); - // Merged dimensions without __time. List mergedDimensions = mergedDimensionsWithTime.stream() .filter(dim -> !ColumnHolder.TIME_COLUMN_NAME.equals(dim)) .collect(Collectors.toList()); - - Metadata segmentMetadata; - if (metricAggs != null) { - AggregatorFactory[] combiningMetricAggs = new AggregatorFactory[metricAggs.length]; - for (int i = 0; i < metricAggs.length; i++) { - combiningMetricAggs[i] = metricAggs[i].getCombiningFactory(); - } - segmentMetadata = Metadata.merge( - metadataList, - combiningMetricAggs - ); - } else { - segmentMetadata = Metadata.merge( - metadataList, - null - ); - } - - if (segmentMetadata != null - && segmentMetadata.getOrdering() != null - && segmentMetadata.getOrdering() - .stream() - .noneMatch(orderBy -> ColumnHolder.TIME_COLUMN_NAME.equals(orderBy.getColumnName()))) { - throw DruidException.defensive( - "sortOrder[%s] must include[%s]", - segmentMetadata.getOrdering(), - ColumnHolder.TIME_COLUMN_NAME - ); - } - Closer closer = Closer.create(); try { final FileSmoosher v9Smoosher = new FileSmoosher(outDir); @@ -302,20 +270,21 @@ private File makeIndexFiles( progress.stopSection(section); - if (segmentMetadata != null && !CollectionUtils.isNullOrEmpty(segmentMetadata.getProjections())) { - segmentMetadata = makeProjections( - v9Smoosher, - segmentMetadata.getProjections(), - adapters, - indexSpec, - segmentWriteOutMedium, - progress, - outDir, - closer, - mergersMap, - segmentMetadata - ); - } + // Recompute the projections. + final Metadata finalMetadata = + segmentMetadata == null || CollectionUtils.isNullOrEmpty(segmentMetadata.getProjections()) + ? segmentMetadata + : makeProjections(v9Smoosher, + segmentMetadata.getProjections(), + adapters, + indexSpec, + segmentWriteOutMedium, + progress, + outDir, + closer, + mergersMap, + segmentMetadata + ); /************* Make index.drd & metadata.drd files **************/ progress.progress(); @@ -330,7 +299,7 @@ private File makeIndexFiles( mergers, dimensionsSpecInspector ); - makeMetadataBinary(v9Smoosher, progress, segmentMetadata); + makeMetadataBinary(v9Smoosher, progress, finalMetadata); v9Smoosher.close(); progress.stop(); @@ -677,6 +646,7 @@ private void makeMetricsColumns( { makeMetricsColumns(v9Smoosher, progress, mergedMetrics, metricsTypes, metWriters, indexSpec, ""); } + private void makeMetricsColumns( final FileSmoosher v9Smoosher, final ProgressIndicator progress, @@ -1145,6 +1115,9 @@ public File mergeQueryableIndex( ); } + /** + * The indexes here must have the same {@link Metadata}, otherwise an error would be thrown. + */ @Override public File merge( List indexes, @@ -1377,9 +1350,39 @@ private File merge( rowMergerFn = MergingRowIterator::new; } + List metadataList = Lists.transform(indexes, IndexableAdapter::getMetadata); + final Metadata segmentMetadata; + if (metricAggs != null) { + AggregatorFactory[] combiningMetricAggs = new AggregatorFactory[metricAggs.length]; + for (int i = 0; i < metricAggs.length; i++) { + combiningMetricAggs[i] = metricAggs[i].getCombiningFactory(); + } + segmentMetadata = Metadata.merge( + metadataList, + combiningMetricAggs + ); + } else { + segmentMetadata = Metadata.merge( + metadataList, + null + ); + } + + if (segmentMetadata != null + && segmentMetadata.getOrdering() != null + && segmentMetadata.getOrdering() + .stream() + .noneMatch(orderBy -> ColumnHolder.TIME_COLUMN_NAME.equals(orderBy.getColumnName()))) { + throw DruidException.defensive( + "sortOrder[%s] must include[%s]", + segmentMetadata.getOrdering(), + ColumnHolder.TIME_COLUMN_NAME + ); + } + return makeIndexFiles( indexes, - sortedMetricAggs, + segmentMetadata, outDir, progress, mergedDimensionsWithTime, diff --git a/processing/src/test/java/org/apache/druid/query/DoubleStorageTest.java b/processing/src/test/java/org/apache/druid/query/DoubleStorageTest.java index 03b5fe7e45e3..678c5c2d3669 100644 --- a/processing/src/test/java/org/apache/druid/query/DoubleStorageTest.java +++ b/processing/src/test/java/org/apache/druid/query/DoubleStorageTest.java @@ -197,6 +197,7 @@ public static Collection dataFeeder() null, null, null, + null, null ); @@ -248,6 +249,7 @@ public static Collection dataFeeder() null, null, null, + null, null ); diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisBuilder.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisBuilder.java new file mode 100644 index 000000000000..9015ec0b75c4 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisBuilder.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.metadata; + +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.metadata.metadata.ColumnAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentAnalysis; +import org.apache.druid.segment.AggregateProjectionMetadata; +import org.apache.druid.timeline.SegmentId; +import org.joda.time.Interval; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Helper class to build {@link SegmentAnalysis} objects for testing purposes. + */ +public class SegmentAnalysisBuilder +{ + String segmentId; + List intervals = null; + LinkedHashMap columns = new LinkedHashMap<>(); + Map aggregators = new LinkedHashMap<>(); + Map projections = new LinkedHashMap<>(); + Optional size = Optional.empty(); + Optional numRows = Optional.empty(); + Optional rollup = Optional.empty(); + + SegmentAnalysisBuilder(String segmentId) + { + this.segmentId = segmentId; + } + + SegmentAnalysisBuilder(SegmentId segmentId) + { + this.segmentId = segmentId.toString(); + } + + SegmentAnalysisBuilder size(int size) + { + if (this.size.isEmpty()) { + this.size = Optional.of(size); + } else { + throw new IllegalStateException("Size is already set: " + this.size.get()); + } + return this; + } + + SegmentAnalysisBuilder numRows(int numRows) + { + if (this.numRows.isEmpty()) { + this.numRows = Optional.of(numRows); + } else { + throw new IllegalStateException("NumRows is already set: " + this.numRows.get()); + } + return this; + } + + SegmentAnalysisBuilder rollup(boolean rollup) + { + if (this.rollup.isEmpty()) { + this.rollup = Optional.of(rollup); + } else { + throw new IllegalStateException("Rollup is already set: " + this.rollup.get()); + } + return this; + } + + SegmentAnalysisBuilder interval(Interval interval) + { + if (this.intervals == null) { + this.intervals = new ArrayList<>(); + } + this.intervals.add(interval); + return this; + } + + SegmentAnalysisBuilder column(String columnName, ColumnAnalysis columnAnalysis) + { + this.columns.put(columnName, columnAnalysis); + return this; + } + + SegmentAnalysisBuilder aggregator(String name, AggregatorFactory aggregatorFactory) + { + this.aggregators.put(name, aggregatorFactory); + return this; + } + + SegmentAnalysisBuilder projection(String name, AggregateProjectionMetadata projection) + { + this.projections.put(name, projection); + return this; + } + + SegmentAnalysis build() + { + return new SegmentAnalysis( + segmentId, + intervals, + columns, + size.orElse(0), + numRows.orElse(0), + aggregators.isEmpty() ? null : aggregators, + projections.isEmpty() ? null : projections, + null, + null, + rollup.orElse(null) + ); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java index 4f68c9e059dd..56800c4c3c41 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisTest.java @@ -22,13 +22,19 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.OrderBy; +import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; +import org.apache.druid.segment.AggregateProjectionMetadata; import org.apache.druid.segment.TestHelper; +import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; import org.junit.Assert; import org.junit.Test; @@ -37,6 +43,15 @@ public class SegmentAnalysisTest { + + @Test + public void testEquals() + { + EqualsVerifier.forClass(SegmentAnalysis.class) + .usingGetClass() + .verify(); + } + @Test public void testSerde() throws Exception { @@ -67,6 +82,27 @@ public void testSerde() throws Exception 1, 2, ImmutableMap.of("cnt", new CountAggregatorFactory("cnt")), + ImmutableMap.of("channel_added_hourly", new AggregateProjectionMetadata( + new AggregateProjectionMetadata.Schema( + "channel_added_hourly", + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, + VirtualColumns.create( + Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + ) + ), + ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel"), + new AggregatorFactory[] { + new LongSumAggregatorFactory("sum_added", "added") + }, + ImmutableList.of( + OrderBy.ascending(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + OrderBy.ascending("channel") + ) + ), + 16 + )), new TimestampSpec(null, null, null), Granularities.SECOND, true diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java index 4afb202b4331..525616bb8545 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java @@ -22,15 +22,16 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.druid.error.DruidException; import org.apache.druid.error.DruidExceptionMatcher; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.query.CacheStrategy; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; +import org.apache.druid.query.OrderBy; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.UnionDataSource; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -43,6 +44,8 @@ import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; import org.apache.druid.query.spec.LegacySegmentSpec; +import org.apache.druid.segment.AggregateProjectionMetadata; +import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.ValueType; import org.apache.druid.timeline.LogicalSegment; @@ -51,29 +54,53 @@ import org.joda.time.Interval; import org.joda.time.Period; import org.junit.Assert; +import org.junit.Assume; import org.junit.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; public class SegmentMetadataQueryQueryToolChestTest { private static final DataSource TEST_DATASOURCE = new TableDataSource("dummy"); - private static final SegmentId TEST_SEGMENT_ID1 = SegmentId.of( - TEST_DATASOURCE.toString(), - Intervals.of("2020-01-01/2020-01-02"), - "test", - 0 + private static final Interval INTERVAL_2020 = Intervals.of("2020-01-01/2020-01-02"); + private static final Interval INTERVAL_2021 = Intervals.of("2021-01-01/2021-01-02"); + private static final SegmentId TEST_SEGMENT_ID1 = SegmentId.of(TEST_DATASOURCE.toString(), INTERVAL_2020, "test", 0); + private static final SegmentId TEST_SEGMENT_ID2 = SegmentId.of(TEST_DATASOURCE.toString(), INTERVAL_2021, "test", 0); + + private static final AggregateProjectionMetadata.Schema PROJECTION_CHANNEL_ADDED_HOURLY = new AggregateProjectionMetadata.Schema( + "name1-doesnot-matter", + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, + VirtualColumns.create(Granularities.toVirtualColumn( + Granularities.HOUR, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + )), + ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel"), + new AggregatorFactory[]{ + new LongSumAggregatorFactory("channel_sum", "channel") + }, + ImmutableList.of( + OrderBy.ascending(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + OrderBy.ascending("channel") + ) ); - private static final SegmentId TEST_SEGMENT_ID2 = SegmentId.of( - TEST_DATASOURCE.toString(), - Intervals.of("2021-01-01/2021-01-02"), - "test", - 0 + private static final AggregateProjectionMetadata.Schema PROJECTION_CHANNEL_ADDED_DAILY = new AggregateProjectionMetadata.Schema( + "name2-doesnot-matter", + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, + VirtualColumns.create(Granularities.toVirtualColumn( + Granularities.DAY, + Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME + )), + ImmutableList.of(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, "channel"), + new AggregatorFactory[]{ + new LongSumAggregatorFactory("channel_sum", "channel") + }, + ImmutableList.of( + OrderBy.ascending(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME), + OrderBy.ascending("channel") + ) ); @Test @@ -99,32 +126,25 @@ public void testCacheStrategy() throws Exception byte[] actualKey = strategy.computeCacheKey(query); Assert.assertArrayEquals(expectedKey, actualKey); - SegmentAnalysis result = new SegmentAnalysis( - TEST_SEGMENT_ID1.toString(), - ImmutableList.of(Intervals.of("2011-01-12T00:00:00.000Z/2011-04-15T00:00:00.001Z")), - new LinkedHashMap<>( - ImmutableMap.of( - "placement", - new ColumnAnalysis( - ColumnType.STRING, - ValueType.STRING.name(), - true, - false, - 10881, - 1, - "preferred", - "preferred", - null - ) + SegmentAnalysis result = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + .interval(Intervals.of("2011-01-12T00:00:00.000Z/2011-04-15T00:00:00.001Z")) + .column( + "placement", + new ColumnAnalysis( + ColumnType.STRING, + ValueType.STRING.name(), + true, + false, + 10881, + 1, + "preferred", + "preferred", + null ) - ), - 71982, - 100, - null, - null, - null, - null - ); + ) + .size(71982) + .numRows(100) + .build(); Object preparedValue = strategy.prepareForSegmentLevelCache().apply(result); @@ -139,980 +159,363 @@ public void testCacheStrategy() throws Exception Assert.assertEquals(result, fromCacheResult); } - @Test - public void testMergeAggregators() + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testProjections(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysis( - TEST_SEGMENT_ID1.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "baz", new DoubleSumAggregatorFactory("baz", "baz") - ), - null, - null, - null - ); - final SegmentAnalysis analysis2 = new SegmentAnalysis( - TEST_SEGMENT_ID2.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar") - ), - null, - null, - null - ); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar"), - "baz", new DoubleSumAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeStrict(analysis1, analysis2) - ); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar"), - "baz", new DoubleSumAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeLenient(analysis1, analysis2) - ); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar"), - "baz", new DoubleSumAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeEarliest(analysis1, analysis2) - ); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar"), - "baz", new DoubleSumAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeLatest(analysis1, analysis2) - ); + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) + .build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 200)) + .build(); + + final SegmentAnalysis expected = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 300)) + .build(); + Assert.assertEquals(expected, mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy)); } - @Test - public void testMergeAggregatorsWithIntervals() + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testProjectionsWithNull(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysis( - TEST_SEGMENT_ID1.toString(), - ImmutableList.of(TEST_SEGMENT_ID1.getInterval()), - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "baz", new DoubleSumAggregatorFactory("baz", "baz") - ), - null, - null, - null - ); - final SegmentAnalysis analysis2 = new SegmentAnalysis( - TEST_SEGMENT_ID2.toString(), - ImmutableList.of(TEST_SEGMENT_ID2.getInterval()), - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar") - ), - null, - null, - null - ); - - final List expectedIntervals = new ArrayList<>(); - expectedIntervals.addAll(analysis1.getIntervals()); - expectedIntervals.addAll(analysis2.getIntervals()); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - expectedIntervals, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar"), - "baz", new DoubleSumAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeStrict(analysis1, analysis2) - ); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - expectedIntervals, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar"), - "baz", new DoubleSumAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeLenient(analysis1, analysis2) - ); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - expectedIntervals, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "baz", new DoubleSumAggregatorFactory("baz", "baz"), - "bar", new DoubleSumAggregatorFactory("bar", "bar") - ), - null, - null, - null - ), - mergeEarliest(analysis1, analysis2) + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) + .build(); + final SegmentAnalysis analysis1NullProjection = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 200)) + .build(); + final SegmentAnalysis analysis2NullProjection = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).build(); + + Assert.assertNull(mergeWithStrategy(analysis1NullProjection, analysis2, aggregatorMergeStrategy).getProjections()); + Assert.assertNull(mergeWithStrategy(analysis1, analysis2NullProjection, aggregatorMergeStrategy).getProjections()); + Assert.assertNull( + mergeWithStrategy(analysis1NullProjection, analysis2NullProjection, aggregatorMergeStrategy).getProjections() ); + } - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - expectedIntervals, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar"), - "baz", new DoubleSumAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeLatest(analysis1, analysis2) - ); + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testProjectionsWithConflict(AggregatorMergeStrategy aggregatorMergeStrategy) + { + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) + .projection("channel_sum_1", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) + .projection("conflict_projection", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) + .build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 200)) + .projection("channel_sum_2", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_DAILY, 200)) + .projection("conflict_projection", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_DAILY, 200)) + .build(); + + // conflict projection is ignored. + final SegmentAnalysis expectedStrict = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 300)) + .build(); + Assert.assertEquals(expectedStrict, mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy)); } - @Test - public void testMergeAggregatorsOneNull() + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testMergeAggregators(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysis( - TEST_SEGMENT_ID1.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - null - ); - final SegmentAnalysis analysis2 = new SegmentAnalysis( - TEST_SEGMENT_ID2.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar") - ), - null, - null, - null - ); + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("baz", new DoubleSumAggregatorFactory("baz", "baz")) + .build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .build(); + + final SegmentAnalysis expected = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .aggregator("baz", new DoubleSumAggregatorFactory("baz", "baz")) + .build(); + Assert.assertEquals(expected, mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy)); + } - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - null - ), - mergeStrict(analysis1, analysis2) - ); + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testMergeAggregatorsWithIntervals(AggregatorMergeStrategy aggregatorMergeStrategy) + { + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + .interval(TEST_SEGMENT_ID1.getInterval()) + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("baz", new DoubleSumAggregatorFactory("baz", "baz")) + .build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + .interval(TEST_SEGMENT_ID2.getInterval()) + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .build(); + + SegmentAnalysis expected = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .interval(TEST_SEGMENT_ID1.getInterval()) + .interval(TEST_SEGMENT_ID2.getInterval()) + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .aggregator("baz", new DoubleSumAggregatorFactory("baz", "baz")) + .build(); + Assert.assertEquals(expected, mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy)); + } - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar") - ), - null, - null, - null - ), - mergeLenient(analysis1, analysis2) - ); + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testMergeAggregatorsOneNullStrict(AggregatorMergeStrategy aggregatorMergeStrategy) + { + Assume.assumeTrue(aggregatorMergeStrategy == AggregatorMergeStrategy.STRICT); - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar") - ), - null, - null, - null - ), - mergeEarliest(analysis1, analysis2) - ); + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .build(); - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar") - ), - null, - null, - null - ), - mergeLatest(analysis1, analysis2) - ); + final SegmentAnalysis expectedNullAggregators = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged").build(); + Assert.assertEquals(expectedNullAggregators, mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy)); } - @Test - public void testMergeAggregatorsAllNull() + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testMergeAggregatorsOneNullNotStrict(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysis( - TEST_SEGMENT_ID1.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - null - ); - final SegmentAnalysis analysis2 = new SegmentAnalysis( - TEST_SEGMENT_ID2.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - null - ); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - null - ), - mergeStrict(analysis1, analysis2) - ); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - null - ), - mergeLenient(analysis1, analysis2) - ); + Assume.assumeTrue(aggregatorMergeStrategy != AggregatorMergeStrategy.STRICT); + + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .build(); + + final SegmentAnalysis expected = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .build(); + Assert.assertEquals(expected, mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy)); + } - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - null - ), - mergeEarliest(analysis1, analysis2) - ); + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testMergeAggregatorsAllNull(AggregatorMergeStrategy aggregatorMergeStrategy) + { + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).build(); - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - null - ), - mergeLatest(analysis1, analysis2) - ); + final SegmentAnalysis expected = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged").build(); + Assert.assertEquals(expected, mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy)); } @Test public void testMergeAggregatorsConflict() { - final SegmentAnalysis analysis1 = new SegmentAnalysis( - TEST_SEGMENT_ID1.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar") - ), - null, - null, - null - ); - final SegmentAnalysis analysis2 = new SegmentAnalysis( - TEST_SEGMENT_ID2.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleMaxAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ); - - final Map expectedLenient = new HashMap<>(); - expectedLenient.put("foo", new LongSumAggregatorFactory("foo", "foo")); - expectedLenient.put("bar", null); - expectedLenient.put("baz", new LongMaxAggregatorFactory("baz", "baz")); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - null - ), - mergeStrict(analysis1, analysis2) - ); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - expectedLenient, - null, - null, - null - ), - mergeLenient(analysis1, analysis2) - ); - + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleMaxAggregatorFactory("bar", "bar")) + .aggregator("baz", new LongMaxAggregatorFactory("baz", "baz")) + .build(); + + // Test strict merge, returns null aggregators as there's a conflict on "bar" + final SegmentAnalysis expectedStrict = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged").build(); + Assert.assertEquals(expectedStrict, mergeWithStrategy(analysis1, analysis2, AggregatorMergeStrategy.STRICT)); + + // Test lenient merge, returns a map with null for "bar" as it has a conflict + final SegmentAnalysis expectedLenient = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", null) + .aggregator("baz", new LongMaxAggregatorFactory("baz", "baz")) + .build(); + Assert.assertEquals(expectedLenient, mergeLenient(analysis1, analysis2)); // Simulate multi-level lenient merge Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - expectedLenient, - null, - null, - null - ), - mergeLenient( - mergeLenient(analysis1, analysis2), - mergeLenient(analysis1, analysis2) - ) + expectedLenient, + mergeLenient(mergeLenient(analysis1, analysis2), mergeLenient(analysis1, analysis2)) ); - // Simulate multi-level lenient merge (unmerged first) - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - expectedLenient, - null, - null, - null - ), - mergeLenient( - analysis1, - mergeLenient(analysis1, analysis2) - ) - ); - + Assert.assertEquals(expectedLenient, mergeLenient(analysis1, mergeLenient(analysis1, analysis2))); // Simulate multi-level lenient merge (unmerged second) - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - expectedLenient, - null, - null, - null - ), - mergeLenient( - mergeLenient(analysis1, analysis2), - analysis1 - ) - ); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeEarliest(analysis1, analysis2) - ); - + Assert.assertEquals(expectedLenient, mergeLenient(mergeLenient(analysis1, analysis2), analysis1)); + + // Test earliest merge, returns a map with "bar" as DoubleSumAggregatorFactory since analysis1 is earlier + final SegmentAnalysis expectedEarliest = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .aggregator("baz", new LongMaxAggregatorFactory("baz", "baz")) + .build(); + Assert.assertEquals(expectedEarliest, mergeEarliest(analysis1, analysis2)); // Simulate multi-level earliest merge Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeEarliest( - mergeEarliest(analysis1, analysis2), - mergeEarliest(analysis1, analysis2) - ) - ); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleMaxAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeLatest(analysis1, analysis2) + expectedEarliest, + mergeEarliest(mergeEarliest(analysis1, analysis2), mergeEarliest(analysis1, analysis2)) ); + // Test latest merge, returns a map with "bar" as DoubleMaxAggregatorFactory since analysis2 is later + final SegmentAnalysis expectedLatest = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleMaxAggregatorFactory("bar", "bar")) + .aggregator("baz", new LongMaxAggregatorFactory("baz", "baz")) + .build(); + Assert.assertEquals(expectedLatest, mergeLatest(analysis1, analysis2)); // Simulate multi-level latest merge Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleMaxAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeLatest( - mergeLatest(analysis1, analysis2), - mergeLatest(analysis1, analysis2) - ) + expectedLatest, + mergeLatest(mergeLatest(analysis1, analysis2), mergeLatest(analysis1, analysis2)) ); } @Test public void testMergeAggregatorsConflictWithDifferentOrder() { - final SegmentAnalysis analysis1 = new SegmentAnalysis( - TEST_SEGMENT_ID2.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar") - ), - null, - null, - null - ); - - final SegmentAnalysis analysis2 = new SegmentAnalysis( - TEST_SEGMENT_ID1.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleMaxAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ); - - final Map expectedLenient = new HashMap<>(); - expectedLenient.put("foo", new LongSumAggregatorFactory("foo", "foo")); - expectedLenient.put("bar", null); - expectedLenient.put("baz", new LongMaxAggregatorFactory("baz", "baz")); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - null - ), + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleMaxAggregatorFactory("bar", "bar")) + .aggregator("baz", new LongMaxAggregatorFactory("baz", "baz")) + .build(); + + // Test strict merge, returns null aggregators as there's a conflict on "bar" + Assert.assertEquals( + new SegmentAnalysisBuilder("dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .build(), mergeStrict(analysis1, analysis2) ); + // Test lenient merge, returns a map with null for "bar" as it has a conflict + final SegmentAnalysis expectedLenient = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", null) + .aggregator("baz", new LongMaxAggregatorFactory("baz", "baz")) + .build(); Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - expectedLenient, - null, - null, - null - ), + expectedLenient, mergeLenient(analysis1, analysis2) ); - // Simulate multi-level lenient merge Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - expectedLenient, - null, - null, - null - ), + expectedLenient, mergeLenient( mergeLenient(analysis1, analysis2), mergeLenient(analysis1, analysis2) ) ); - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleMaxAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeEarliest(analysis1, analysis2) - ); - + // Test earliest merge, returns a map with "bar" as DoubleMaxAggregatorFactory since analysis2 is earlier + final SegmentAnalysis expectedEarliest = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleMaxAggregatorFactory("bar", "bar")) + .aggregator("baz", new LongMaxAggregatorFactory("baz", "baz")) + .build(); + Assert.assertEquals(expectedEarliest, mergeEarliest(analysis1, analysis2)); // Simulate multi-level earliest merge Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleMaxAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeEarliest( - mergeEarliest(analysis1, analysis2), - mergeEarliest(analysis1, analysis2) - ) - ); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeLatest(analysis1, analysis2) + expectedEarliest, + mergeEarliest(mergeEarliest(analysis1, analysis2), mergeEarliest(analysis1, analysis2)) ); + // Test latest merge, returns a map with "bar" as DoubleSumAggregatorFactory since analysis1 is later + final SegmentAnalysis expectedLatest = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .aggregator("baz", new LongMaxAggregatorFactory("baz", "baz")) + .build(); + Assert.assertEquals(expectedLatest, mergeLatest(analysis1, analysis2)); // Simulate multi-level latest merge Assert.assertEquals( - new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeLatest( - mergeLatest(analysis1, analysis2), - mergeLatest(analysis1, analysis2) - ) + expectedLatest, + mergeLatest(mergeLatest(analysis1, analysis2), mergeLatest(analysis1, analysis2)) ); } @Test public void testMergeAggregatorsConflictWithEqualSegmentIntervalsAndDifferentPartitions() { - final SegmentId segmentId1 = SegmentId.of( - TEST_DATASOURCE.toString(), - Intervals.of("2023-01-01/2023-01-02"), - "test", - 1 - ); - final SegmentId segmentId2 = SegmentId.of( - TEST_DATASOURCE.toString(), - Intervals.of("2023-01-01/2023-01-02"), - "test", - 2 - ); + Interval interval = Intervals.of("2023-01-01/2023-01-02"); + final SegmentId segmentId1 = SegmentId.of(TEST_DATASOURCE.toString(), interval, "test", 1); + final SegmentId segmentId2 = SegmentId.of(TEST_DATASOURCE.toString(), interval, "test", 2); - final SegmentAnalysis analysis1 = new SegmentAnalysis( - segmentId1.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar") - ), - null, - null, - null - ); + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(segmentId1) + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .build(); - final SegmentAnalysis analysis2 = new SegmentAnalysis( - segmentId2.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleMaxAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ); - - final Map expectedLenient = new HashMap<>(); - expectedLenient.put("foo", new LongSumAggregatorFactory("foo", "foo")); - expectedLenient.put("bar", null); - expectedLenient.put("baz", new LongMaxAggregatorFactory("baz", "baz")); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(segmentId2) + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleMaxAggregatorFactory("bar", "bar")) + .aggregator("baz", new LongMaxAggregatorFactory("baz", "baz")) + .build(); + // Test strict merge, returns null aggregators as there's a conflict on "bar" Assert.assertEquals( - new SegmentAnalysis( - "dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2", - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - null - ), + new SegmentAnalysisBuilder("dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2") + .build(), mergeStrict(analysis1, analysis2) ); - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2", - null, - new LinkedHashMap<>(), - 0, - 0, - expectedLenient, - null, - null, - null - ), - mergeLenient(analysis1, analysis2) - ); - + // Test lenient merge, returns a map with null for "bar" as it has a conflict + SegmentAnalysis expectedLenient = new SegmentAnalysisBuilder( + "dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2") + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", null) + .aggregator("baz", new LongMaxAggregatorFactory("baz", "baz")) + .build(); + Assert.assertEquals(expectedLenient, mergeLenient(analysis1, analysis2)); // Simulate multi-level lenient merge Assert.assertEquals( - new SegmentAnalysis( - "dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2", - null, - new LinkedHashMap<>(), - 0, - 0, - expectedLenient, - null, - null, - null - ), - mergeLenient( - mergeLenient(analysis1, analysis2), - mergeLenient(analysis1, analysis2) - ) - ); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeEarliest(analysis1, analysis2) + expectedLenient, + mergeLenient(mergeLenient(analysis1, analysis2), mergeLenient(analysis1, analysis2)) ); + // Test earliest merge, returns a map with "bar" as DoubleSumAggregatorFactory since analysis1 has the earlier partition + SegmentAnalysis expectedEarliest = new SegmentAnalysisBuilder( + "dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2") + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .aggregator("baz", new LongMaxAggregatorFactory("baz", "baz")) + .build(); + Assert.assertEquals(expectedEarliest, mergeEarliest(analysis1, analysis2)); // Simulate multi-level earliest merge Assert.assertEquals( - new SegmentAnalysis( - "dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeEarliest( - mergeEarliest(analysis1, analysis2), - mergeEarliest(analysis1, analysis2) - ) - ); - - Assert.assertEquals( - new SegmentAnalysis( - "dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleMaxAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeLatest(analysis1, analysis2) + expectedEarliest, + mergeEarliest(mergeEarliest(analysis1, analysis2), mergeEarliest(analysis1, analysis2)) ); + // Test latest merge, returns a map with "bar" as DoubleMaxAggregatorFactory since analysis2 has the later partition + SegmentAnalysis expectedLatest = new SegmentAnalysisBuilder( + "dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2") + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleMaxAggregatorFactory("bar", "bar")) + .aggregator("baz", new LongMaxAggregatorFactory("baz", "baz")) + .build(); + Assert.assertEquals(expectedLatest, mergeLatest(analysis1, analysis2)); // Simulate multi-level latest merge Assert.assertEquals( - new SegmentAnalysis( - "dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleMaxAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ), - mergeLatest( - mergeLatest(analysis1, analysis2), - mergeLatest(analysis1, analysis2) - ) + expectedLatest, + mergeLatest(mergeLatest(analysis1, analysis2), mergeLatest(analysis1, analysis2)) ); } @@ -1161,115 +564,29 @@ public Interval getTrueInterval() } @SuppressWarnings("ArgumentParameterSwap") - @Test - public void testMergeRollup() + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testMergeRollup(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysis( - TEST_SEGMENT_ID1.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - null - ); - final SegmentAnalysis analysis2 = new SegmentAnalysis( - TEST_SEGMENT_ID2.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - false - ); - final SegmentAnalysis analysis3 = new SegmentAnalysis( - TEST_SEGMENT_ID1.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - false - ); - final SegmentAnalysis analysis4 = new SegmentAnalysis( - TEST_SEGMENT_ID2.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - true - ); - final SegmentAnalysis analysis5 = new SegmentAnalysis( - TEST_SEGMENT_ID1.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - true - ); - - Assert.assertNull(mergeStrict(analysis1, analysis2).isRollup()); - Assert.assertNull(mergeStrict(analysis1, analysis4).isRollup()); - Assert.assertNull(mergeStrict(analysis2, analysis4).isRollup()); - Assert.assertFalse(mergeStrict(analysis2, analysis3).isRollup()); - Assert.assertTrue(mergeStrict(analysis4, analysis5).isRollup()); - - Assert.assertNull(mergeLenient(analysis1, analysis2).isRollup()); - Assert.assertNull(mergeLenient(analysis1, analysis4).isRollup()); - Assert.assertNull(mergeLenient(analysis2, analysis4).isRollup()); - Assert.assertFalse(mergeLenient(analysis2, analysis3).isRollup()); - Assert.assertTrue(mergeLenient(analysis4, analysis5).isRollup()); - - Assert.assertNull(mergeEarliest(analysis1, analysis2).isRollup()); - Assert.assertNull(mergeEarliest(analysis1, analysis4).isRollup()); - Assert.assertNull(mergeEarliest(analysis2, analysis4).isRollup()); - Assert.assertFalse(mergeEarliest(analysis2, analysis3).isRollup()); - Assert.assertTrue(mergeEarliest(analysis4, analysis5).isRollup()); - - Assert.assertNull(mergeLatest(analysis1, analysis2).isRollup()); - Assert.assertNull(mergeLatest(analysis1, analysis4).isRollup()); - Assert.assertNull(mergeLatest(analysis2, analysis4).isRollup()); - Assert.assertFalse(mergeLatest(analysis2, analysis3).isRollup()); - Assert.assertTrue(mergeLatest(analysis4, analysis5).isRollup()); + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).rollup(false).build(); + final SegmentAnalysis analysis3 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).rollup(false).build(); + final SegmentAnalysis analysis4 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).rollup(true).build(); + final SegmentAnalysis analysis5 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).rollup(true).build(); + + Assert.assertNull(mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy).isRollup()); + Assert.assertNull(mergeWithStrategy(analysis1, analysis4, aggregatorMergeStrategy).isRollup()); + Assert.assertNull(mergeWithStrategy(analysis2, analysis4, aggregatorMergeStrategy).isRollup()); + Assert.assertFalse(mergeWithStrategy(analysis2, analysis3, aggregatorMergeStrategy).isRollup()); + Assert.assertTrue(mergeWithStrategy(analysis4, analysis5, aggregatorMergeStrategy).isRollup()); } - @Test - public void testInvalidMergeAggregatorsWithNullOrEmptyDatasource() + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testInvalidMergeAggregatorsWithNullOrEmptyDatasource(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysis( - TEST_SEGMENT_ID1.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - null - ); - final SegmentAnalysis analysis2 = new SegmentAnalysis( - TEST_SEGMENT_ID2.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - false - ); + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).build(); MatcherAssert.assertThat( Assert.assertThrows( @@ -1278,13 +595,10 @@ public void testInvalidMergeAggregatorsWithNullOrEmptyDatasource() null, analysis1, analysis2, - AggregatorMergeStrategy.STRICT + aggregatorMergeStrategy ) ), - DruidExceptionMatcher - .invalidInput() - .expectMessageIs( - "SegementMetadata queries require at least one datasource.") + DruidExceptionMatcher.defensive().expectMessageIs("SegementMetadata queries require at least one datasource.") ); MatcherAssert.assertThat( @@ -1294,201 +608,113 @@ public void testInvalidMergeAggregatorsWithNullOrEmptyDatasource() ImmutableSet.of(), analysis1, analysis2, - AggregatorMergeStrategy.STRICT + aggregatorMergeStrategy ) ), DruidExceptionMatcher - .invalidInput() + .defensive() .expectMessageIs( "SegementMetadata queries require at least one datasource.") ); } - - @Test - public void testMergeWithUnionDatasource() + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testMergeWithUnionDatasource(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysis( - TEST_SEGMENT_ID1.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleSumAggregatorFactory("bar", "bar") - ), - null, - null, - null - ); - final SegmentAnalysis analysis2 = new SegmentAnalysis( - TEST_SEGMENT_ID2.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleMaxAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - false - ); - - final SegmentAnalysis expectedMergedAnalysis = new SegmentAnalysis( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged", - null, - new LinkedHashMap<>(), - 0, - 0, - ImmutableMap.of( - "foo", new LongSumAggregatorFactory("foo", "foo"), - "bar", new DoubleMaxAggregatorFactory("bar", "bar"), - "baz", new LongMaxAggregatorFactory("baz", "baz") - ), - null, - null, - null - ); + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .build(); + + final SegmentAnalysis expectedMergedAnalysis = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) + .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) + .build(); + + UnionDataSource dataSource1 = new UnionDataSource(ImmutableList.of( + new TableDataSource("foo"), + new TableDataSource("dummy") + )); + UnionDataSource dataSource2 = new UnionDataSource(ImmutableList.of( + new TableDataSource("dummy"), + new TableDataSource("foo"), + new TableDataSource("bar") + )); Assert.assertEquals( expectedMergedAnalysis, - SegmentMetadataQueryQueryToolChest.finalizeAnalysis( - SegmentMetadataQueryQueryToolChest.mergeAnalyses( - new UnionDataSource( - ImmutableList.of( - new TableDataSource("foo"), - new TableDataSource("dummy") - ) - ).getTableNames(), - analysis1, - analysis2, - AggregatorMergeStrategy.LATEST - ) - ) + SegmentMetadataQueryQueryToolChest.finalizeAnalysis(SegmentMetadataQueryQueryToolChest.mergeAnalyses( + dataSource1.getTableNames(), + analysis1, + analysis2, + aggregatorMergeStrategy + )) ); - Assert.assertEquals( expectedMergedAnalysis, SegmentMetadataQueryQueryToolChest.finalizeAnalysis( SegmentMetadataQueryQueryToolChest.mergeAnalyses( - new UnionDataSource( - ImmutableList.of( - new TableDataSource("dummy"), - new TableDataSource("foo"), - new TableDataSource("bar") - ) - ).getTableNames(), + dataSource2.getTableNames(), analysis1, analysis2, - AggregatorMergeStrategy.LATEST + aggregatorMergeStrategy ) ) ); } - @Test - public void testMergeWithNullAnalyses() + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testMergeWithNullAnalyses(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysis( - TEST_SEGMENT_ID1.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - null - ); - final SegmentAnalysis analysis2 = new SegmentAnalysis( - TEST_SEGMENT_ID2.toString(), - null, - new LinkedHashMap<>(), - 0, - 0, - null, - null, - null, - false - ); + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).build(); - Assert.assertEquals( - analysis1, - SegmentMetadataQueryQueryToolChest - .mergeAnalyses(TEST_DATASOURCE.getTableNames(), analysis1, null, AggregatorMergeStrategy.STRICT) - ); - Assert.assertEquals( - analysis2, - SegmentMetadataQueryQueryToolChest - .mergeAnalyses(TEST_DATASOURCE.getTableNames(), null, analysis2, AggregatorMergeStrategy.STRICT) - ); - Assert.assertNull( - SegmentMetadataQueryQueryToolChest - .mergeAnalyses(TEST_DATASOURCE.getTableNames(), null, null, AggregatorMergeStrategy.STRICT) - ); + Assert.assertEquals(analysis1, mergeWithStrategy(analysis1, null, aggregatorMergeStrategy)); + Assert.assertEquals(analysis2, mergeWithStrategy(null, analysis2, aggregatorMergeStrategy)); Assert.assertNull( SegmentMetadataQueryQueryToolChest - .mergeAnalyses(TEST_DATASOURCE.getTableNames(), null, null, AggregatorMergeStrategy.LENIENT) - ); - Assert.assertNull( - SegmentMetadataQueryQueryToolChest - .mergeAnalyses(TEST_DATASOURCE.getTableNames(), null, null, AggregatorMergeStrategy.EARLIEST) - ); - Assert.assertNull( - SegmentMetadataQueryQueryToolChest - .mergeAnalyses(TEST_DATASOURCE.getTableNames(), null, null, AggregatorMergeStrategy.LATEST) - ); + .mergeAnalyses(TEST_DATASOURCE.getTableNames(), null, null, aggregatorMergeStrategy)); } - private static SegmentAnalysis mergeStrict(SegmentAnalysis analysis1, SegmentAnalysis analysis2) + private static SegmentAnalysis mergeWithStrategy( + SegmentAnalysis analysis1, + SegmentAnalysis analysis2, + AggregatorMergeStrategy strategy + ) { return SegmentMetadataQueryQueryToolChest.finalizeAnalysis( SegmentMetadataQueryQueryToolChest.mergeAnalyses( TEST_DATASOURCE.getTableNames(), analysis1, analysis2, - AggregatorMergeStrategy.STRICT - ) - ); + strategy + )); + } + + private static SegmentAnalysis mergeStrict(SegmentAnalysis analysis1, SegmentAnalysis analysis2) + { + return mergeWithStrategy(analysis1, analysis2, AggregatorMergeStrategy.STRICT); } private static SegmentAnalysis mergeLenient(SegmentAnalysis analysis1, SegmentAnalysis analysis2) { - return SegmentMetadataQueryQueryToolChest.finalizeAnalysis( - SegmentMetadataQueryQueryToolChest.mergeAnalyses( - TEST_DATASOURCE.getTableNames(), - analysis1, - analysis2, - AggregatorMergeStrategy.LENIENT - ) - ); + return mergeWithStrategy(analysis1, analysis2, AggregatorMergeStrategy.LENIENT); } private static SegmentAnalysis mergeEarliest(SegmentAnalysis analysis1, SegmentAnalysis analysis2) { - return SegmentMetadataQueryQueryToolChest.finalizeAnalysis( - SegmentMetadataQueryQueryToolChest.mergeAnalyses( - TEST_DATASOURCE.getTableNames(), - analysis1, - analysis2, - AggregatorMergeStrategy.EARLIEST - ) - ); + return mergeWithStrategy(analysis1, analysis2, AggregatorMergeStrategy.EARLIEST); } private static SegmentAnalysis mergeLatest(SegmentAnalysis analysis1, SegmentAnalysis analysis2) { - return SegmentMetadataQueryQueryToolChest.finalizeAnalysis( - SegmentMetadataQueryQueryToolChest.mergeAnalyses( - TEST_DATASOURCE.getTableNames(), - analysis1, - analysis2, - AggregatorMergeStrategy.LATEST - ) - ); + return mergeWithStrategy(analysis1, analysis2, AggregatorMergeStrategy.LATEST); } } diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java index 47decc388e2b..172f1927657f 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java @@ -62,6 +62,7 @@ import org.apache.druid.query.policy.NoRestrictionPolicy; import org.apache.druid.query.policy.RowFilterPolicy; import org.apache.druid.query.spec.LegacySegmentSpec; +import org.apache.druid.segment.AggregateProjectionMetadata; import org.apache.druid.segment.IncrementalIndexSegment; import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.QueryableIndexSegment; @@ -205,7 +206,8 @@ public SegmentMetadataQueryTest( SegmentMetadataQuery.AnalysisType.SIZE, SegmentMetadataQuery.AnalysisType.INTERVAL, SegmentMetadataQuery.AnalysisType.MINMAX, - SegmentMetadataQuery.AnalysisType.AGGREGATORS + SegmentMetadataQuery.AnalysisType.AGGREGATORS, + SegmentMetadataQuery.AnalysisType.PROJECTIONS ) .merge(true) .build(); @@ -221,6 +223,11 @@ public SegmentMetadataQueryTest( for (AggregatorFactory agg : TestIndex.METRIC_AGGS) { expectedAggregators.put(agg.getName(), agg.getCombiningFactory()); } + final AggregateProjectionMetadata.Schema projectionSchema = TestIndex.PROJECTIONS.get(0).toMetadataSchema(); + final Map expectedProjections = ImmutableMap.of( + projectionSchema.getName(), + new AggregateProjectionMetadata(projectionSchema, 279) + ); expectedSegmentAnalysis1 = new SegmentAnalysis( id1.toString(), @@ -268,6 +275,7 @@ public SegmentMetadataQueryTest( overallSize, 1209, expectedAggregators, + expectedProjections, null, null, null @@ -318,6 +326,7 @@ public SegmentMetadataQueryTest( overallSize, 1209, expectedAggregators, + expectedProjections, null, null, null @@ -399,6 +408,7 @@ public void testSegmentMetadataQueryWithRollupMerge() null, null, null, + null, rollup1 != rollup2 ? null : rollup1 ); @@ -473,6 +483,7 @@ public void testSegmentMetadataQueryWithHasMultipleValuesMerge() null, null, null, + null, null ); @@ -547,6 +558,7 @@ public void testSegmentMetadataQueryWithComplexColumnMerge() null, null, null, + null, null ); @@ -691,6 +703,8 @@ private void testSegmentMetadataQueryWithDefaultAnalysisMerge( expectedSegmentAnalysis1.getSize() + expectedSegmentAnalysis2.getSize(), expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows(), expectedAggregators, + AggregateProjectionMetadata.merge(expectedSegmentAnalysis1.getProjections(), + expectedSegmentAnalysis2.getProjections()), null, null, null @@ -749,6 +763,7 @@ public void testSegmentMetadataQueryWithNoAnalysisTypesMerge() null, null, null, + null, null ); @@ -815,6 +830,7 @@ public void testSegmentMetadataQueryWithAggregatorsMerge() expectedAggregators, null, null, + null, null ); @@ -881,6 +897,7 @@ public void testSegmentMetadataQueryWithAggregatorsMergeLenientStrategy() expectedAggregators, null, null, + null, null ); @@ -942,6 +959,7 @@ public void testSegmentMetadataQueryWithTimestampSpecMerge() 0, expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows(), null, + null, new TimestampSpec("ts", "iso", null), null, null @@ -1005,6 +1023,7 @@ public void testSegmentMetadataQueryWithQueryGranularityMerge() expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows(), null, null, + null, Granularities.NONE, null ); diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java index 33147ce01b0e..7082dd744ed4 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataUnionQueryTest.java @@ -122,6 +122,7 @@ public void testSegmentMetadataUnionQuery() null, null, null, + null, null ); SegmentMetadataQuery query = new Druids.SegmentMetadataQueryBuilder() diff --git a/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java b/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java index 636df8691b39..844929e94068 100644 --- a/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java +++ b/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; import it.unimi.dsi.fastutil.objects.ObjectAVLTreeSet; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.error.DruidException; @@ -28,6 +29,7 @@ import org.apache.druid.query.OrderBy; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.LongMaxAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; @@ -72,6 +74,76 @@ public void testSerde() throws JsonProcessingException ); } + @Test + public void testMerge() + { + AggregateProjectionMetadata countChocula = new AggregateProjectionMetadata( + new AggregateProjectionMetadata.Schema( + "countChocula", + "theTime", + VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "theTime")), + Arrays.asList("theTime", "a", "b", "c"), + new AggregatorFactory[]{new CountAggregatorFactory("chocula")}, + Arrays.asList( + OrderBy.ascending("theTime"), + OrderBy.ascending("a"), + OrderBy.ascending("b"), + OrderBy.ascending("c") + ) + ), + 123 + ); + AggregateProjectionMetadata sumChocula = new AggregateProjectionMetadata( + new AggregateProjectionMetadata.Schema( + "sumChocula", + "theTime", + VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "theTime")), + Arrays.asList("theTime", "a", "b", "c"), + new AggregatorFactory[]{new LongSumAggregatorFactory("sumChocula", "chocula")}, + Arrays.asList( + OrderBy.ascending("theTime"), + OrderBy.ascending("a"), + OrderBy.ascending("b"), + OrderBy.ascending("c") + ) + ), + 123 + ); + AggregateProjectionMetadata maxChocula = new AggregateProjectionMetadata( + new AggregateProjectionMetadata.Schema( + "maxChocula", + "theTime", + VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "theTime")), + Arrays.asList("theTime", "a", "b", "c"), + new AggregatorFactory[]{new LongMaxAggregatorFactory("maxChocula", "chocula")}, + Arrays.asList( + OrderBy.ascending("theTime"), + OrderBy.ascending("a"), + OrderBy.ascending("b"), + OrderBy.ascending("c") + ) + ), + 123 + ); + + ImmutableMap metadata1 = ImmutableMap.of( + "countChocula", countChocula, + "sumChocula", sumChocula + ); + ImmutableMap metadata2 = ImmutableMap.of( + "countChocula", countChocula, + "maxChocula", maxChocula + ); + + ImmutableMap expected = ImmutableMap.of( + "countChocula", + new AggregateProjectionMetadata( + countChocula.getSchema(), + 246 + ) + ); + Assert.assertEquals(expected, AggregateProjectionMetadata.merge(metadata1, metadata2)); + } @Test public void testComparator() @@ -83,9 +155,7 @@ public void testComparator() "theTime", VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "theTime")), Arrays.asList("theTime", "a", "b", "c"), - new AggregatorFactory[] { - new CountAggregatorFactory("chocula") - }, + new AggregatorFactory[]{new CountAggregatorFactory("chocula")}, Arrays.asList( OrderBy.ascending("theTime"), OrderBy.ascending("a"), @@ -95,14 +165,30 @@ public void testComparator() ), 123 ); - // same row count, but more aggs more better - AggregateProjectionMetadata better = new AggregateProjectionMetadata( + // same row count, but less grouping columns aggs more better + AggregateProjectionMetadata betterLessGroupingColumns = new AggregateProjectionMetadata( + new AggregateProjectionMetadata.Schema( + "betterLessGroupingColumns", + "theTime", + VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "theTime")), + Arrays.asList("c", "d", "theTime"), + new AggregatorFactory[]{new CountAggregatorFactory("chocula")}, + Arrays.asList( + OrderBy.ascending("c"), + OrderBy.ascending("d"), + OrderBy.ascending("theTime") + ) + ), + 123 + ); + // same grouping columns, but more aggregators + AggregateProjectionMetadata evenBetterMoreAggs = new AggregateProjectionMetadata( new AggregateProjectionMetadata.Schema( - "better", + "evenBetterMoreAggs", "theTime", VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "theTime")), Arrays.asList("c", "d", "theTime"), - new AggregatorFactory[] { + new AggregatorFactory[]{ new CountAggregatorFactory("chocula"), new LongSumAggregatorFactory("e", "e") }, @@ -114,11 +200,10 @@ public void testComparator() ), 123 ); - // small rows is best AggregateProjectionMetadata best = new AggregateProjectionMetadata( new AggregateProjectionMetadata.Schema( - "better", + "best", null, VirtualColumns.EMPTY, Arrays.asList("f", "g"), @@ -128,10 +213,14 @@ public void testComparator() 10 ); metadataBest.add(good); - metadataBest.add(better); + metadataBest.add(betterLessGroupingColumns); + metadataBest.add(evenBetterMoreAggs); metadataBest.add(best); Assert.assertEquals(best, metadataBest.first()); - Assert.assertEquals(good, metadataBest.last()); + Assert.assertEquals( + new AggregateProjectionMetadata[]{best, evenBetterMoreAggs, betterLessGroupingColumns, good}, + metadataBest.toArray() + ); } @Test @@ -141,12 +230,12 @@ public void testInvalidGrouping() DruidException.class, () -> new AggregateProjectionMetadata( new AggregateProjectionMetadata.Schema( - "other_projection", - null, - null, - null, - null, - null + "other_projection", + null, + null, + null, + null, + null ), 0 ) @@ -157,12 +246,12 @@ public void testInvalidGrouping() DruidException.class, () -> new AggregateProjectionMetadata( new AggregateProjectionMetadata.Schema( - "other_projection", - null, - null, - Collections.emptyList(), - null, - null + "other_projection", + null, + null, + Collections.emptyList(), + null, + null ), 0 ) diff --git a/processing/src/test/java/org/apache/druid/segment/MetadataTest.java b/processing/src/test/java/org/apache/druid/segment/MetadataTest.java index 5d06d2ab8c25..23318406bb47 100644 --- a/processing/src/test/java/org/apache/druid/segment/MetadataTest.java +++ b/processing/src/test/java/org/apache/druid/segment/MetadataTest.java @@ -66,8 +66,18 @@ public void testSerde() throws Exception null ); + // Empty projection is not included in the json object + Metadata metadataWithEmptyProjection = new Metadata( + Collections.singletonMap("k", "v"), + aggregators, + null, + Granularities.ALL, + Boolean.FALSE, + null, + ImmutableList.of() + ); Metadata other = jsonMapper.readValue( - jsonMapper.writeValueAsString(metadata), + jsonMapper.writeValueAsString(metadataWithEmptyProjection), Metadata.class ); diff --git a/processing/src/test/java/org/apache/druid/segment/TestIndex.java b/processing/src/test/java/org/apache/druid/segment/TestIndex.java index 83f1ec476495..8ade3c4d28be 100644 --- a/processing/src/test/java/org/apache/druid/segment/TestIndex.java +++ b/processing/src/test/java/org/apache/druid/segment/TestIndex.java @@ -27,6 +27,7 @@ import com.google.common.io.Resources; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.ResourceInputSource; +import org.apache.druid.data.input.impl.AggregateProjectionSpec; import org.apache.druid.data.input.impl.DelimitedParseSpec; import org.apache.druid.data.input.impl.DimensionSchema; import org.apache.druid.data.input.impl.DimensionsSpec; @@ -40,6 +41,7 @@ import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.common.parsers.JSONPathSpec; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -179,6 +181,17 @@ public class TestIndex new DoubleMaxAggregatorFactory(DOUBLE_METRICS[2], VIRTUAL_COLUMNS.getVirtualColumns()[0].getOutputName()), new HyperUniquesAggregatorFactory("quality_uniques", "quality") }; + public static final ImmutableList PROJECTIONS = ImmutableList.of( + new AggregateProjectionSpec( + "index_projection", + VirtualColumns.create(Granularities.toVirtualColumn(Granularities.DAY, "__gran")), + Arrays.asList( + new LongDimensionSchema("__gran"), + new StringDimensionSchema("market") + ), + new AggregatorFactory[]{new DoubleSumAggregatorFactory(DOUBLE_METRICS[0], "index")} + ) + ); public static final IndexSpec INDEX_SPEC = IndexSpec.DEFAULT; public static final JsonInputFormat DEFAULT_JSON_INPUT_FORMAT = new JsonInputFormat( @@ -406,9 +419,14 @@ private static IncrementalIndex fromJsonResource( new IncrementalIndexSchema.Builder() .withMinTimestamp(DateTimes.of("2011-01-12T00:00:00.000Z").getMillis()) .withTimestampSpec(new TimestampSpec("ts", "iso", null)) - .withDimensionsSpec(DimensionsSpec.builder().setDimensions(dimensionsSpec.getDimensions()).setDimensionExclusions(ImmutableList.of("index")).setIncludeAllDimensions(true).build()) + .withDimensionsSpec(DimensionsSpec.builder() + .setDimensions(dimensionsSpec.getDimensions()) + .setDimensionExclusions(ImmutableList.of("index")) + .setIncludeAllDimensions(true) + .build()) .withVirtualColumns(VIRTUAL_COLUMNS) .withMetrics(METRIC_AGGS) + .withProjections(PROJECTIONS) .withRollup(rollup) .build(), DEFAULT_JSON_INPUT_FORMAT @@ -523,6 +541,7 @@ public static IncrementalIndex makeIncrementalIndexFromTsvCharSource(final CharS .withDimensionsSpec(DIMENSIONS_SPEC) .withVirtualColumns(VIRTUAL_COLUMNS) .withMetrics(METRIC_AGGS) + .withProjections(PROJECTIONS) .withRollup(true) .build(); final IncrementalIndex retVal = new OnheapIncrementalIndex.Builder() diff --git a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java index 8d63da5f757e..a62114a62a50 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java @@ -1113,6 +1113,7 @@ public void testSegmentMetadataColumnType() null, null, null, + null, null ) ); @@ -1179,6 +1180,7 @@ public void testSegmentMetadataFallbackType() null, null, null, + null, null ) ); From 94fea7a624f2c655c8d1fe0046f274e1e41cb74e Mon Sep 17 00:00:00 2001 From: cecemei Date: Wed, 11 Jun 2025 08:56:39 -0700 Subject: [PATCH 2/9] fix merge --- .../main/java/org/apache/druid/segment/IndexMergerV9.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java index 846ed5cec9f2..7d2e4b2480f8 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java @@ -1352,10 +1352,10 @@ private File merge( List metadataList = Lists.transform(indexes, IndexableAdapter::getMetadata); final Metadata segmentMetadata; - if (metricAggs != null) { - AggregatorFactory[] combiningMetricAggs = new AggregatorFactory[metricAggs.length]; - for (int i = 0; i < metricAggs.length; i++) { - combiningMetricAggs[i] = metricAggs[i].getCombiningFactory(); + if (sortedMetricAggs != null) { + AggregatorFactory[] combiningMetricAggs = new AggregatorFactory[sortedMetricAggs.length]; + for (int i = 0; i < sortedMetricAggs.length; i++) { + combiningMetricAggs[i] = sortedMetricAggs[i].getCombiningFactory(); } segmentMetadata = Metadata.merge( metadataList, From de7d0ad70299a7a53f4d242d78066f3303eeecf4 Mon Sep 17 00:00:00 2001 From: cecemei Date: Wed, 11 Jun 2025 18:48:09 -0700 Subject: [PATCH 3/9] fix test --- .../src/test/java/org/apache/druid/segment/TestIndex.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/segment/TestIndex.java b/processing/src/test/java/org/apache/druid/segment/TestIndex.java index 8ade3c4d28be..1eb5e493b5bb 100644 --- a/processing/src/test/java/org/apache/druid/segment/TestIndex.java +++ b/processing/src/test/java/org/apache/druid/segment/TestIndex.java @@ -51,6 +51,7 @@ import org.apache.druid.query.aggregation.FloatMaxAggregatorFactory; import org.apache.druid.query.aggregation.FloatMinAggregatorFactory; import org.apache.druid.query.aggregation.FloatSumAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.DoubleFirstAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde; import org.apache.druid.query.expression.TestExprMacroTable; @@ -181,7 +182,7 @@ public class TestIndex new DoubleMaxAggregatorFactory(DOUBLE_METRICS[2], VIRTUAL_COLUMNS.getVirtualColumns()[0].getOutputName()), new HyperUniquesAggregatorFactory("quality_uniques", "quality") }; - public static final ImmutableList PROJECTIONS = ImmutableList.of( + public static final ImmutableList PROJECTIONS = ImmutableList.of( new AggregateProjectionSpec( "index_projection", VirtualColumns.create(Granularities.toVirtualColumn(Granularities.DAY, "__gran")), @@ -189,7 +190,7 @@ public class TestIndex new LongDimensionSchema("__gran"), new StringDimensionSchema("market") ), - new AggregatorFactory[]{new DoubleSumAggregatorFactory(DOUBLE_METRICS[0], "index")} + new AggregatorFactory[]{new DoubleMaxAggregatorFactory("maxQuality", "qualityLong")} ) ); public static final IndexSpec INDEX_SPEC = IndexSpec.DEFAULT; From 884671a9cb20d044530c03c7596e88f8d3f73813 Mon Sep 17 00:00:00 2001 From: cecemei Date: Thu, 12 Jun 2025 10:38:50 -0700 Subject: [PATCH 4/9] fix test and style --- .../test/resources/queries/wikipedia_editstream_queries.json | 1 + .../src/test/java/org/apache/druid/segment/TestIndex.java | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json b/integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json index 4cb4c0ec4857..54901737a913 100644 --- a/integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json +++ b/integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json @@ -1433,6 +1433,7 @@ "size":0, "numRows":4462111, "aggregators":null, + "projections":null, "timestampSpec":null, "queryGranularity":null, "rollup":null diff --git a/processing/src/test/java/org/apache/druid/segment/TestIndex.java b/processing/src/test/java/org/apache/druid/segment/TestIndex.java index 1eb5e493b5bb..bb9c52b2d200 100644 --- a/processing/src/test/java/org/apache/druid/segment/TestIndex.java +++ b/processing/src/test/java/org/apache/druid/segment/TestIndex.java @@ -51,7 +51,6 @@ import org.apache.druid.query.aggregation.FloatMaxAggregatorFactory; import org.apache.druid.query.aggregation.FloatMinAggregatorFactory; import org.apache.druid.query.aggregation.FloatSumAggregatorFactory; -import org.apache.druid.query.aggregation.firstlast.first.DoubleFirstAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde; import org.apache.druid.query.expression.TestExprMacroTable; @@ -182,7 +181,7 @@ public class TestIndex new DoubleMaxAggregatorFactory(DOUBLE_METRICS[2], VIRTUAL_COLUMNS.getVirtualColumns()[0].getOutputName()), new HyperUniquesAggregatorFactory("quality_uniques", "quality") }; - public static final ImmutableList PROJECTIONS = ImmutableList.of( + public static final ImmutableList PROJECTIONS = ImmutableList.of( new AggregateProjectionSpec( "index_projection", VirtualColumns.create(Granularities.toVirtualColumn(Granularities.DAY, "__gran")), From 3e022413c6c9fee9350f72294dd0686edc30d68a Mon Sep 17 00:00:00 2001 From: cecemei Date: Thu, 12 Jun 2025 17:57:35 -0700 Subject: [PATCH 5/9] cleanup --- .../SegmentMetadataQueryRunnerFactory.java | 1 - .../apache/druid/segment/IndexMergerV9.java | 19 +-- ...egmentMetadataQueryQueryToolChestTest.java | 127 +++++++++--------- 3 files changed, 68 insertions(+), 79 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java index ab4305d776dd..6320df7d06c0 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java @@ -136,7 +136,6 @@ public Sequence run(QueryPlus inQ, ResponseCon } final Map projectionsMap; - if (updatedQuery.hasProjections() && ((metadata = Objects.isNull(metadata) ? getMetadata(segment) : metadata)) != null && metadata.getProjections() != null) { diff --git a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java index 7d2e4b2480f8..faf4df5806f0 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java @@ -1351,22 +1351,11 @@ private File merge( } List metadataList = Lists.transform(indexes, IndexableAdapter::getMetadata); - final Metadata segmentMetadata; - if (sortedMetricAggs != null) { - AggregatorFactory[] combiningMetricAggs = new AggregatorFactory[sortedMetricAggs.length]; - for (int i = 0; i < sortedMetricAggs.length; i++) { - combiningMetricAggs[i] = sortedMetricAggs[i].getCombiningFactory(); - } - segmentMetadata = Metadata.merge( - metadataList, - combiningMetricAggs - ); - } else { - segmentMetadata = Metadata.merge( - metadataList, - null - ); + AggregatorFactory[] combiningMetricAggs = new AggregatorFactory[sortedMetricAggs.length]; + for (int i = 0; i < sortedMetricAggs.length; i++) { + combiningMetricAggs[i] = sortedMetricAggs[i].getCombiningFactory(); } + final Metadata segmentMetadata = Metadata.merge(metadataList, combiningMetricAggs); if (segmentMetadata != null && segmentMetadata.getOrdering() != null diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java index 525616bb8545..8c06ee3863f0 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java @@ -71,7 +71,7 @@ public class SegmentMetadataQueryQueryToolChestTest private static final SegmentId TEST_SEGMENT_ID2 = SegmentId.of(TEST_DATASOURCE.toString(), INTERVAL_2021, "test", 0); private static final AggregateProjectionMetadata.Schema PROJECTION_CHANNEL_ADDED_HOURLY = new AggregateProjectionMetadata.Schema( - "name1-doesnot-matter", + "name1-does-not-matter", Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, VirtualColumns.create(Granularities.toVirtualColumn( Granularities.HOUR, @@ -87,7 +87,7 @@ public class SegmentMetadataQueryQueryToolChestTest ) ); private static final AggregateProjectionMetadata.Schema PROJECTION_CHANNEL_ADDED_DAILY = new AggregateProjectionMetadata.Schema( - "name2-doesnot-matter", + "name2-does-not-matter", Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME, VirtualColumns.create(Granularities.toVirtualColumn( Granularities.DAY, @@ -159,67 +159,6 @@ public void testCacheStrategy() throws Exception Assert.assertEquals(result, fromCacheResult); } - @EnumSource(AggregatorMergeStrategy.class) - @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") - public void testProjections(AggregatorMergeStrategy aggregatorMergeStrategy) - { - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) - .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) - .build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) - .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 200)) - .build(); - - final SegmentAnalysis expected = new SegmentAnalysisBuilder( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") - .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 300)) - .build(); - Assert.assertEquals(expected, mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy)); - } - - @EnumSource(AggregatorMergeStrategy.class) - @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") - public void testProjectionsWithNull(AggregatorMergeStrategy aggregatorMergeStrategy) - { - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) - .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) - .build(); - final SegmentAnalysis analysis1NullProjection = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) - .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 200)) - .build(); - final SegmentAnalysis analysis2NullProjection = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).build(); - - Assert.assertNull(mergeWithStrategy(analysis1NullProjection, analysis2, aggregatorMergeStrategy).getProjections()); - Assert.assertNull(mergeWithStrategy(analysis1, analysis2NullProjection, aggregatorMergeStrategy).getProjections()); - Assert.assertNull( - mergeWithStrategy(analysis1NullProjection, analysis2NullProjection, aggregatorMergeStrategy).getProjections() - ); - } - - @EnumSource(AggregatorMergeStrategy.class) - @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") - public void testProjectionsWithConflict(AggregatorMergeStrategy aggregatorMergeStrategy) - { - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) - .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) - .projection("channel_sum_1", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) - .projection("conflict_projection", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) - .build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) - .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 200)) - .projection("channel_sum_2", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_DAILY, 200)) - .projection("conflict_projection", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_DAILY, 200)) - .build(); - - // conflict projection is ignored. - final SegmentAnalysis expectedStrict = new SegmentAnalysisBuilder( - "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") - .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 300)) - .build(); - Assert.assertEquals(expectedStrict, mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy)); - } - @EnumSource(AggregatorMergeStrategy.class) @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") public void testMergeAggregators(AggregatorMergeStrategy aggregatorMergeStrategy) @@ -683,6 +622,68 @@ public void testMergeWithNullAnalyses(AggregatorMergeStrategy aggregatorMergeStr .mergeAnalyses(TEST_DATASOURCE.getTableNames(), null, null, aggregatorMergeStrategy)); } + + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testProjections(AggregatorMergeStrategy aggregatorMergeStrategy) + { + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) + .build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 200)) + .build(); + + final SegmentAnalysis expected = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 300)) + .build(); + Assert.assertEquals(expected, mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy)); + } + + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testProjectionsWithNull(AggregatorMergeStrategy aggregatorMergeStrategy) + { + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) + .build(); + final SegmentAnalysis analysis1NullProjection = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 200)) + .build(); + final SegmentAnalysis analysis2NullProjection = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).build(); + + Assert.assertNull(mergeWithStrategy(analysis1NullProjection, analysis2, aggregatorMergeStrategy).getProjections()); + Assert.assertNull(mergeWithStrategy(analysis1, analysis2NullProjection, aggregatorMergeStrategy).getProjections()); + Assert.assertNull( + mergeWithStrategy(analysis1NullProjection, analysis2NullProjection, aggregatorMergeStrategy).getProjections() + ); + } + + @EnumSource(AggregatorMergeStrategy.class) + @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") + public void testProjectionsWithConflict(AggregatorMergeStrategy aggregatorMergeStrategy) + { + final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) + .projection("channel_sum_1", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) + .projection("conflict_projection", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) + .build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 200)) + .projection("channel_sum_2", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_DAILY, 200)) + .projection("conflict_projection", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_DAILY, 200)) + .build(); + + // conflict projection is ignored. + final SegmentAnalysis expectedStrict = new SegmentAnalysisBuilder( + "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 300)) + .build(); + Assert.assertEquals(expectedStrict, mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy)); + } + private static SegmentAnalysis mergeWithStrategy( SegmentAnalysis analysis1, SegmentAnalysis analysis2, From 40421f96299afa48e64698b9e155841ba13b41f6 Mon Sep 17 00:00:00 2001 From: cecemei Date: Thu, 12 Jun 2025 18:26:23 -0700 Subject: [PATCH 6/9] more cleanup --- .../metadata/SegmentMetadataQueryQueryToolChestTest.java | 2 +- .../druid/segment/AggregateProjectionMetadataTest.java | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java index 8c06ee3863f0..03bf90d3e43f 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java @@ -613,7 +613,7 @@ public void testMergeWithUnionDatasource(AggregatorMergeStrategy aggregatorMerge public void testMergeWithNullAnalyses(AggregatorMergeStrategy aggregatorMergeStrategy) { final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).rollup(false).build(); Assert.assertEquals(analysis1, mergeWithStrategy(analysis1, null, aggregatorMergeStrategy)); Assert.assertEquals(analysis2, mergeWithStrategy(null, analysis2, aggregatorMergeStrategy)); diff --git a/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java b/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java index 844929e94068..3c54072658a3 100644 --- a/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java +++ b/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java @@ -128,11 +128,13 @@ public void testMerge() ImmutableMap metadata1 = ImmutableMap.of( "countChocula", countChocula, - "sumChocula", sumChocula + "chocula-conflict", sumChocula, + "sumChocula-only-in-metadata1", sumChocula ); ImmutableMap metadata2 = ImmutableMap.of( "countChocula", countChocula, - "maxChocula", maxChocula + "chocula-conflict", maxChocula, + "maxChocula-only-in-metadata2", maxChocula ); ImmutableMap expected = ImmutableMap.of( @@ -217,7 +219,7 @@ public void testComparator() metadataBest.add(evenBetterMoreAggs); metadataBest.add(best); Assert.assertEquals(best, metadataBest.first()); - Assert.assertEquals( + Assert.assertArrayEquals( new AggregateProjectionMetadata[]{best, evenBetterMoreAggs, betterLessGroupingColumns, good}, metadataBest.toArray() ); From df72efe4641d57b6564e29cb713b499f20c49684 Mon Sep 17 00:00:00 2001 From: cecemei Date: Thu, 12 Jun 2025 19:26:24 -0700 Subject: [PATCH 7/9] fix dependency --- .../query/metadata/SegmentMetadataQueryQueryToolChest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java index fde69d13a9c7..057e4453137d 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java @@ -30,7 +30,6 @@ import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import com.google.inject.Inject; -import org.apache.commons.collections.MapUtils; import org.apache.druid.common.guava.CombiningSequence; import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.error.DruidException; @@ -450,7 +449,7 @@ public static SegmentAnalysis mergeAnalyses( arg1.getSize() + arg2.getSize(), arg1.getNumRows() + arg2.getNumRows(), aggregators.isEmpty() ? null : aggregators, - MapUtils.isEmpty(projections) ? null : projections, + (projections == null || projections.isEmpty()) ? null : projections, timestampSpec, queryGranularity, rollup From 292edbedcd334047071db5746312e2fa54ea5aab Mon Sep 17 00:00:00 2001 From: cecemei Date: Fri, 13 Jun 2025 11:00:49 -0700 Subject: [PATCH 8/9] fix twitter query test --- .../src/test/resources/queries/twitterstream_queries.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration-tests/src/test/resources/queries/twitterstream_queries.json b/integration-tests/src/test/resources/queries/twitterstream_queries.json index acbeb3e60b89..c83d616d789d 100644 --- a/integration-tests/src/test/resources/queries/twitterstream_queries.json +++ b/integration-tests/src/test/resources/queries/twitterstream_queries.json @@ -603,6 +603,7 @@ "size":0, "numRows":3702583, "aggregators":null, + "projections":null, "timestampSpec":null, "queryGranularity":null, "rollup":null @@ -626,6 +627,7 @@ "size":0, "numRows":3743002, "aggregators":null, + "projections":null, "timestampSpec":null, "queryGranularity":null, "rollup":null @@ -649,6 +651,7 @@ "size":0, "numRows":3502959, "aggregators":null, + "projections":null, "timestampSpec":null, "queryGranularity":null, "rollup":null From 72fd33b7cb1299a6f3445c18fc89624748de7491 Mon Sep 17 00:00:00 2001 From: cecemei Date: Fri, 13 Jun 2025 16:46:19 -0700 Subject: [PATCH 9/9] updates --- docs/querying/segmentmetadataquery.md | 9 +- .../SegmentMetadataQueryQueryToolChest.java | 23 ++- .../metadata/metadata/SegmentAnalysis.java | 102 ++++++++++++++ .../segment/AggregateProjectionMetadata.java | 31 ----- .../apache/druid/segment/IndexMergerV9.java | 31 +++-- .../metadata/SegmentAnalysisBuilder.java | 131 ------------------ ...egmentMetadataQueryQueryToolChestTest.java | 112 +++++++-------- .../metadata/SegmentMetadataQueryTest.java | 13 +- .../AggregateProjectionMetadataTest.java | 74 ---------- 9 files changed, 209 insertions(+), 317 deletions(-) delete mode 100644 processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisBuilder.java diff --git a/docs/querying/segmentmetadataquery.md b/docs/querying/segmentmetadataquery.md index beae6ad8a4c7..de223b9fed2c 100644 --- a/docs/querying/segmentmetadataquery.md +++ b/docs/querying/segmentmetadataquery.md @@ -197,7 +197,12 @@ null if the aggregators are unknown or unmergeable (if merging is enabled). * `rollup` in the result is true/false/null. * When merging is enabled, if some are rollup, others are not, result is null. -### aggregatorMergeStrategy +### projections + +* `projections` in the result will contain the list of projections in segments. +* if any conflicting projections are identified, the conflicting one will be excluded, while the non-conflicting ones will be included. + +## aggregatorMergeStrategy Conflicts between aggregator metadata across segments can occur if some segments have unknown aggregators, or if two segments use incompatible aggregators for the same column, such as `longSum` changed to `doubleSum`. @@ -213,6 +218,6 @@ Druid supports the following aggregator merge strategies: for that particular column. -### lenientAggregatorMerge (deprecated) +## lenientAggregatorMerge (deprecated) Deprecated. Use [`aggregatorMergeStrategy`](#aggregatormergestrategy) instead. diff --git a/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java index 057e4453137d..4ad82123055b 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java @@ -437,10 +437,25 @@ public static SegmentAnalysis mergeAnalyses( rollup = null; } - Map projections = - (arg1.getProjections() == null || arg2.getProjections() == null) - ? null - : AggregateProjectionMetadata.merge(arg1.getProjections(), arg2.getProjections()); + final Map projections; + if (arg1.getProjections() != null && arg2.getProjections() != null) { + projections = new HashMap<>(); + // Merge two maps of AggregateProjectionMetadata, returning a new map with the same keys and merged metadata. + // If the schemas do not match, the metadata is not merged and the key is not included in the result. + for (String name : Sets.intersection(arg1.getProjections().keySet(), arg2.getProjections().keySet())) { + AggregateProjectionMetadata spec1 = arg1.getProjections().get(name); + AggregateProjectionMetadata spec2 = arg2.getProjections().get(name); + if (spec1.getSchema().equals(spec2.getSchema())) { + // If the schemas are equal, we can merge the metadata + projections.put( + name, + new AggregateProjectionMetadata(spec1.getSchema(), spec1.getNumRows() + spec2.getNumRows()) + ); + } + } + } else { + projections = null; + } return new SegmentAnalysis( mergedId, diff --git a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java index 5b642d1b1f98..8ba23be919e0 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java @@ -25,12 +25,15 @@ import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.AggregateProjectionMetadata; +import org.apache.druid.timeline.SegmentId; import org.joda.time.Interval; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; public class SegmentAnalysis implements Comparable { @@ -212,4 +215,103 @@ public int compareTo(SegmentAnalysis rhs) { return id.compareTo(rhs.getId()); } + + /** + * Helper class to build {@link SegmentAnalysis} objects. + */ + public static class Builder + { + private final String segmentId; + private final LinkedHashMap columns = new LinkedHashMap<>(); + private final Map aggregators = new LinkedHashMap<>(); + private final Map projections = new LinkedHashMap<>(); + + private List intervals = null; + private Optional size = Optional.empty(); + private Optional numRows = Optional.empty(); + private Optional rollup = Optional.empty(); + + public Builder(String segmentId) + { + this.segmentId = segmentId; + } + + public Builder(SegmentId segmentId) + { + this.segmentId = segmentId.toString(); + } + + public Builder size(int size) + { + if (this.size.isEmpty()) { + this.size = Optional.of(size); + } else { + throw new IllegalStateException("Size is already set: " + this.size.get()); + } + return this; + } + + public Builder numRows(int numRows) + { + if (this.numRows.isEmpty()) { + this.numRows = Optional.of(numRows); + } else { + throw new IllegalStateException("NumRows is already set: " + this.numRows.get()); + } + return this; + } + + public Builder rollup(boolean rollup) + { + if (this.rollup.isEmpty()) { + this.rollup = Optional.of(rollup); + } else { + throw new IllegalStateException("Rollup is already set: " + this.rollup.get()); + } + return this; + } + + public Builder interval(Interval interval) + { + if (this.intervals == null) { + this.intervals = new ArrayList<>(); + } + this.intervals.add(interval); + return this; + } + + public Builder column(String columnName, ColumnAnalysis columnAnalysis) + { + this.columns.put(columnName, columnAnalysis); + return this; + } + + public Builder aggregator(String name, AggregatorFactory aggregatorFactory) + { + this.aggregators.put(name, aggregatorFactory); + return this; + } + + public Builder projection(String name, AggregateProjectionMetadata projection) + { + this.projections.put(name, projection); + return this; + } + + public SegmentAnalysis build() + { + return new SegmentAnalysis( + segmentId, + intervals, + columns, + size.orElse(0), + numRows.orElse(0), + aggregators.isEmpty() ? null : aggregators, + projections.isEmpty() ? null : projections, + null, + null, + rollup.orElse(null) + ); + } + } } diff --git a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java index 4cb0cec6b3b5..7e9aec5e8081 100644 --- a/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java +++ b/processing/src/main/java/org/apache/druid/segment/AggregateProjectionMetadata.java @@ -28,7 +28,6 @@ import com.google.common.collect.Interner; import com.google.common.collect.Interners; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import org.apache.druid.data.input.impl.AggregateProjectionSpec; import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.granularity.Granularities; @@ -43,9 +42,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; /** @@ -122,34 +119,6 @@ public String toString() '}'; } - /** - * Merge two maps of {@link AggregateProjectionMetadata} objects, returning a new map with the same keys and merged - * metadata. If the schemas do not match, the metadata is not merged and the key is not included in the result. - * - * @param projections1 first map of projections to merge - * @param projections2 second map of projections to merge - * @return merged map of projections - */ - public static Map merge( - Map projections1, - Map projections2 - ) - { - final Map projections = new HashMap<>(); - for (String name : Sets.intersection(projections1.keySet(), projections2.keySet())) { - AggregateProjectionMetadata spec1 = projections1.get(name); - AggregateProjectionMetadata spec2 = projections2.get(name); - if (spec1.getSchema().equals(spec2.getSchema())) { - // If the schemas are equal, we can merge the metadata - projections.put( - name, - new AggregateProjectionMetadata(spec1.getSchema(), spec1.getNumRows() + spec2.getNumRows()) - ); - } - } - return projections; - } - public static class Schema { /** diff --git a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java index faf4df5806f0..d337dd0d0aa1 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java @@ -271,20 +271,23 @@ private File makeIndexFiles( progress.stopSection(section); // Recompute the projections. - final Metadata finalMetadata = - segmentMetadata == null || CollectionUtils.isNullOrEmpty(segmentMetadata.getProjections()) - ? segmentMetadata - : makeProjections(v9Smoosher, - segmentMetadata.getProjections(), - adapters, - indexSpec, - segmentWriteOutMedium, - progress, - outDir, - closer, - mergersMap, - segmentMetadata - ); + final Metadata finalMetadata; + if (segmentMetadata == null || CollectionUtils.isNullOrEmpty(segmentMetadata.getProjections())) { + finalMetadata = segmentMetadata; + } else { + finalMetadata = makeProjections( + v9Smoosher, + segmentMetadata.getProjections(), + adapters, + indexSpec, + segmentWriteOutMedium, + progress, + outDir, + closer, + mergersMap, + segmentMetadata + ); + } /************* Make index.drd & metadata.drd files **************/ progress.progress(); diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisBuilder.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisBuilder.java deleted file mode 100644 index 9015ec0b75c4..000000000000 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalysisBuilder.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.metadata; - -import org.apache.druid.query.aggregation.AggregatorFactory; -import org.apache.druid.query.metadata.metadata.ColumnAnalysis; -import org.apache.druid.query.metadata.metadata.SegmentAnalysis; -import org.apache.druid.segment.AggregateProjectionMetadata; -import org.apache.druid.timeline.SegmentId; -import org.joda.time.Interval; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * Helper class to build {@link SegmentAnalysis} objects for testing purposes. - */ -public class SegmentAnalysisBuilder -{ - String segmentId; - List intervals = null; - LinkedHashMap columns = new LinkedHashMap<>(); - Map aggregators = new LinkedHashMap<>(); - Map projections = new LinkedHashMap<>(); - Optional size = Optional.empty(); - Optional numRows = Optional.empty(); - Optional rollup = Optional.empty(); - - SegmentAnalysisBuilder(String segmentId) - { - this.segmentId = segmentId; - } - - SegmentAnalysisBuilder(SegmentId segmentId) - { - this.segmentId = segmentId.toString(); - } - - SegmentAnalysisBuilder size(int size) - { - if (this.size.isEmpty()) { - this.size = Optional.of(size); - } else { - throw new IllegalStateException("Size is already set: " + this.size.get()); - } - return this; - } - - SegmentAnalysisBuilder numRows(int numRows) - { - if (this.numRows.isEmpty()) { - this.numRows = Optional.of(numRows); - } else { - throw new IllegalStateException("NumRows is already set: " + this.numRows.get()); - } - return this; - } - - SegmentAnalysisBuilder rollup(boolean rollup) - { - if (this.rollup.isEmpty()) { - this.rollup = Optional.of(rollup); - } else { - throw new IllegalStateException("Rollup is already set: " + this.rollup.get()); - } - return this; - } - - SegmentAnalysisBuilder interval(Interval interval) - { - if (this.intervals == null) { - this.intervals = new ArrayList<>(); - } - this.intervals.add(interval); - return this; - } - - SegmentAnalysisBuilder column(String columnName, ColumnAnalysis columnAnalysis) - { - this.columns.put(columnName, columnAnalysis); - return this; - } - - SegmentAnalysisBuilder aggregator(String name, AggregatorFactory aggregatorFactory) - { - this.aggregators.put(name, aggregatorFactory); - return this; - } - - SegmentAnalysisBuilder projection(String name, AggregateProjectionMetadata projection) - { - this.projections.put(name, projection); - return this; - } - - SegmentAnalysis build() - { - return new SegmentAnalysis( - segmentId, - intervals, - columns, - size.orElse(0), - numRows.orElse(0), - aggregators.isEmpty() ? null : aggregators, - projections.isEmpty() ? null : projections, - null, - null, - rollup.orElse(null) - ); - } -} diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java index 03bf90d3e43f..69be9165ac9d 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java @@ -126,7 +126,7 @@ public void testCacheStrategy() throws Exception byte[] actualKey = strategy.computeCacheKey(query); Assert.assertArrayEquals(expectedKey, actualKey); - SegmentAnalysis result = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + SegmentAnalysis result = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1) .interval(Intervals.of("2011-01-12T00:00:00.000Z/2011-04-15T00:00:00.001Z")) .column( "placement", @@ -163,16 +163,16 @@ public void testCacheStrategy() throws Exception @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") public void testMergeAggregators(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1) .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("baz", new DoubleSumAggregatorFactory("baz", "baz")) .build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2) .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) .build(); - final SegmentAnalysis expected = new SegmentAnalysisBuilder( + final SegmentAnalysis expected = new SegmentAnalysis.Builder( "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) @@ -185,18 +185,18 @@ public void testMergeAggregators(AggregatorMergeStrategy aggregatorMergeStrategy @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") public void testMergeAggregatorsWithIntervals(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1) .interval(TEST_SEGMENT_ID1.getInterval()) .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("baz", new DoubleSumAggregatorFactory("baz", "baz")) .build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2) .interval(TEST_SEGMENT_ID2.getInterval()) .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) .build(); - SegmentAnalysis expected = new SegmentAnalysisBuilder( + SegmentAnalysis expected = new SegmentAnalysis.Builder( "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") .interval(TEST_SEGMENT_ID1.getInterval()) .interval(TEST_SEGMENT_ID2.getInterval()) @@ -213,13 +213,13 @@ public void testMergeAggregatorsOneNullStrict(AggregatorMergeStrategy aggregator { Assume.assumeTrue(aggregatorMergeStrategy == AggregatorMergeStrategy.STRICT); - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2) .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) .build(); - final SegmentAnalysis expectedNullAggregators = new SegmentAnalysisBuilder( + final SegmentAnalysis expectedNullAggregators = new SegmentAnalysis.Builder( "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged").build(); Assert.assertEquals(expectedNullAggregators, mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy)); } @@ -230,13 +230,13 @@ public void testMergeAggregatorsOneNullNotStrict(AggregatorMergeStrategy aggrega { Assume.assumeTrue(aggregatorMergeStrategy != AggregatorMergeStrategy.STRICT); - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2) .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) .build(); - final SegmentAnalysis expected = new SegmentAnalysisBuilder( + final SegmentAnalysis expected = new SegmentAnalysis.Builder( "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) @@ -248,10 +248,10 @@ public void testMergeAggregatorsOneNullNotStrict(AggregatorMergeStrategy aggrega @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") public void testMergeAggregatorsAllNull(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).build(); + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2).build(); - final SegmentAnalysis expected = new SegmentAnalysisBuilder( + final SegmentAnalysis expected = new SegmentAnalysis.Builder( "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged").build(); Assert.assertEquals(expected, mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy)); } @@ -259,23 +259,23 @@ public void testMergeAggregatorsAllNull(AggregatorMergeStrategy aggregatorMergeS @Test public void testMergeAggregatorsConflict() { - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1) .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) .build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2) .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleMaxAggregatorFactory("bar", "bar")) .aggregator("baz", new LongMaxAggregatorFactory("baz", "baz")) .build(); // Test strict merge, returns null aggregators as there's a conflict on "bar" - final SegmentAnalysis expectedStrict = new SegmentAnalysisBuilder( + final SegmentAnalysis expectedStrict = new SegmentAnalysis.Builder( "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged").build(); Assert.assertEquals(expectedStrict, mergeWithStrategy(analysis1, analysis2, AggregatorMergeStrategy.STRICT)); // Test lenient merge, returns a map with null for "bar" as it has a conflict - final SegmentAnalysis expectedLenient = new SegmentAnalysisBuilder( + final SegmentAnalysis expectedLenient = new SegmentAnalysis.Builder( "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", null) @@ -293,7 +293,7 @@ public void testMergeAggregatorsConflict() Assert.assertEquals(expectedLenient, mergeLenient(mergeLenient(analysis1, analysis2), analysis1)); // Test earliest merge, returns a map with "bar" as DoubleSumAggregatorFactory since analysis1 is earlier - final SegmentAnalysis expectedEarliest = new SegmentAnalysisBuilder( + final SegmentAnalysis expectedEarliest = new SegmentAnalysis.Builder( "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) @@ -307,7 +307,7 @@ public void testMergeAggregatorsConflict() ); // Test latest merge, returns a map with "bar" as DoubleMaxAggregatorFactory since analysis2 is later - final SegmentAnalysis expectedLatest = new SegmentAnalysisBuilder( + final SegmentAnalysis expectedLatest = new SegmentAnalysis.Builder( "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleMaxAggregatorFactory("bar", "bar")) @@ -324,11 +324,11 @@ public void testMergeAggregatorsConflict() @Test public void testMergeAggregatorsConflictWithDifferentOrder() { - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2) .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) .build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1) .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleMaxAggregatorFactory("bar", "bar")) .aggregator("baz", new LongMaxAggregatorFactory("baz", "baz")) @@ -336,13 +336,13 @@ public void testMergeAggregatorsConflictWithDifferentOrder() // Test strict merge, returns null aggregators as there's a conflict on "bar" Assert.assertEquals( - new SegmentAnalysisBuilder("dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") + new SegmentAnalysis.Builder("dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") .build(), mergeStrict(analysis1, analysis2) ); // Test lenient merge, returns a map with null for "bar" as it has a conflict - final SegmentAnalysis expectedLenient = new SegmentAnalysisBuilder( + final SegmentAnalysis expectedLenient = new SegmentAnalysis.Builder( "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", null) @@ -362,7 +362,7 @@ public void testMergeAggregatorsConflictWithDifferentOrder() ); // Test earliest merge, returns a map with "bar" as DoubleMaxAggregatorFactory since analysis2 is earlier - final SegmentAnalysis expectedEarliest = new SegmentAnalysisBuilder( + final SegmentAnalysis expectedEarliest = new SegmentAnalysis.Builder( "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleMaxAggregatorFactory("bar", "bar")) @@ -376,7 +376,7 @@ public void testMergeAggregatorsConflictWithDifferentOrder() ); // Test latest merge, returns a map with "bar" as DoubleSumAggregatorFactory since analysis1 is later - final SegmentAnalysis expectedLatest = new SegmentAnalysisBuilder( + final SegmentAnalysis expectedLatest = new SegmentAnalysis.Builder( "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) @@ -397,12 +397,12 @@ public void testMergeAggregatorsConflictWithEqualSegmentIntervalsAndDifferentPar final SegmentId segmentId1 = SegmentId.of(TEST_DATASOURCE.toString(), interval, "test", 1); final SegmentId segmentId2 = SegmentId.of(TEST_DATASOURCE.toString(), interval, "test", 2); - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(segmentId1) + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(segmentId1) .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) .build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(segmentId2) + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(segmentId2) .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleMaxAggregatorFactory("bar", "bar")) .aggregator("baz", new LongMaxAggregatorFactory("baz", "baz")) @@ -410,13 +410,13 @@ public void testMergeAggregatorsConflictWithEqualSegmentIntervalsAndDifferentPar // Test strict merge, returns null aggregators as there's a conflict on "bar" Assert.assertEquals( - new SegmentAnalysisBuilder("dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2") + new SegmentAnalysis.Builder("dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2") .build(), mergeStrict(analysis1, analysis2) ); // Test lenient merge, returns a map with null for "bar" as it has a conflict - SegmentAnalysis expectedLenient = new SegmentAnalysisBuilder( + SegmentAnalysis expectedLenient = new SegmentAnalysis.Builder( "dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2") .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", null) @@ -430,7 +430,7 @@ public void testMergeAggregatorsConflictWithEqualSegmentIntervalsAndDifferentPar ); // Test earliest merge, returns a map with "bar" as DoubleSumAggregatorFactory since analysis1 has the earlier partition - SegmentAnalysis expectedEarliest = new SegmentAnalysisBuilder( + SegmentAnalysis expectedEarliest = new SegmentAnalysis.Builder( "dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2") .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) @@ -444,7 +444,7 @@ public void testMergeAggregatorsConflictWithEqualSegmentIntervalsAndDifferentPar ); // Test latest merge, returns a map with "bar" as DoubleMaxAggregatorFactory since analysis2 has the later partition - SegmentAnalysis expectedLatest = new SegmentAnalysisBuilder( + SegmentAnalysis expectedLatest = new SegmentAnalysis.Builder( "dummy_2023-01-01T00:00:00.000Z_2023-01-02T00:00:00.000Z_merged_2") .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleMaxAggregatorFactory("bar", "bar")) @@ -507,11 +507,11 @@ public Interval getTrueInterval() @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") public void testMergeRollup(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).rollup(false).build(); - final SegmentAnalysis analysis3 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).rollup(false).build(); - final SegmentAnalysis analysis4 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).rollup(true).build(); - final SegmentAnalysis analysis5 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).rollup(true).build(); + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2).rollup(false).build(); + final SegmentAnalysis analysis3 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1).rollup(false).build(); + final SegmentAnalysis analysis4 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2).rollup(true).build(); + final SegmentAnalysis analysis5 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1).rollup(true).build(); Assert.assertNull(mergeWithStrategy(analysis1, analysis2, aggregatorMergeStrategy).isRollup()); Assert.assertNull(mergeWithStrategy(analysis1, analysis4, aggregatorMergeStrategy).isRollup()); @@ -524,8 +524,8 @@ public void testMergeRollup(AggregatorMergeStrategy aggregatorMergeStrategy) @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") public void testInvalidMergeAggregatorsWithNullOrEmptyDatasource(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).build(); + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2).build(); MatcherAssert.assertThat( Assert.assertThrows( @@ -561,16 +561,16 @@ public void testInvalidMergeAggregatorsWithNullOrEmptyDatasource(AggregatorMerge @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") public void testMergeWithUnionDatasource(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1) .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) .build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2) .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) .build(); - final SegmentAnalysis expectedMergedAnalysis = new SegmentAnalysisBuilder( + final SegmentAnalysis expectedMergedAnalysis = new SegmentAnalysis.Builder( "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") .aggregator("foo", new LongSumAggregatorFactory("foo", "foo")) .aggregator("bar", new DoubleSumAggregatorFactory("bar", "bar")) @@ -612,8 +612,8 @@ public void testMergeWithUnionDatasource(AggregatorMergeStrategy aggregatorMerge @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") public void testMergeWithNullAnalyses(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).rollup(false).build(); + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2).rollup(false).build(); Assert.assertEquals(analysis1, mergeWithStrategy(analysis1, null, aggregatorMergeStrategy)); Assert.assertEquals(analysis2, mergeWithStrategy(null, analysis2, aggregatorMergeStrategy)); @@ -627,14 +627,14 @@ public void testMergeWithNullAnalyses(AggregatorMergeStrategy aggregatorMergeStr @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") public void testProjections(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1) .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) .build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2) .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 200)) .build(); - final SegmentAnalysis expected = new SegmentAnalysisBuilder( + final SegmentAnalysis expected = new SegmentAnalysis.Builder( "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 300)) .build(); @@ -645,14 +645,14 @@ public void testProjections(AggregatorMergeStrategy aggregatorMergeStrategy) @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") public void testProjectionsWithNull(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1) .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) .build(); - final SegmentAnalysis analysis1NullProjection = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1).build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + final SegmentAnalysis analysis1NullProjection = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1).build(); + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2) .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 200)) .build(); - final SegmentAnalysis analysis2NullProjection = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2).build(); + final SegmentAnalysis analysis2NullProjection = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2).build(); Assert.assertNull(mergeWithStrategy(analysis1NullProjection, analysis2, aggregatorMergeStrategy).getProjections()); Assert.assertNull(mergeWithStrategy(analysis1, analysis2NullProjection, aggregatorMergeStrategy).getProjections()); @@ -665,19 +665,19 @@ public void testProjectionsWithNull(AggregatorMergeStrategy aggregatorMergeStrat @ParameterizedTest(name = "{index}: with AggregatorMergeStrategy {0}") public void testProjectionsWithConflict(AggregatorMergeStrategy aggregatorMergeStrategy) { - final SegmentAnalysis analysis1 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID1) + final SegmentAnalysis analysis1 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID1) .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) .projection("channel_sum_1", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) .projection("conflict_projection", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 100)) .build(); - final SegmentAnalysis analysis2 = new SegmentAnalysisBuilder(TEST_SEGMENT_ID2) + final SegmentAnalysis analysis2 = new SegmentAnalysis.Builder(TEST_SEGMENT_ID2) .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 200)) .projection("channel_sum_2", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_DAILY, 200)) .projection("conflict_projection", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_DAILY, 200)) .build(); // conflict projection is ignored. - final SegmentAnalysis expectedStrict = new SegmentAnalysisBuilder( + final SegmentAnalysis expectedStrict = new SegmentAnalysis.Builder( "dummy_2021-01-01T00:00:00.000Z_2021-01-02T00:00:00.000Z_merged") .projection("channel_sum", new AggregateProjectionMetadata(PROJECTION_CHANNEL_ADDED_HOURLY, 300)) .build(); diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java index 172f1927657f..5fd5ea98d576 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java @@ -103,6 +103,8 @@ public class SegmentMetadataQueryTest extends InitializedNullHandlingTest ); private static final ObjectMapper MAPPER = new DefaultObjectMapper(); private static final String DATASOURCE = "testDatasource"; + private static final AggregateProjectionMetadata.Schema PROJECTION_SCHEMA = TestIndex.PROJECTIONS.get(0).toMetadataSchema(); + private static final int PROJECTION_ROWS = 279; @SuppressWarnings("unchecked") public static QueryRunner makeMMappedQueryRunner( @@ -223,10 +225,9 @@ public SegmentMetadataQueryTest( for (AggregatorFactory agg : TestIndex.METRIC_AGGS) { expectedAggregators.put(agg.getName(), agg.getCombiningFactory()); } - final AggregateProjectionMetadata.Schema projectionSchema = TestIndex.PROJECTIONS.get(0).toMetadataSchema(); final Map expectedProjections = ImmutableMap.of( - projectionSchema.getName(), - new AggregateProjectionMetadata(projectionSchema, 279) + PROJECTION_SCHEMA.getName(), + new AggregateProjectionMetadata(PROJECTION_SCHEMA, PROJECTION_ROWS) ); expectedSegmentAnalysis1 = new SegmentAnalysis( @@ -703,8 +704,10 @@ private void testSegmentMetadataQueryWithDefaultAnalysisMerge( expectedSegmentAnalysis1.getSize() + expectedSegmentAnalysis2.getSize(), expectedSegmentAnalysis1.getNumRows() + expectedSegmentAnalysis2.getNumRows(), expectedAggregators, - AggregateProjectionMetadata.merge(expectedSegmentAnalysis1.getProjections(), - expectedSegmentAnalysis2.getProjections()), + ImmutableMap.of( + PROJECTION_SCHEMA.getName(), + new AggregateProjectionMetadata(PROJECTION_SCHEMA, PROJECTION_ROWS * 2) + ), null, null, null diff --git a/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java b/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java index 3c54072658a3..1b2018994bdd 100644 --- a/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java +++ b/processing/src/test/java/org/apache/druid/segment/AggregateProjectionMetadataTest.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableMap; import it.unimi.dsi.fastutil.objects.ObjectAVLTreeSet; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.error.DruidException; @@ -29,7 +28,6 @@ import org.apache.druid.query.OrderBy; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregatorFactory; -import org.apache.druid.query.aggregation.LongMaxAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; @@ -74,78 +72,6 @@ public void testSerde() throws JsonProcessingException ); } - @Test - public void testMerge() - { - AggregateProjectionMetadata countChocula = new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "countChocula", - "theTime", - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "theTime")), - Arrays.asList("theTime", "a", "b", "c"), - new AggregatorFactory[]{new CountAggregatorFactory("chocula")}, - Arrays.asList( - OrderBy.ascending("theTime"), - OrderBy.ascending("a"), - OrderBy.ascending("b"), - OrderBy.ascending("c") - ) - ), - 123 - ); - AggregateProjectionMetadata sumChocula = new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "sumChocula", - "theTime", - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "theTime")), - Arrays.asList("theTime", "a", "b", "c"), - new AggregatorFactory[]{new LongSumAggregatorFactory("sumChocula", "chocula")}, - Arrays.asList( - OrderBy.ascending("theTime"), - OrderBy.ascending("a"), - OrderBy.ascending("b"), - OrderBy.ascending("c") - ) - ), - 123 - ); - AggregateProjectionMetadata maxChocula = new AggregateProjectionMetadata( - new AggregateProjectionMetadata.Schema( - "maxChocula", - "theTime", - VirtualColumns.create(Granularities.toVirtualColumn(Granularities.HOUR, "theTime")), - Arrays.asList("theTime", "a", "b", "c"), - new AggregatorFactory[]{new LongMaxAggregatorFactory("maxChocula", "chocula")}, - Arrays.asList( - OrderBy.ascending("theTime"), - OrderBy.ascending("a"), - OrderBy.ascending("b"), - OrderBy.ascending("c") - ) - ), - 123 - ); - - ImmutableMap metadata1 = ImmutableMap.of( - "countChocula", countChocula, - "chocula-conflict", sumChocula, - "sumChocula-only-in-metadata1", sumChocula - ); - ImmutableMap metadata2 = ImmutableMap.of( - "countChocula", countChocula, - "chocula-conflict", maxChocula, - "maxChocula-only-in-metadata2", maxChocula - ); - - ImmutableMap expected = ImmutableMap.of( - "countChocula", - new AggregateProjectionMetadata( - countChocula.getSchema(), - 246 - ) - ); - Assert.assertEquals(expected, AggregateProjectionMetadata.merge(metadata1, metadata2)); - } @Test public void testComparator()