From 3b2de518902ad253cdaf4bba3dff08901756d266 Mon Sep 17 00:00:00 2001 From: Phua Guan Wei Date: Mon, 27 Jan 2025 06:32:29 +0000 Subject: [PATCH 01/65] Add Bitmap64 extension --- distribution/pom.xml | 2 + extensions-contrib/druid-exactcount/README.md | 238 ++++++++++++++++++ extensions-contrib/druid-exactcount/pom.xml | 182 ++++++++++++++ .../exactcount/bitmap64/Bitmap64Counter.java | 33 +++ .../Bitmap64ExactCountAggregatorFactory.java | 210 ++++++++++++++++ .../Bitmap64ExactCountBuildAggregator.java | 68 +++++ ...map64ExactCountBuildAggregatorFactory.java | 73 ++++++ ...tmap64ExactCountBuildBufferAggregator.java | 134 ++++++++++ ...ap64ExactCountBuildComplexMetricSerde.java | 52 ++++ .../Bitmap64ExactCountMergeAggregator.java | 68 +++++ ...map64ExactCountMergeAggregatorFactory.java | 76 ++++++ ...tmap64ExactCountMergeBufferAggregator.java | 107 ++++++++ ...ap64ExactCountMergeComplexMetricSerde.java | 106 ++++++++ .../bitmap64/Bitmap64ExactCountModule.java | 72 ++++++ .../Bitmap64ExactCountObjectStrategy.java | 64 +++++ .../Bitmap64ExactCountPostAggregator.java | 138 ++++++++++ .../bitmap64/RoaringBitmap64Counter.java | 97 +++++++ .../RoaringBitmap64CounterJsonSerializer.java | 42 ++++ .../sql/Bitmap64ExactCountSqlAggregator.java | 163 ++++++++++++ ...rg.apache.druid.initialization.DruidModule | 17 ++ pom.xml | 1 + .../query/aggregation/AggregatorUtil.java | 9 + .../aggregation/post/PostAggregatorIds.java | 2 + 23 files changed, 1954 insertions(+) create mode 100644 extensions-contrib/druid-exactcount/README.md create mode 100644 extensions-contrib/druid-exactcount/pom.xml create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64Counter.java create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountAggregatorFactory.java create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregator.java create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregatorFactory.java create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildBufferAggregator.java create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildComplexMetricSerde.java create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregator.java create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregatorFactory.java create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeBufferAggregator.java create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeComplexMetricSerde.java create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountModule.java create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountObjectStrategy.java create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountPostAggregator.java create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/RoaringBitmap64Counter.java create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/RoaringBitmap64CounterJsonSerializer.java create mode 100644 extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/sql/Bitmap64ExactCountSqlAggregator.java create mode 100644 extensions-contrib/druid-exactcount/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule diff --git a/distribution/pom.xml b/distribution/pom.xml index da9631c48673..d9e529302ff4 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -462,6 +462,8 @@ org.apache.druid.extensions.contrib:grpc-query -c org.apache.druid.extensions.contrib:druid-ranger-security + -c + org.apache.druid.extensions:druid-exactcount diff --git a/extensions-contrib/druid-exactcount/README.md b/extensions-contrib/druid-exactcount/README.md new file mode 100644 index 000000000000..3d9a50af8727 --- /dev/null +++ b/extensions-contrib/druid-exactcount/README.md @@ -0,0 +1,238 @@ + + +This module provides exact distinct count for long type column. + +Usage for `Bitmap64ExactCountBuild` and `Bitmap64ExactCountMerge`: +Kafka ingestion task spec: +``` +{ + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": "ticker_event_bitmap64_exact_count_rollup", + "timestampSpec": { + "column": "timestamp", + "format": "millis", + "missingValue": null + }, + "dimensionsSpec": { + "dimensions": [ + { + "type": "string", + "name": "key" + } + ], + "dimensionExclusions": [] + }, + "metricsSpec": [ + { + "type": "Bitmap64ExactCountBuild", + "name": "count", + "fieldName": "value" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "HOUR", + "queryGranularity": "HOUR", + "rollup": true, + "intervals": null + }, + "transformSpec": { + "filter": null, + "transforms": [] + } + }, + "ioConfig": { + "topic": "ticker_event", + "inputFormat": { + "type": "json", + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [] + }, + "featureSpec": {} + }, + "replicas": 1, + "taskCount": 1, + "taskDuration": "PT3600S", + "consumerProperties": { + "bootstrap.servers": "localhost:9092" + }, + "pollTimeout": 100, + "startDelay": "PT5S", + "period": "PT30S", + "useEarliestOffset": false, + "completionTimeout": "PT1800S", + "lateMessageRejectionPeriod": null, + "earlyMessageRejectionPeriod": null, + "lateMessageRejectionStartDateTime": null, + "stream": "ticker_event", + "useEarliestSequenceNumber": false, + "type": "kafka" + } + } +} +``` +Query from rollup datasource: +``` +{ + "queryType": "timeseries", + "dataSource": { + "type": "table", + "name": "ticker_event_bitmap64_exact_count_rollup" + }, + "intervals": { + "type": "intervals", + "intervals": [ + "2020-09-13T06:35:35.000Z/146140482-04-24T15:36:27.903Z" + ] + }, + "descending": false, + "virtualColumns": [], + "filter": null, + "granularity": { + "type": "all" + }, + "aggregations": [ + { + "type": "Bitmap64ExactCountMerge", + "name": "a0", + "fieldName": "count" + } + ], + "postAggregations": [], + "limit": 2147483647 +} +``` +query with post aggregator: +``` +{ + "queryType": "timeseries", + "dataSource": { + "type": "table", + "name": "ticker_event_bitmap64_exact_count_rollup" + }, + "intervals": { + "type": "intervals", + "intervals": [ + "2020-09-13T06:35:35.000Z/146140482-04-24T15:36:27.903Z" + ] + }, + "descending": false, + "virtualColumns": [], + "filter": null, + "granularity": { + "type": "all" + }, + "aggregations": [ + { + "type": "count", + "name": "cnt" + }, + { + "type": "Bitmap64ExactCountMerge", + "name": "a0", + "fieldName": "count" + } + ], + "postAggregations": [ + { + "type": "arithmetic", + "fn": "/", + "fields": [ + { + "type": "bitmap64ExactCountCardinality", + "name": "a0", + "fieldName": "a0" + }, + { + "type": "fieldAccess", + "name": "cnt", + "fieldName": "cnt" + } + ], + "name": "rollup_rate" + } + ], + "limit": 2147483647 +} +``` +sql query: +``` +SELECT "key", BITMAP64_EXACT_COUNT("count") +FROM "ticker_event_bitmap64_exact_count_rollup" +WHERE "__time" >= CURRENT_TIMESTAMP - INTERVAL '1' DAY +GROUP BY key +``` + +Note: this `longExactCount` aggregator is recommended to use in `timeseries` type query though it also works for `topN` +and `groupBy` query. eg: +``` +{ + "queryType": "timeseries", + "dataSource": "ticker_event", + "intervals": { + "type": "intervals", + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" + ] + }, + "descending": false, + "virtualColumns": [], + "filter": null, + "granularity": { + "type": "all" + }, + "aggregations": [ + { + "type": "longExactCount", + "name": "exactCount", + "fieldName": "value", + "expression": null + } + ] +} +``` +``` +{ + "queryType": "groupBy", + "dataSource": "ticker_event", + "dimensions": ["key"], + "intervals": { + "type": "intervals", + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" + ] + }, + "granularity": { + "type": "all" + }, + "aggregations": [ + { + "type": "longExactCount", + "name": "a0", + "fieldName": "value" + } + ] +} +``` + + diff --git a/extensions-contrib/druid-exactcount/pom.xml b/extensions-contrib/druid-exactcount/pom.xml new file mode 100644 index 000000000000..7f139e311ef0 --- /dev/null +++ b/extensions-contrib/druid-exactcount/pom.xml @@ -0,0 +1,182 @@ + + + + + 4.0.0 + + org.apache.druid.extensions + druid-exactcount + druid-exactcount + Druid Aggregators for exact dinstinct count + + + org.apache.druid + druid + 31.0.1-SNAPSHOT + ../../pom.xml + + + + + org.apache.calcite + calcite-core + provided + + + org.apache.commons + commons-math3 + + + org.apache.druid + druid-server + ${project.parent.version} + provided + + + org.apache.druid + druid-processing + ${project.parent.version} + provided + + + org.apache.druid + druid-sql + ${project.parent.version} + provided + + + com.google.code.findbugs + jsr305 + provided + + + com.google.guava + guava + provided + + + com.google.inject + guice + provided + + + commons-codec + commons-codec + provided + + + it.unimi.dsi + fastutil + provided + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + com.fasterxml.jackson.core + jackson-core + provided + + + com.fasterxml.jackson.core + jackson-databind + provided + + + com.fasterxml.jackson.datatype + jackson-datatype-guava + provided + + + com.fasterxml.jackson.datatype + jackson-datatype-joda + provided + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + provided + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider + provided + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-smile-provider + provided + + + + + junit + junit + test + + + org.easymock + easymock + test + + + org.apache.druid + druid-processing + ${project.parent.version} + test-jar + test + + + org.apache.druid + druid-server + ${project.parent.version} + test + test-jar + + + org.apache.druid + druid-sql + ${project.parent.version} + test-jar + test + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + + + diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64Counter.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64Counter.java new file mode 100644 index 000000000000..3403243298f6 --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64Counter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64; + +import java.nio.ByteBuffer; + +public interface Bitmap64Counter +{ + void add(long value); + + long getCardinality(); + + Bitmap64Counter fold(Bitmap64Counter rhs); + + ByteBuffer toByteBuffer(); +} diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountAggregatorFactory.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountAggregatorFactory.java new file mode 100644 index 000000000000..53e036cd6b9d --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountAggregatorFactory.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.query.aggregation.AggregateCombiner; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.ObjectAggregateCombiner; +import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.ColumnValueSelector; + +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; + +/** + * Base class for both build and merge factories + */ +@SuppressWarnings("NullableProblems") +public abstract class Bitmap64ExactCountAggregatorFactory extends AggregatorFactory +{ + static final int MAX_INTERMEDIATE_SIZE = 5 * 1024 * 1024; // 5 MB + static final Comparator COMPARATOR = + Comparator.nullsFirst(Comparator.comparingLong(Bitmap64Counter::getCardinality)); + + private final String name; + private final String fieldName; + + Bitmap64ExactCountAggregatorFactory( + final String name, + final String fieldName + ) + { + this.name = Objects.requireNonNull(name); + this.fieldName = Objects.requireNonNull(fieldName); + } + + @Override + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public String getFieldName() + { + return fieldName; + } + + @Override + public List requiredFields() + { + return Collections.singletonList(fieldName); + } + + /** + * This is a convoluted way to return a list of input field names this aggregator needs. + * Currently the returned factories are only used to obtain a field name by calling getName() method. + */ + @Override + public List getRequiredColumns() + { + return Collections.singletonList( + new Bitmap64ExactCountBuildAggregatorFactory(fieldName, fieldName) + ); + } + + @Override + public Bitmap64Counter deserialize(final Object object) + { + return Bitmap64ExactCountMergeComplexMetricSerde.deserializeRoaringBitmap64Counter(object); + } + + @Override + public Bitmap64Counter combine(final Object objectA, final Object objectB) + { + if (objectB == null) { + return (Bitmap64Counter) objectA; + } + if (objectA == null) { + return (Bitmap64Counter) objectB; + } + ((Bitmap64Counter) objectA).fold((Bitmap64Counter) objectB); + return (Bitmap64Counter) objectA; + } + + @Override + public AggregateCombiner makeAggregateCombiner() + { + return new ObjectAggregateCombiner() + { + private Bitmap64Counter union = new RoaringBitmap64Counter(); + + @Override + public void reset(final ColumnValueSelector selector) + { + union = new RoaringBitmap64Counter(); + fold(selector); + } + + @Override + public void fold(final ColumnValueSelector selector) + { + final Bitmap64Counter bitmap64Counter = (Bitmap64Counter) selector.getObject(); + union.fold(bitmap64Counter); + } + + @Nullable + @Override + public Bitmap64Counter getObject() + { + return union; + } + + @Override + public Class classOfObject() + { + return Bitmap64Counter.class; + } + }; + } + + @Nullable + @Override + public Object finalizeComputation(@Nullable final Object object) + { + if (object == null) { + return null; + } + return ((Bitmap64Counter) object).getCardinality(); + } + + @Override + public Comparator getComparator() + { + return COMPARATOR; + } + + @Override + public AggregatorFactory getCombiningFactory() + { + return new Bitmap64ExactCountMergeAggregatorFactory(getName(), getName()); + } + + @Override + public byte[] getCacheKey() + { + return new CacheKeyBuilder(getCacheTypeId()).appendString(name).appendString(fieldName).build(); + } + + @Override + public int getMaxIntermediateSize() + { + return MAX_INTERMEDIATE_SIZE; + } + + @Override + public boolean equals(final Object object) + { + if (this == object) { + return true; + } + if (object == null || !getClass().equals(object.getClass())) { + return false; + } + final Bitmap64ExactCountAggregatorFactory that = (Bitmap64ExactCountAggregatorFactory) object; + if (!name.equals(that.getName())) { + return false; + } + return fieldName.equals(that.getFieldName()); + } + + @Override + public int hashCode() + { + return Objects.hash(name, fieldName); + } + + @Override + public String toString() + { + return getClass().getSimpleName() + " {" + + " name=" + name + + ", fieldName=" + fieldName + + " }"; + } + + protected abstract byte getCacheTypeId(); + +} diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregator.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregator.java new file mode 100644 index 000000000000..f286cb1287a0 --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregator.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64; + +import org.apache.druid.query.aggregation.Aggregator; +import org.apache.druid.segment.BaseLongColumnValueSelector; + +import javax.annotation.Nullable; + +public class Bitmap64ExactCountBuildAggregator implements Aggregator +{ + private BaseLongColumnValueSelector selector; + private Bitmap64Counter bitmap; + + public Bitmap64ExactCountBuildAggregator(BaseLongColumnValueSelector selector) + { + this.selector = selector; + this.bitmap = new RoaringBitmap64Counter(); + } + + @Override + public void aggregate() + { + bitmap.add(selector.getLong()); + } + + @Nullable + @Override + public Object get() + { + return bitmap; + } + + @Override + public float getFloat() + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public long getLong() + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public void close() + { + bitmap = null; + } +} diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregatorFactory.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregatorFactory.java new file mode 100644 index 000000000000..3f436f071670 --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregatorFactory.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.query.aggregation.Aggregator; +import org.apache.druid.query.aggregation.AggregatorUtil; +import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnType; + +@SuppressWarnings("NullableProblems") +public class Bitmap64ExactCountBuildAggregatorFactory extends Bitmap64ExactCountAggregatorFactory +{ + public static final ColumnType TYPE = ColumnType.ofComplex(Bitmap64ExactCountModule.BUILD_TYPE_NAME); + + @JsonCreator + public Bitmap64ExactCountBuildAggregatorFactory( + @JsonProperty("name") String name, + @JsonProperty("fieldName") String fieldName + ) + { + super(name, fieldName); + } + + @Override + protected byte getCacheTypeId() + { + return AggregatorUtil.BITMAP64_EXACT_COUNT_BUILD_CACHE_TYPE_ID; + } + + @Override + public Aggregator factorize(ColumnSelectorFactory metricFactory) + { + return new Bitmap64ExactCountBuildAggregator(metricFactory.makeColumnValueSelector(getFieldName())); + } + + @Override + public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + { + return new Bitmap64ExactCountBuildBufferAggregator(metricFactory.makeColumnValueSelector(getFieldName())); + } + + @Override + public ColumnType getIntermediateType() + { + return TYPE; + } + + @Override + public ColumnType getResultType() + { + return ColumnType.LONG; + } +} diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildBufferAggregator.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildBufferAggregator.java new file mode 100644 index 000000000000..24c6723c7a03 --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildBufferAggregator.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64; + +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.segment.BaseLongColumnValueSelector; + +import java.nio.ByteBuffer; +import java.util.IdentityHashMap; + +public class Bitmap64ExactCountBuildBufferAggregator implements BufferAggregator +{ + private final BaseLongColumnValueSelector selector; + private final IdentityHashMap> collectors = new IdentityHashMap<>(); + + public Bitmap64ExactCountBuildBufferAggregator(BaseLongColumnValueSelector selector) + { + this.selector = selector; + } + + @Override + public void init(ByteBuffer buf, int position) + { + createNewCollector(buf, position); + } + + @Override + public void aggregate(ByteBuffer buf, int position) + { + final int oldPosition = buf.position(); + try { + buf.position(position); + Bitmap64Counter bitmap64Counter = getOrCreateCollector(buf, position); + bitmap64Counter.add(selector.getLong()); + } + finally { + buf.position(oldPosition); + } + } + + @Override + public Object get(ByteBuffer buf, int position) + { + return getOrCreateCollector(buf, position); + } + + @Override + public long getLong(ByteBuffer buf, int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public double getDouble(ByteBuffer buf, int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public float getFloat(ByteBuffer buf, int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public void close() + { + + } + + @Override + public void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) + { + createNewCollector(newBuffer, newPosition); + Bitmap64Counter collector = collectors.get(oldBuffer).get(oldPosition); + putCollectors(newBuffer, newPosition, collector); + Int2ObjectMap collectorMap = collectors.get(oldBuffer); + if (collectorMap != null) { + collectorMap.remove(oldPosition); + if (collectorMap.isEmpty()) { + collectors.remove(oldBuffer); + } + } + } + + private void putCollectors(final ByteBuffer buffer, final int position, final Bitmap64Counter collector) + { + Int2ObjectMap map = collectors.computeIfAbsent(buffer, buf -> new Int2ObjectOpenHashMap<>()); + map.put(position, collector); + } + + private Bitmap64Counter getOrCreateCollector(ByteBuffer buf, int position) + { + Int2ObjectMap collectMap = collectors.get(buf); + Bitmap64Counter bitmap64Counter = collectMap != null ? collectMap.get(position) : null; + if (bitmap64Counter != null) { + return bitmap64Counter; + } + + return createNewCollector(buf, position); + } + + private Bitmap64Counter createNewCollector(ByteBuffer buf, int position) + { + buf.position(position); + Bitmap64Counter bitmap64Counter = new RoaringBitmap64Counter(); + Int2ObjectMap collectorMap = collectors.get(buf); + if (collectorMap == null) { + collectorMap = new Int2ObjectOpenHashMap<>(); + collectors.put(buf, collectorMap); + } + collectorMap.put(position, bitmap64Counter); + return bitmap64Counter; + } +} diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildComplexMetricSerde.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildComplexMetricSerde.java new file mode 100644 index 000000000000..ed11da37af0b --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildComplexMetricSerde.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64; + +import org.apache.druid.data.input.InputRow; +import org.apache.druid.segment.serde.ComplexMetricExtractor; + +import javax.annotation.Nullable; + +public class Bitmap64ExactCountBuildComplexMetricSerde extends Bitmap64ExactCountMergeComplexMetricSerde +{ + private static final ComplexMetricExtractor EXTRACTOR = new ComplexMetricExtractor() + { + + @Override + public Class extractedClass() + { + return Object.class; + } + + @Nullable + @Override + public Object extractValue(InputRow inputRow, String metricName) + { + return inputRow.getRaw(metricName); + } + }; + + @Override + public ComplexMetricExtractor getExtractor() + { + return EXTRACTOR; + } + +} diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregator.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregator.java new file mode 100644 index 000000000000..c5dabe4e8f61 --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregator.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64; + +import org.apache.druid.query.aggregation.Aggregator; +import org.apache.druid.segment.ColumnValueSelector; + +import javax.annotation.Nullable; + +public class Bitmap64ExactCountMergeAggregator implements Aggregator +{ + private final ColumnValueSelector selector; + private Bitmap64Counter bitmap; + + public Bitmap64ExactCountMergeAggregator(ColumnValueSelector selector) + { + this.selector = selector; + this.bitmap = new RoaringBitmap64Counter(); + } + + @Override + public void aggregate() + { + bitmap.fold(selector.getObject()); + } + + @Nullable + @Override + public Object get() + { + return bitmap; + } + + @Override + public float getFloat() + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public long getLong() + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public void close() + { + bitmap = null; + } +} diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregatorFactory.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregatorFactory.java new file mode 100644 index 000000000000..f17910d9813a --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregatorFactory.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.query.aggregation.Aggregator; +import org.apache.druid.query.aggregation.AggregatorUtil; +import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.column.ColumnType; + +@SuppressWarnings("NullableProblems") +public class Bitmap64ExactCountMergeAggregatorFactory extends Bitmap64ExactCountAggregatorFactory +{ + public static final ColumnType TYPE = ColumnType.ofComplex(Bitmap64ExactCountModule.MERGE_TYPE_NAME); + + @JsonCreator + public Bitmap64ExactCountMergeAggregatorFactory( + @JsonProperty("name") String name, + @JsonProperty("fieldName") String fieldName + ) + { + super(name, fieldName); + } + + @Override + protected byte getCacheTypeId() + { + return AggregatorUtil.BITMAP64_EXACT_COUNT_MERGE_CACHE_TYPE_ID; + } + + @Override + public Aggregator factorize(ColumnSelectorFactory metricFactory) + { + ColumnValueSelector selector = metricFactory.makeColumnValueSelector(getFieldName()); + return new Bitmap64ExactCountMergeAggregator(selector); + } + + @Override + public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + { + ColumnValueSelector selector = metricFactory.makeColumnValueSelector(getFieldName()); + return new Bitmap64ExactCountMergeBufferAggregator(selector); + } + + @Override + public ColumnType getIntermediateType() + { + return TYPE; + } + + @Override + public ColumnType getResultType() + { + return ColumnType.LONG; + } +} diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeBufferAggregator.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeBufferAggregator.java new file mode 100644 index 000000000000..3cc89c962c4f --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeBufferAggregator.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64; + +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.segment.ColumnValueSelector; + +import java.nio.ByteBuffer; +import java.util.IdentityHashMap; + +public class Bitmap64ExactCountMergeBufferAggregator implements BufferAggregator +{ + private final ColumnValueSelector selector; + private final IdentityHashMap> counterCache = new IdentityHashMap<>(); + + public Bitmap64ExactCountMergeBufferAggregator(ColumnValueSelector selector) + { + this.selector = selector; + } + + @Override + public void init(ByteBuffer buf, int position) + { + RoaringBitmap64Counter emptyCounter = new RoaringBitmap64Counter(); + addToCache(buf, position, emptyCounter); + } + + @Override + public void aggregate(ByteBuffer buf, int position) + { + Object x = selector.getObject(); + if (x == null) { + return; + } + Bitmap64Counter bitmap64Counter = counterCache.get(buf).get(position); + bitmap64Counter.fold((RoaringBitmap64Counter) x); + } + + @Override + public Object get(ByteBuffer buf, int position) + { + return counterCache.get(buf).get(position); + } + + @Override + public long getLong(ByteBuffer buf, int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public double getDouble(ByteBuffer buf, int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public float getFloat(ByteBuffer buf, int position) + { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public void close() + { + counterCache.clear(); + } + + @Override + public void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) + { + Bitmap64Counter counter = counterCache.get(oldBuffer).get(oldPosition); + addToCache(newBuffer, newPosition, counter); + Int2ObjectMap counterMap = counterCache.get(oldBuffer); + if (counterMap != null) { + counterMap.remove(oldPosition); + if (counterMap.isEmpty()) { + counterCache.remove(oldBuffer); + } + } + } + + private void addToCache(final ByteBuffer buffer, final int position, final Bitmap64Counter counter) + { + Int2ObjectMap map = counterCache.computeIfAbsent(buffer, buf -> new Int2ObjectOpenHashMap<>()); + map.put(position, counter); + } +} diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeComplexMetricSerde.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeComplexMetricSerde.java new file mode 100644 index 000000000000..7a5ebda49d11 --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeComplexMetricSerde.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64; + +import org.apache.druid.data.input.InputRow; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.segment.GenericColumnSerializer; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.data.GenericIndexed; +import org.apache.druid.segment.data.ObjectStrategy; +import org.apache.druid.segment.serde.ComplexColumnPartSupplier; +import org.apache.druid.segment.serde.ComplexMetricExtractor; +import org.apache.druid.segment.serde.ComplexMetricSerde; +import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; + +import java.nio.ByteBuffer; + +public class Bitmap64ExactCountMergeComplexMetricSerde extends ComplexMetricSerde +{ + + static RoaringBitmap64Counter deserializeRoaringBitmap64Counter(final Object object) + { + if (object instanceof String) { + return RoaringBitmap64Counter.fromBytes(StringUtils.decodeBase64(StringUtils.toUtf8((String) object))); + } else if (object instanceof byte[]) { + return RoaringBitmap64Counter.fromBytes((byte[]) object); + } else if (object instanceof RoaringBitmap64Counter) { + return (RoaringBitmap64Counter) object; + } + throw new IAE("Object is not of a type that can be deserialized to an RoaringBitmap64Counter:" + object.getClass() + .getName()); + } + + @Override + public String getTypeName() + { + return Bitmap64ExactCountModule.TYPE_NAME; // must be common type name + } + + @Override + public ObjectStrategy getObjectStrategy() + { + return Bitmap64ExactCountObjectStrategy.STRATEGY; + } + + @Override + public ComplexMetricExtractor getExtractor() + { + return new ComplexMetricExtractor() + { + @Override + public Class extractedClass() + { + return RoaringBitmap64Counter.class; + } + + @Override + public RoaringBitmap64Counter extractValue(final InputRow inputRow, final String metricName) + { + final Object object = inputRow.getRaw(metricName); + if (object == null) { + return null; + } + return deserializeRoaringBitmap64Counter(object); + } + }; + } + + @Override + public void deserializeColumn(final ByteBuffer buf, final ColumnBuilder columnBuilder) + { + columnBuilder.setComplexColumnSupplier( + new ComplexColumnPartSupplier( + getTypeName(), + GenericIndexed.read(buf, Bitmap64ExactCountObjectStrategy.STRATEGY, columnBuilder.getFileMapper()) + ) + ); + } + + // support large columns + @Override + public GenericColumnSerializer getSerializer(final SegmentWriteOutMedium segmentWriteOutMedium, final String column) + { + return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); + } + +} diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountModule.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountModule.java new file mode 100644 index 000000000000..695c0ec0cf3f --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountModule.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.common.annotations.VisibleForTesting; +import com.google.inject.Binder; +import org.apache.druid.initialization.DruidModule; +import org.apache.druid.query.aggregation.exactcount.bitmap64.sql.Bitmap64ExactCountSqlAggregator; +import org.apache.druid.segment.serde.ComplexMetrics; +import org.apache.druid.sql.guice.SqlBindings; + +import java.util.Collections; +import java.util.List; + +public class Bitmap64ExactCountModule implements DruidModule +{ + public static final String TYPE_NAME = "Bitmap64ExactCount"; // common type name to be associated with segment data + public static final String BUILD_TYPE_NAME = "Bitmap64ExactCountBuild"; + public static final String MERGE_TYPE_NAME = "Bitmap64ExactCountMerge"; + + @Override + public List getJacksonModules() + { + return Collections.singletonList( + new SimpleModule("Bitmap64ExactCountModule") + .registerSubtypes( + new NamedType(Bitmap64ExactCountMergeAggregatorFactory.class, MERGE_TYPE_NAME), + new NamedType(Bitmap64ExactCountBuildAggregatorFactory.class, BUILD_TYPE_NAME), + new NamedType(Bitmap64ExactCountPostAggregator.class, "bitmap64ExactCountCardinality") + ) + .addSerializer( + RoaringBitmap64Counter.class, + new RoaringBitmap64CounterJsonSerializer() + ) + ); + } + + @Override + public void configure(Binder binder) + { + registerSerde(); + SqlBindings.addAggregator(binder, Bitmap64ExactCountSqlAggregator.class); + } + + @VisibleForTesting + public static void registerSerde() + { + ComplexMetrics.registerSerde(TYPE_NAME, new Bitmap64ExactCountMergeComplexMetricSerde()); + ComplexMetrics.registerSerde(BUILD_TYPE_NAME, new Bitmap64ExactCountBuildComplexMetricSerde()); + ComplexMetrics.registerSerde(MERGE_TYPE_NAME, new Bitmap64ExactCountMergeComplexMetricSerde()); + } +} diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountObjectStrategy.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountObjectStrategy.java new file mode 100644 index 000000000000..44fbcf57ad3e --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountObjectStrategy.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64; + +import org.apache.druid.segment.data.ObjectStrategy; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +public class Bitmap64ExactCountObjectStrategy implements ObjectStrategy +{ + + static final Bitmap64ExactCountObjectStrategy STRATEGY = new Bitmap64ExactCountObjectStrategy(); + + @Override + public Class getClazz() + { + return RoaringBitmap64Counter.class; + } + + @Nullable + @Override + public Bitmap64Counter fromByteBuffer(ByteBuffer buffer, int numBytes) + { + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes); + byte[] bytes = new byte[readOnlyBuffer.remaining()]; + readOnlyBuffer.get(bytes, 0, numBytes); + return RoaringBitmap64Counter.fromBytes(bytes); + } + + @Nullable + @Override + public byte[] toBytes(@Nullable Bitmap64Counter val) + { + if (val == null) { + return new byte[0]; + } + return val.toByteBuffer().array(); + } + + @Override + public int compare(Bitmap64Counter o1, Bitmap64Counter o2) + { + return Bitmap64ExactCountAggregatorFactory.COMPARATOR.compare(o1, o2); + } +} diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountPostAggregator.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountPostAggregator.java new file mode 100644 index 000000000000..a83df3090976 --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountPostAggregator.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.collect.Sets; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.PostAggregator; +import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator; +import org.apache.druid.query.aggregation.post.PostAggregatorIds; +import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.ColumnInspector; +import org.apache.druid.segment.column.ColumnType; + +import javax.annotation.Nullable; +import java.util.Comparator; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +@JsonTypeName("bitmap64ExactCountCardinality") +public class Bitmap64ExactCountPostAggregator implements PostAggregator +{ + private final String name; + private final String fieldName; + + @JsonCreator + public Bitmap64ExactCountPostAggregator( + @JsonProperty("name") String name, + @JsonProperty("fieldName") String fieldName + ) + { + this.name = name; + this.fieldName = fieldName; + } + + @JsonProperty + public String getFieldName() + { + return fieldName; + } + + @Override + public Set getDependentFields() + { + return Sets.newHashSet(fieldName); + } + + @Override + public Comparator getComparator() + { + return ArithmeticPostAggregator.DEFAULT_COMPARATOR; + } + + @Override + @JsonProperty + public Object compute(Map combinedAggregators) + { + Object value = combinedAggregators.get(fieldName); + return ((Bitmap64Counter) value).getCardinality(); + } + + @Override + @JsonProperty + public String getName() + { + return name; + } + + @Nullable + @Override + public ColumnType getType(ColumnInspector signature) + { + return ColumnType.LONG; + } + + @Override + public PostAggregator decorate(Map aggregators) + { + return this; + } + + @Override + public byte[] getCacheKey() + { + return new CacheKeyBuilder(PostAggregatorIds.BITMAP64_EXACT_COUNT_MERGE_TYPE_ID) + .appendString(fieldName) + .build(); + } + + @Override + public String toString() + { + return "Bitmap64ExactCountPostAggregator{" + + "name='" + name + '\'' + + ", field=" + fieldName + + '}'; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Bitmap64ExactCountPostAggregator that = (Bitmap64ExactCountPostAggregator) o; + return name.equals(that.name) && + fieldName.equals(that.fieldName); + } + + @Override + public int hashCode() + { + return Objects.hash(name, fieldName); + } +} diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/RoaringBitmap64Counter.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/RoaringBitmap64Counter.java new file mode 100644 index 000000000000..2ab47541db71 --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/RoaringBitmap64Counter.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64; + +import com.google.common.base.Throwables; +import org.apache.druid.java.util.common.logger.Logger; +import org.roaringbitmap.longlong.Roaring64NavigableMap; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.nio.ByteBuffer; + +public class RoaringBitmap64Counter implements Bitmap64Counter +{ + private static Logger logger = new Logger(RoaringBitmap64Counter.class); + + private Roaring64NavigableMap bitmap; + + public RoaringBitmap64Counter() + { + this.bitmap = new Roaring64NavigableMap(); + } + + private RoaringBitmap64Counter(Roaring64NavigableMap bitmap) + { + this.bitmap = bitmap; + } + + public static RoaringBitmap64Counter fromBytes(byte[] bytes) + { + ByteArrayInputStream byteIn = new ByteArrayInputStream(bytes); + try { + DataInputStream in = new DataInputStream(byteIn); + Roaring64NavigableMap bitmap = new Roaring64NavigableMap(); + bitmap.deserialize(in); + return new RoaringBitmap64Counter(bitmap); + } + catch (Exception e) { + logger.info(e.getMessage(), e); + } + return null; + } + + @Override + public void add(long value) + { + bitmap.addLong(value); + } + + @Override + public long getCardinality() + { + return bitmap.getLongCardinality(); + } + + @Override + public Bitmap64Counter fold(Bitmap64Counter rhs) + { + // TODO: 7/9/20 is it correct? + bitmap.or(((RoaringBitmap64Counter) rhs).bitmap); + return this; + } + + @Override + public ByteBuffer toByteBuffer() + { + bitmap.runOptimize(); + try { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + bitmap.serialize(new DataOutputStream(out)); + return ByteBuffer.wrap(out.toByteArray()); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + +} diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/RoaringBitmap64CounterJsonSerializer.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/RoaringBitmap64CounterJsonSerializer.java new file mode 100644 index 000000000000..452890ba2c97 --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/RoaringBitmap64CounterJsonSerializer.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; + +public class RoaringBitmap64CounterJsonSerializer extends JsonSerializer +{ + + @Override + public void serialize( + final RoaringBitmap64Counter bitmap64Counter, + final JsonGenerator jgen, + final SerializerProvider provider + ) + throws IOException + { + jgen.writeBinary(bitmap64Counter.toByteBuffer().array()); + } + +} diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/sql/Bitmap64ExactCountSqlAggregator.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/sql/Bitmap64ExactCountSqlAggregator.java new file mode 100644 index 000000000000..dfa369f0eb51 --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/sql/Bitmap64ExactCountSqlAggregator.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exactcount.bitmap64.sql; + +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.type.InferTypes; +import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.exactcount.bitmap64.Bitmap64ExactCountBuildAggregatorFactory; +import org.apache.druid.query.aggregation.exactcount.bitmap64.Bitmap64ExactCountMergeAggregatorFactory; +import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; +import org.apache.druid.query.dimension.DefaultDimensionSpec; +import org.apache.druid.query.dimension.DimensionSpec; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.sql.calcite.aggregation.Aggregation; +import org.apache.druid.sql.calcite.aggregation.SqlAggregator; +import org.apache.druid.sql.calcite.expression.DruidExpression; +import org.apache.druid.sql.calcite.expression.Expressions; +import org.apache.druid.sql.calcite.expression.OperatorConversions; +import org.apache.druid.sql.calcite.planner.Calcites; +import org.apache.druid.sql.calcite.planner.PlannerContext; +import org.apache.druid.sql.calcite.rel.VirtualColumnRegistry; + +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; + +public class Bitmap64ExactCountSqlAggregator implements SqlAggregator +{ + + private static final String NAME = "BITMAP64_EXACT_COUNT"; + private static final SqlAggFunction FUNCTION_INSTANCE = OperatorConversions.aggregatorBuilder(NAME) + .operandNames("column") + .operandTypes(SqlTypeFamily.ANY) + .operandTypeInference(InferTypes.VARCHAR_1024) + .requiredOperandCount(1) + .returnTypeNonNull(SqlTypeName.BIGINT) + .functionCategory(SqlFunctionCategory.NUMERIC) + .build(); + + @Override + public SqlAggFunction calciteFunction() + { + return FUNCTION_INSTANCE; + } + + @Nullable + @Override + public Aggregation toDruidAggregation( + PlannerContext plannerContext, + RowSignature rowSignature, + VirtualColumnRegistry virtualColumnRegistry, + RexBuilder rexBuilder, + String name, + AggregateCall aggregateCall, + Project project, + List existingAggregations, + boolean finalizeAggregations + ) + { + // Don't use Aggregations.getArgumentsForSimpleAggregator, since it won't let us use direct column access + // for string columns. + final RexNode columnRexNode = Expressions.fromFieldAccess( + rexBuilder.getTypeFactory(), + rowSignature, + project, + aggregateCall.getArgList().get(0) + ); + + final DruidExpression columnArg = Expressions.toDruidExpression(plannerContext, rowSignature, columnRexNode); + if (columnArg == null) { + return null; + } + + final AggregatorFactory aggregatorFactory; + final String aggregatorName = finalizeAggregations ? Calcites.makePrefixedName(name, "a") : name; + + if (columnArg.isDirectColumnAccess() + && rowSignature.getColumnType(columnArg.getDirectColumn()) + .map(type -> type.is(ValueType.COMPLEX)) + .orElse(false)) { + aggregatorFactory = new Bitmap64ExactCountMergeAggregatorFactory( + aggregatorName, + columnArg.getDirectColumn() + ); + } else { + final RelDataType dataType = columnRexNode.getType(); + final ColumnType inputType = Calcites.getColumnTypeForRelDataType(dataType); + if (inputType == null) { + throw new ISE( + "Cannot translate sqlTypeName[%s] to Druid type for field[%s]", + dataType.getSqlTypeName(), + aggregatorName + ); + } + + final DimensionSpec dimensionSpec; + + if (columnArg.isDirectColumnAccess()) { + dimensionSpec = columnArg.getSimpleExtraction().toDimensionSpec(null, inputType); + } else { + String virtualColumnName = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( + columnArg, + dataType + ); + dimensionSpec = new DefaultDimensionSpec(virtualColumnName, null, inputType); + } + + aggregatorFactory = new Bitmap64ExactCountBuildAggregatorFactory( + aggregatorName, + dimensionSpec.getDimension() + ); + } + + return toAggregation( + name, + finalizeAggregations, + aggregatorFactory + ); + } + + private Aggregation toAggregation( + String name, + boolean finalizeAggregations, + AggregatorFactory aggregatorFactory + ) + { + return Aggregation.create( + Collections.singletonList(aggregatorFactory), + finalizeAggregations ? new FinalizingFieldAccessPostAggregator( + name, + aggregatorFactory.getName() + ) : null + ); + } +} diff --git a/extensions-contrib/druid-exactcount/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/druid-exactcount/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 000000000000..8618cf4efb1f --- /dev/null +++ b/extensions-contrib/druid-exactcount/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.druid.query.aggregation.exactcount.bitmap64.Bitmap64ExactCountModule +org.apache.druid.query.aggregation.exactcount.bitmap32.Bitmap32ExactCountModule diff --git a/pom.xml b/pom.xml index 68ee72ef2e30..8b46031dd1a6 100644 --- a/pom.xml +++ b/pom.xml @@ -233,6 +233,7 @@ extensions-contrib/cloudfiles-extensions extensions-contrib/graphite-emitter extensions-contrib/distinctcount + extensions-contrib/druid-exactcount extensions-contrib/statsd-emitter extensions-contrib/time-min-max extensions-contrib/virtual-columns diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java index c4c9a7875ef0..9c265c7cebf8 100755 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java @@ -167,6 +167,15 @@ public class AggregatorUtil // DDSketch aggregator public static final byte DDSKETCH_CACHE_TYPE_ID = 0x50; + // Bitmap64 exact count aggregator + public static final byte BITMAP64_EXACT_COUNT_BUILD_CACHE_TYPE_ID = 0x60; + public static final byte BITMAP64_EXACT_COUNT_MERGE_CACHE_TYPE_ID = 0x61; + + // Bitmap32 exact count aggregator + public static final byte BITMAP32_EXACT_COUNT_BUILD_CACHE_TYPE_ID = 0x62; + public static final byte BITMAP32_EXACT_COUNT_MERGE_CACHE_TYPE_ID = 0x63; + + /** * Given a list of PostAggregators and the name of an output column, returns the minimal list of PostAggregators * required to compute the output column. diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/PostAggregatorIds.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/PostAggregatorIds.java index 9d097b6e4f29..e904596a301c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/PostAggregatorIds.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/PostAggregatorIds.java @@ -70,4 +70,6 @@ public class PostAggregatorIds public static final byte SPECTATOR_HISTOGRAM_SKETCH_PERCENTILES_CACHE_TYPE_ID = 46; public static final byte DDSKETCH_QUANTILES_TYPE_ID = 51; public static final byte DDSKETCH_QUANTILE_TYPE_ID = 52; + public static final byte BITMAP64_EXACT_COUNT_MERGE_TYPE_ID = 53; + public static final byte BITMAP32_EXACT_COUNT_MERGE_TYPE_ID = 54; } From 7b3f455b2f4ee689b0e392be99a33d36a889b055 Mon Sep 17 00:00:00 2001 From: GWphua Date: Mon, 17 Mar 2025 15:29:28 +0800 Subject: [PATCH 02/65] Update max intermediate size --- .../bitmap64/Bitmap64ExactCountAggregatorFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountAggregatorFactory.java b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountAggregatorFactory.java index 53e036cd6b9d..0b76e42c1b0d 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountAggregatorFactory.java +++ b/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountAggregatorFactory.java @@ -38,7 +38,7 @@ @SuppressWarnings("NullableProblems") public abstract class Bitmap64ExactCountAggregatorFactory extends AggregatorFactory { - static final int MAX_INTERMEDIATE_SIZE = 5 * 1024 * 1024; // 5 MB + static final int MAX_INTERMEDIATE_SIZE = 1024; // 1 KiB static final Comparator COMPARATOR = Comparator.nullsFirst(Comparator.comparingLong(Bitmap64Counter::getCardinality)); From 52c3c4875ba861c3621377f9c8e6104c64265cd4 Mon Sep 17 00:00:00 2001 From: GWphua Date: Tue, 20 May 2025 15:03:04 +0800 Subject: [PATCH 03/65] Changes to Bitmap64 after build --- .../README.md | 0 .../pom.xml | 6 ++-- .../bitmap64/Bitmap64Counter.java | 2 +- ...p64ExactCardinalityAggregatorFactory.java} | 30 +++++++------------ ...map64ExactCardinalityBuildAggregator.java} | 6 ++-- ...actCardinalityBuildAggregatorFactory.java} | 13 ++++---- ...xactCardinalityBuildBufferAggregator.java} | 12 +++----- ...ctCardinalityBuildComplexMetricSerde.java} | 4 +-- ...map64ExactCardinalityMergeAggregator.java} | 6 ++-- ...actCardinalityMergeAggregatorFactory.java} | 13 ++++---- ...xactCardinalityMergeBufferAggregator.java} | 6 ++-- ...ctCardinalityMergeComplexMetricSerde.java} | 10 +++---- .../Bitmap64ExactCardinalityModule.java} | 18 +++++------ ...tmap64ExactCardinalityObjectStrategy.java} | 8 ++--- ...tmap64ExactCardinalityPostAggregator.java} | 8 ++--- .../bitmap64/RoaringBitmap64Counter.java | 3 +- .../RoaringBitmap64CounterJsonSerializer.java | 2 +- .../sql/Bitmap64ExactCountSqlAggregator.java | 10 +++---- ...rg.apache.druid.initialization.DruidModule | 2 +- pom.xml | 2 +- .../query/aggregation/AggregatorUtil.java | 5 ---- .../aggregation/post/PostAggregatorIds.java | 1 - 22 files changed, 73 insertions(+), 94 deletions(-) rename extensions-contrib/{druid-exactcount => druid-exact-cardinality}/README.md (100%) rename extensions-contrib/{druid-exactcount => druid-exact-cardinality}/pom.xml (97%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality}/bitmap64/Bitmap64Counter.java (93%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountAggregatorFactory.java => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java} (83%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregator.java => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregator.java} (87%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregatorFactory.java => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactory.java} (80%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildBufferAggregator.java => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregator.java} (89%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildComplexMetricSerde.java => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildComplexMetricSerde.java} (88%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregator.java => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregator.java} (86%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregatorFactory.java => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactory.java} (83%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeBufferAggregator.java => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeBufferAggregator.java} (92%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeComplexMetricSerde.java => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeComplexMetricSerde.java} (89%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountModule.java => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModule.java} (75%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountObjectStrategy.java => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityObjectStrategy.java} (82%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountPostAggregator.java => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregator.java} (92%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality}/bitmap64/RoaringBitmap64Counter.java (96%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality}/bitmap64/RoaringBitmap64CounterJsonSerializer.java (95%) rename extensions-contrib/{druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount => druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality}/bitmap64/sql/Bitmap64ExactCountSqlAggregator.java (92%) rename extensions-contrib/{druid-exactcount => druid-exact-cardinality}/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule (90%) diff --git a/extensions-contrib/druid-exactcount/README.md b/extensions-contrib/druid-exact-cardinality/README.md similarity index 100% rename from extensions-contrib/druid-exactcount/README.md rename to extensions-contrib/druid-exact-cardinality/README.md diff --git a/extensions-contrib/druid-exactcount/pom.xml b/extensions-contrib/druid-exact-cardinality/pom.xml similarity index 97% rename from extensions-contrib/druid-exactcount/pom.xml rename to extensions-contrib/druid-exact-cardinality/pom.xml index 7f139e311ef0..9e205c08e841 100644 --- a/extensions-contrib/druid-exactcount/pom.xml +++ b/extensions-contrib/druid-exact-cardinality/pom.xml @@ -23,14 +23,14 @@ 4.0.0 org.apache.druid.extensions - druid-exactcount - druid-exactcount + druid-exact-cardinality + druid-exact-cardinality Druid Aggregators for exact dinstinct count org.apache.druid druid - 31.0.1-SNAPSHOT + 34.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64Counter.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64Counter.java similarity index 93% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64Counter.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64Counter.java index 3403243298f6..d02377c80d26 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64Counter.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64Counter.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; import java.nio.ByteBuffer; diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountAggregatorFactory.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java similarity index 83% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountAggregatorFactory.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java index 0b76e42c1b0d..5b4a63f8c26a 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountAggregatorFactory.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.query.aggregation.AggregateCombiner; @@ -25,6 +25,7 @@ import org.apache.druid.query.aggregation.ObjectAggregateCombiner; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnValueSelector; +import org.jetbrains.annotations.NotNull; import javax.annotation.Nullable; import java.util.Collections; @@ -35,17 +36,19 @@ /** * Base class for both build and merge factories */ -@SuppressWarnings("NullableProblems") -public abstract class Bitmap64ExactCountAggregatorFactory extends AggregatorFactory +public abstract class Bitmap64ExactCardinalityAggregatorFactory extends AggregatorFactory { static final int MAX_INTERMEDIATE_SIZE = 1024; // 1 KiB static final Comparator COMPARATOR = Comparator.nullsFirst(Comparator.comparingLong(Bitmap64Counter::getCardinality)); + @NotNull private final String name; + + @NotNull private final String fieldName; - Bitmap64ExactCountAggregatorFactory( + Bitmap64ExactCardinalityAggregatorFactory( final String name, final String fieldName ) @@ -68,27 +71,16 @@ public String getFieldName() } @Override + @NotNull public List requiredFields() { return Collections.singletonList(fieldName); } - /** - * This is a convoluted way to return a list of input field names this aggregator needs. - * Currently the returned factories are only used to obtain a field name by calling getName() method. - */ - @Override - public List getRequiredColumns() - { - return Collections.singletonList( - new Bitmap64ExactCountBuildAggregatorFactory(fieldName, fieldName) - ); - } - @Override public Bitmap64Counter deserialize(final Object object) { - return Bitmap64ExactCountMergeComplexMetricSerde.deserializeRoaringBitmap64Counter(object); + return Bitmap64ExactCardinalityMergeComplexMetricSerde.deserializeRoaringBitmap64Counter(object); } @Override @@ -159,7 +151,7 @@ public Comparator getComparator() @Override public AggregatorFactory getCombiningFactory() { - return new Bitmap64ExactCountMergeAggregatorFactory(getName(), getName()); + return new Bitmap64ExactCardinalityMergeAggregatorFactory(getName(), getName()); } @Override @@ -183,7 +175,7 @@ public boolean equals(final Object object) if (object == null || !getClass().equals(object.getClass())) { return false; } - final Bitmap64ExactCountAggregatorFactory that = (Bitmap64ExactCountAggregatorFactory) object; + final Bitmap64ExactCardinalityAggregatorFactory that = (Bitmap64ExactCardinalityAggregatorFactory) object; if (!name.equals(that.getName())) { return false; } diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregator.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregator.java similarity index 87% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregator.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregator.java index f286cb1287a0..941628108f1e 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregator.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregator.java @@ -17,19 +17,19 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.segment.BaseLongColumnValueSelector; import javax.annotation.Nullable; -public class Bitmap64ExactCountBuildAggregator implements Aggregator +public class Bitmap64ExactCardinalityBuildAggregator implements Aggregator { private BaseLongColumnValueSelector selector; private Bitmap64Counter bitmap; - public Bitmap64ExactCountBuildAggregator(BaseLongColumnValueSelector selector) + public Bitmap64ExactCardinalityBuildAggregator(BaseLongColumnValueSelector selector) { this.selector = selector; this.bitmap = new RoaringBitmap64Counter(); diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregatorFactory.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactory.java similarity index 80% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregatorFactory.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactory.java index 3f436f071670..e63fbcc36c87 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildAggregatorFactory.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactory.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -27,13 +27,12 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.column.ColumnType; -@SuppressWarnings("NullableProblems") -public class Bitmap64ExactCountBuildAggregatorFactory extends Bitmap64ExactCountAggregatorFactory +public class Bitmap64ExactCardinalityBuildAggregatorFactory extends Bitmap64ExactCardinalityAggregatorFactory { - public static final ColumnType TYPE = ColumnType.ofComplex(Bitmap64ExactCountModule.BUILD_TYPE_NAME); + public static final ColumnType TYPE = ColumnType.ofComplex(Bitmap64ExactCardinalityModule.BUILD_TYPE_NAME); @JsonCreator - public Bitmap64ExactCountBuildAggregatorFactory( + public Bitmap64ExactCardinalityBuildAggregatorFactory( @JsonProperty("name") String name, @JsonProperty("fieldName") String fieldName ) @@ -50,13 +49,13 @@ protected byte getCacheTypeId() @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new Bitmap64ExactCountBuildAggregator(metricFactory.makeColumnValueSelector(getFieldName())); + return new Bitmap64ExactCardinalityBuildAggregator(metricFactory.makeColumnValueSelector(getFieldName())); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new Bitmap64ExactCountBuildBufferAggregator(metricFactory.makeColumnValueSelector(getFieldName())); + return new Bitmap64ExactCardinalityBuildBufferAggregator(metricFactory.makeColumnValueSelector(getFieldName())); } @Override diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildBufferAggregator.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregator.java similarity index 89% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildBufferAggregator.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregator.java index 24c6723c7a03..c40635d0c376 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildBufferAggregator.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregator.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; @@ -27,12 +27,12 @@ import java.nio.ByteBuffer; import java.util.IdentityHashMap; -public class Bitmap64ExactCountBuildBufferAggregator implements BufferAggregator +public class Bitmap64ExactCardinalityBuildBufferAggregator implements BufferAggregator { private final BaseLongColumnValueSelector selector; private final IdentityHashMap> collectors = new IdentityHashMap<>(); - public Bitmap64ExactCountBuildBufferAggregator(BaseLongColumnValueSelector selector) + public Bitmap64ExactCardinalityBuildBufferAggregator(BaseLongColumnValueSelector selector) { this.selector = selector; } @@ -123,11 +123,7 @@ private Bitmap64Counter createNewCollector(ByteBuffer buf, int position) { buf.position(position); Bitmap64Counter bitmap64Counter = new RoaringBitmap64Counter(); - Int2ObjectMap collectorMap = collectors.get(buf); - if (collectorMap == null) { - collectorMap = new Int2ObjectOpenHashMap<>(); - collectors.put(buf, collectorMap); - } + Int2ObjectMap collectorMap = collectors.computeIfAbsent(buf, k -> new Int2ObjectOpenHashMap<>()); collectorMap.put(position, bitmap64Counter); return bitmap64Counter; } diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildComplexMetricSerde.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildComplexMetricSerde.java similarity index 88% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildComplexMetricSerde.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildComplexMetricSerde.java index ed11da37af0b..cb06cbaf67af 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountBuildComplexMetricSerde.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildComplexMetricSerde.java @@ -17,14 +17,14 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; import org.apache.druid.data.input.InputRow; import org.apache.druid.segment.serde.ComplexMetricExtractor; import javax.annotation.Nullable; -public class Bitmap64ExactCountBuildComplexMetricSerde extends Bitmap64ExactCountMergeComplexMetricSerde +public class Bitmap64ExactCardinalityBuildComplexMetricSerde extends Bitmap64ExactCardinalityMergeComplexMetricSerde { private static final ComplexMetricExtractor EXTRACTOR = new ComplexMetricExtractor() { diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregator.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregator.java similarity index 86% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregator.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregator.java index c5dabe4e8f61..ae1242574cf1 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregator.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregator.java @@ -17,19 +17,19 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; -public class Bitmap64ExactCountMergeAggregator implements Aggregator +public class Bitmap64ExactCardinalityMergeAggregator implements Aggregator { private final ColumnValueSelector selector; private Bitmap64Counter bitmap; - public Bitmap64ExactCountMergeAggregator(ColumnValueSelector selector) + public Bitmap64ExactCardinalityMergeAggregator(ColumnValueSelector selector) { this.selector = selector; this.bitmap = new RoaringBitmap64Counter(); diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregatorFactory.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactory.java similarity index 83% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregatorFactory.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactory.java index f17910d9813a..99d63ff27a0d 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeAggregatorFactory.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactory.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -28,13 +28,12 @@ import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.column.ColumnType; -@SuppressWarnings("NullableProblems") -public class Bitmap64ExactCountMergeAggregatorFactory extends Bitmap64ExactCountAggregatorFactory +public class Bitmap64ExactCardinalityMergeAggregatorFactory extends Bitmap64ExactCardinalityAggregatorFactory { - public static final ColumnType TYPE = ColumnType.ofComplex(Bitmap64ExactCountModule.MERGE_TYPE_NAME); + public static final ColumnType TYPE = ColumnType.ofComplex(Bitmap64ExactCardinalityModule.MERGE_TYPE_NAME); @JsonCreator - public Bitmap64ExactCountMergeAggregatorFactory( + public Bitmap64ExactCardinalityMergeAggregatorFactory( @JsonProperty("name") String name, @JsonProperty("fieldName") String fieldName ) @@ -52,14 +51,14 @@ protected byte getCacheTypeId() public Aggregator factorize(ColumnSelectorFactory metricFactory) { ColumnValueSelector selector = metricFactory.makeColumnValueSelector(getFieldName()); - return new Bitmap64ExactCountMergeAggregator(selector); + return new Bitmap64ExactCardinalityMergeAggregator(selector); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { ColumnValueSelector selector = metricFactory.makeColumnValueSelector(getFieldName()); - return new Bitmap64ExactCountMergeBufferAggregator(selector); + return new Bitmap64ExactCardinalityMergeBufferAggregator(selector); } @Override diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeBufferAggregator.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeBufferAggregator.java similarity index 92% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeBufferAggregator.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeBufferAggregator.java index 3cc89c962c4f..988069987925 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeBufferAggregator.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeBufferAggregator.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; @@ -27,12 +27,12 @@ import java.nio.ByteBuffer; import java.util.IdentityHashMap; -public class Bitmap64ExactCountMergeBufferAggregator implements BufferAggregator +public class Bitmap64ExactCardinalityMergeBufferAggregator implements BufferAggregator { private final ColumnValueSelector selector; private final IdentityHashMap> counterCache = new IdentityHashMap<>(); - public Bitmap64ExactCountMergeBufferAggregator(ColumnValueSelector selector) + public Bitmap64ExactCardinalityMergeBufferAggregator(ColumnValueSelector selector) { this.selector = selector; } diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeComplexMetricSerde.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeComplexMetricSerde.java similarity index 89% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeComplexMetricSerde.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeComplexMetricSerde.java index 7a5ebda49d11..151cb9b77e77 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountMergeComplexMetricSerde.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeComplexMetricSerde.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; import org.apache.druid.data.input.InputRow; import org.apache.druid.java.util.common.IAE; @@ -34,7 +34,7 @@ import java.nio.ByteBuffer; -public class Bitmap64ExactCountMergeComplexMetricSerde extends ComplexMetricSerde +public class Bitmap64ExactCardinalityMergeComplexMetricSerde extends ComplexMetricSerde { static RoaringBitmap64Counter deserializeRoaringBitmap64Counter(final Object object) @@ -53,13 +53,13 @@ static RoaringBitmap64Counter deserializeRoaringBitmap64Counter(final Object obj @Override public String getTypeName() { - return Bitmap64ExactCountModule.TYPE_NAME; // must be common type name + return Bitmap64ExactCardinalityModule.TYPE_NAME; // must be common type name } @Override public ObjectStrategy getObjectStrategy() { - return Bitmap64ExactCountObjectStrategy.STRATEGY; + return Bitmap64ExactCardinalityObjectStrategy.STRATEGY; } @Override @@ -91,7 +91,7 @@ public void deserializeColumn(final ByteBuffer buf, final ColumnBuilder columnBu columnBuilder.setComplexColumnSupplier( new ComplexColumnPartSupplier( getTypeName(), - GenericIndexed.read(buf, Bitmap64ExactCountObjectStrategy.STRATEGY, columnBuilder.getFileMapper()) + GenericIndexed.read(buf, Bitmap64ExactCardinalityObjectStrategy.STRATEGY, columnBuilder.getFileMapper()) ) ); } diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountModule.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModule.java similarity index 75% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountModule.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModule.java index 695c0ec0cf3f..f8bd98fbc9de 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountModule.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModule.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.jsontype.NamedType; @@ -25,14 +25,14 @@ import com.google.common.annotations.VisibleForTesting; import com.google.inject.Binder; import org.apache.druid.initialization.DruidModule; -import org.apache.druid.query.aggregation.exactcount.bitmap64.sql.Bitmap64ExactCountSqlAggregator; +import org.apache.druid.query.aggregation.exact.cardinality.bitmap64.sql.Bitmap64ExactCountSqlAggregator; import org.apache.druid.segment.serde.ComplexMetrics; import org.apache.druid.sql.guice.SqlBindings; import java.util.Collections; import java.util.List; -public class Bitmap64ExactCountModule implements DruidModule +public class Bitmap64ExactCardinalityModule implements DruidModule { public static final String TYPE_NAME = "Bitmap64ExactCount"; // common type name to be associated with segment data public static final String BUILD_TYPE_NAME = "Bitmap64ExactCountBuild"; @@ -44,9 +44,9 @@ public List getJacksonModules() return Collections.singletonList( new SimpleModule("Bitmap64ExactCountModule") .registerSubtypes( - new NamedType(Bitmap64ExactCountMergeAggregatorFactory.class, MERGE_TYPE_NAME), - new NamedType(Bitmap64ExactCountBuildAggregatorFactory.class, BUILD_TYPE_NAME), - new NamedType(Bitmap64ExactCountPostAggregator.class, "bitmap64ExactCountCardinality") + new NamedType(Bitmap64ExactCardinalityMergeAggregatorFactory.class, MERGE_TYPE_NAME), + new NamedType(Bitmap64ExactCardinalityBuildAggregatorFactory.class, BUILD_TYPE_NAME), + new NamedType(Bitmap64ExactCardinalityPostAggregator.class, "bitmap64ExactCountCardinality") ) .addSerializer( RoaringBitmap64Counter.class, @@ -65,8 +65,8 @@ public void configure(Binder binder) @VisibleForTesting public static void registerSerde() { - ComplexMetrics.registerSerde(TYPE_NAME, new Bitmap64ExactCountMergeComplexMetricSerde()); - ComplexMetrics.registerSerde(BUILD_TYPE_NAME, new Bitmap64ExactCountBuildComplexMetricSerde()); - ComplexMetrics.registerSerde(MERGE_TYPE_NAME, new Bitmap64ExactCountMergeComplexMetricSerde()); + ComplexMetrics.registerSerde(TYPE_NAME, new Bitmap64ExactCardinalityMergeComplexMetricSerde()); + ComplexMetrics.registerSerde(BUILD_TYPE_NAME, new Bitmap64ExactCardinalityBuildComplexMetricSerde()); + ComplexMetrics.registerSerde(MERGE_TYPE_NAME, new Bitmap64ExactCardinalityMergeComplexMetricSerde()); } } diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountObjectStrategy.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityObjectStrategy.java similarity index 82% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountObjectStrategy.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityObjectStrategy.java index 44fbcf57ad3e..9fcbeb7e29d8 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountObjectStrategy.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityObjectStrategy.java @@ -17,17 +17,17 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; import org.apache.druid.segment.data.ObjectStrategy; import javax.annotation.Nullable; import java.nio.ByteBuffer; -public class Bitmap64ExactCountObjectStrategy implements ObjectStrategy +public class Bitmap64ExactCardinalityObjectStrategy implements ObjectStrategy { - static final Bitmap64ExactCountObjectStrategy STRATEGY = new Bitmap64ExactCountObjectStrategy(); + static final Bitmap64ExactCardinalityObjectStrategy STRATEGY = new Bitmap64ExactCardinalityObjectStrategy(); @Override public Class getClazz() @@ -59,6 +59,6 @@ public byte[] toBytes(@Nullable Bitmap64Counter val) @Override public int compare(Bitmap64Counter o1, Bitmap64Counter o2) { - return Bitmap64ExactCountAggregatorFactory.COMPARATOR.compare(o1, o2); + return Bitmap64ExactCardinalityAggregatorFactory.COMPARATOR.compare(o1, o2); } } diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountPostAggregator.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregator.java similarity index 92% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountPostAggregator.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregator.java index a83df3090976..757697e6a624 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/Bitmap64ExactCountPostAggregator.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregator.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -38,13 +38,13 @@ import java.util.Set; @JsonTypeName("bitmap64ExactCountCardinality") -public class Bitmap64ExactCountPostAggregator implements PostAggregator +public class Bitmap64ExactCardinalityPostAggregator implements PostAggregator { private final String name; private final String fieldName; @JsonCreator - public Bitmap64ExactCountPostAggregator( + public Bitmap64ExactCardinalityPostAggregator( @JsonProperty("name") String name, @JsonProperty("fieldName") String fieldName ) @@ -125,7 +125,7 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - Bitmap64ExactCountPostAggregator that = (Bitmap64ExactCountPostAggregator) o; + Bitmap64ExactCardinalityPostAggregator that = (Bitmap64ExactCardinalityPostAggregator) o; return name.equals(that.name) && fieldName.equals(that.fieldName); } diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/RoaringBitmap64Counter.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/RoaringBitmap64Counter.java similarity index 96% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/RoaringBitmap64Counter.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/RoaringBitmap64Counter.java index 2ab47541db71..1d278e10336c 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/RoaringBitmap64Counter.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/RoaringBitmap64Counter.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; import com.google.common.base.Throwables; import org.apache.druid.java.util.common.logger.Logger; @@ -75,7 +75,6 @@ public long getCardinality() @Override public Bitmap64Counter fold(Bitmap64Counter rhs) { - // TODO: 7/9/20 is it correct? bitmap.or(((RoaringBitmap64Counter) rhs).bitmap); return this; } diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/RoaringBitmap64CounterJsonSerializer.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/RoaringBitmap64CounterJsonSerializer.java similarity index 95% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/RoaringBitmap64CounterJsonSerializer.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/RoaringBitmap64CounterJsonSerializer.java index 452890ba2c97..ff4c5ece7b6c 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/RoaringBitmap64CounterJsonSerializer.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/RoaringBitmap64CounterJsonSerializer.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; diff --git a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/sql/Bitmap64ExactCountSqlAggregator.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCountSqlAggregator.java similarity index 92% rename from extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/sql/Bitmap64ExactCountSqlAggregator.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCountSqlAggregator.java index dfa369f0eb51..c44eba73e781 100644 --- a/extensions-contrib/druid-exactcount/src/main/java/org/apache/druid/query/aggregation/exactcount/bitmap64/sql/Bitmap64ExactCountSqlAggregator.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCountSqlAggregator.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.exactcount.bitmap64.sql; +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64.sql; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Project; @@ -31,8 +31,8 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.java.util.common.ISE; import org.apache.druid.query.aggregation.AggregatorFactory; -import org.apache.druid.query.aggregation.exactcount.bitmap64.Bitmap64ExactCountBuildAggregatorFactory; -import org.apache.druid.query.aggregation.exactcount.bitmap64.Bitmap64ExactCountMergeAggregatorFactory; +import org.apache.druid.query.aggregation.exact.cardinality.bitmap64.Bitmap64ExactCardinalityBuildAggregatorFactory; +import org.apache.druid.query.aggregation.exact.cardinality.bitmap64.Bitmap64ExactCardinalityMergeAggregatorFactory; import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.query.dimension.DimensionSpec; @@ -106,7 +106,7 @@ public Aggregation toDruidAggregation( && rowSignature.getColumnType(columnArg.getDirectColumn()) .map(type -> type.is(ValueType.COMPLEX)) .orElse(false)) { - aggregatorFactory = new Bitmap64ExactCountMergeAggregatorFactory( + aggregatorFactory = new Bitmap64ExactCardinalityMergeAggregatorFactory( aggregatorName, columnArg.getDirectColumn() ); @@ -133,7 +133,7 @@ public Aggregation toDruidAggregation( dimensionSpec = new DefaultDimensionSpec(virtualColumnName, null, inputType); } - aggregatorFactory = new Bitmap64ExactCountBuildAggregatorFactory( + aggregatorFactory = new Bitmap64ExactCardinalityBuildAggregatorFactory( aggregatorName, dimensionSpec.getDimension() ); diff --git a/extensions-contrib/druid-exactcount/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/druid-exact-cardinality/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule similarity index 90% rename from extensions-contrib/druid-exactcount/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule rename to extensions-contrib/druid-exact-cardinality/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule index 8618cf4efb1f..c89f417e8c56 100644 --- a/extensions-contrib/druid-exactcount/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule +++ b/extensions-contrib/druid-exact-cardinality/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -13,5 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -org.apache.druid.query.aggregation.exactcount.bitmap64.Bitmap64ExactCountModule +org.apache.druid.query.aggregation.exact.cardinality.bitmap64.Bitmap64ExactCardinalityModule org.apache.druid.query.aggregation.exactcount.bitmap32.Bitmap32ExactCountModule diff --git a/pom.xml b/pom.xml index 8b46031dd1a6..1bdd3cd76896 100644 --- a/pom.xml +++ b/pom.xml @@ -233,7 +233,7 @@ extensions-contrib/cloudfiles-extensions extensions-contrib/graphite-emitter extensions-contrib/distinctcount - extensions-contrib/druid-exactcount + extensions-contrib/druid-exact-cardinality extensions-contrib/statsd-emitter extensions-contrib/time-min-max extensions-contrib/virtual-columns diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java index 9c265c7cebf8..992452227c80 100755 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java @@ -171,11 +171,6 @@ public class AggregatorUtil public static final byte BITMAP64_EXACT_COUNT_BUILD_CACHE_TYPE_ID = 0x60; public static final byte BITMAP64_EXACT_COUNT_MERGE_CACHE_TYPE_ID = 0x61; - // Bitmap32 exact count aggregator - public static final byte BITMAP32_EXACT_COUNT_BUILD_CACHE_TYPE_ID = 0x62; - public static final byte BITMAP32_EXACT_COUNT_MERGE_CACHE_TYPE_ID = 0x63; - - /** * Given a list of PostAggregators and the name of an output column, returns the minimal list of PostAggregators * required to compute the output column. diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/PostAggregatorIds.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/PostAggregatorIds.java index e904596a301c..afd80a78b336 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/PostAggregatorIds.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/PostAggregatorIds.java @@ -71,5 +71,4 @@ public class PostAggregatorIds public static final byte DDSKETCH_QUANTILES_TYPE_ID = 51; public static final byte DDSKETCH_QUANTILE_TYPE_ID = 52; public static final byte BITMAP64_EXACT_COUNT_MERGE_TYPE_ID = 53; - public static final byte BITMAP32_EXACT_COUNT_MERGE_TYPE_ID = 54; } From 9c8d669ae63c3e8326165ef1eee5327dbee3dd85 Mon Sep 17 00:00:00 2001 From: GWphua Date: Tue, 20 May 2025 16:02:03 +0800 Subject: [PATCH 04/65] Replace cardinality count name --- distribution/pom.xml | 2 +- .../druid-exact-cardinality/README.md | 38 +++++++++---------- .../druid-exact-cardinality/pom.xml | 2 +- ...xactCardinalityBuildAggregatorFactory.java | 2 +- ...xactCardinalityMergeAggregatorFactory.java | 2 +- .../Bitmap64ExactCardinalityModule.java | 14 +++---- ...itmap64ExactCardinalityPostAggregator.java | 6 +-- ...itmap64ExactCardinalitySqlAggregator.java} | 4 +- ...rg.apache.druid.initialization.DruidModule | 1 - .../query/aggregation/AggregatorUtil.java | 6 +-- .../aggregation/post/PostAggregatorIds.java | 2 +- 11 files changed, 39 insertions(+), 40 deletions(-) rename extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/{Bitmap64ExactCountSqlAggregator.java => Bitmap64ExactCardinalitySqlAggregator.java} (97%) diff --git a/distribution/pom.xml b/distribution/pom.xml index d9e529302ff4..07eba0f6bb45 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -463,7 +463,7 @@ -c org.apache.druid.extensions.contrib:druid-ranger-security -c - org.apache.druid.extensions:druid-exactcount + org.apache.druid.extensions:druid-exact-cardinality diff --git a/extensions-contrib/druid-exact-cardinality/README.md b/extensions-contrib/druid-exact-cardinality/README.md index 3d9a50af8727..eee2c5ebe77e 100644 --- a/extensions-contrib/druid-exact-cardinality/README.md +++ b/extensions-contrib/druid-exact-cardinality/README.md @@ -17,16 +17,16 @@ ~ under the License. --> -This module provides exact distinct count for long type column. +This module provides exact cardinality count for long type column. -Usage for `Bitmap64ExactCountBuild` and `Bitmap64ExactCountMerge`: +Usage for `Bitmap64ExactCardinalityBuild` and `Bitmap64ExactCardinalityMerge`: Kafka ingestion task spec: ``` { "type": "kafka", "spec": { "dataSchema": { - "dataSource": "ticker_event_bitmap64_exact_count_rollup", + "dataSource": "ticker_event_bitmap64_exact_cardinality_rollup", "timestampSpec": { "column": "timestamp", "format": "millis", @@ -43,8 +43,8 @@ Kafka ingestion task spec: }, "metricsSpec": [ { - "type": "Bitmap64ExactCountBuild", - "name": "count", + "type": "Bitmap64ExactCardinalityBuild", + "name": "cardinality", "fieldName": "value" } ], @@ -97,7 +97,7 @@ Query from rollup datasource: "queryType": "timeseries", "dataSource": { "type": "table", - "name": "ticker_event_bitmap64_exact_count_rollup" + "name": "ticker_event_bitmap64_exact_cardinality_rollup" }, "intervals": { "type": "intervals", @@ -113,9 +113,9 @@ Query from rollup datasource: }, "aggregations": [ { - "type": "Bitmap64ExactCountMerge", + "type": "Bitmap64ExactCardinalityMerge", "name": "a0", - "fieldName": "count" + "fieldName": "cardinality" } ], "postAggregations": [], @@ -128,7 +128,7 @@ query with post aggregator: "queryType": "timeseries", "dataSource": { "type": "table", - "name": "ticker_event_bitmap64_exact_count_rollup" + "name": "ticker_event_bitmap64_exact_cardinality_rollup" }, "intervals": { "type": "intervals", @@ -144,13 +144,13 @@ query with post aggregator: }, "aggregations": [ { - "type": "count", + "type": "cardinality", "name": "cnt" }, { - "type": "Bitmap64ExactCountMerge", + "type": "Bitmap64ExactCardinalityMerge", "name": "a0", - "fieldName": "count" + "fieldName": "cardinality" } ], "postAggregations": [ @@ -159,7 +159,7 @@ query with post aggregator: "fn": "/", "fields": [ { - "type": "bitmap64ExactCountCardinality", + "type": "bitmap64ExactCardinality", "name": "a0", "fieldName": "a0" }, @@ -177,13 +177,13 @@ query with post aggregator: ``` sql query: ``` -SELECT "key", BITMAP64_EXACT_COUNT("count") -FROM "ticker_event_bitmap64_exact_count_rollup" +SELECT "key", BITMAP64_EXACT_CARDINALITY("cardinality") +FROM "ticker_event_bitmap64_exact_cardinality_rollup" WHERE "__time" >= CURRENT_TIMESTAMP - INTERVAL '1' DAY GROUP BY key ``` -Note: this `longExactCount` aggregator is recommended to use in `timeseries` type query though it also works for `topN` +Note: this `longExactCardinality` aggregator is recommended to use in `timeseries` type query though it also works for `topN` and `groupBy` query. eg: ``` { @@ -203,8 +203,8 @@ and `groupBy` query. eg: }, "aggregations": [ { - "type": "longExactCount", - "name": "exactCount", + "type": "longExactCardinality", + "name": "ExactCardinality", "fieldName": "value", "expression": null } @@ -227,7 +227,7 @@ and `groupBy` query. eg: }, "aggregations": [ { - "type": "longExactCount", + "type": "longExactCardinality", "name": "a0", "fieldName": "value" } diff --git a/extensions-contrib/druid-exact-cardinality/pom.xml b/extensions-contrib/druid-exact-cardinality/pom.xml index 9e205c08e841..816eb5c12c2c 100644 --- a/extensions-contrib/druid-exact-cardinality/pom.xml +++ b/extensions-contrib/druid-exact-cardinality/pom.xml @@ -25,7 +25,7 @@ org.apache.druid.extensions druid-exact-cardinality druid-exact-cardinality - Druid Aggregators for exact dinstinct count + Druid Aggregators for exact cardinality org.apache.druid diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactory.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactory.java index e63fbcc36c87..941a97d87a71 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactory.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactory.java @@ -43,7 +43,7 @@ public Bitmap64ExactCardinalityBuildAggregatorFactory( @Override protected byte getCacheTypeId() { - return AggregatorUtil.BITMAP64_EXACT_COUNT_BUILD_CACHE_TYPE_ID; + return AggregatorUtil.BITMAP64_EXACT_CARDINALITY_BUILD_CACHE_TYPE_ID; } @Override diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactory.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactory.java index 99d63ff27a0d..e74a223623c7 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactory.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactory.java @@ -44,7 +44,7 @@ public Bitmap64ExactCardinalityMergeAggregatorFactory( @Override protected byte getCacheTypeId() { - return AggregatorUtil.BITMAP64_EXACT_COUNT_MERGE_CACHE_TYPE_ID; + return AggregatorUtil.BITMAP64_EXACT_CARDINALITY_MERGE_CACHE_TYPE_ID; } @Override diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModule.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModule.java index f8bd98fbc9de..0106ee662e13 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModule.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModule.java @@ -25,7 +25,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.inject.Binder; import org.apache.druid.initialization.DruidModule; -import org.apache.druid.query.aggregation.exact.cardinality.bitmap64.sql.Bitmap64ExactCountSqlAggregator; +import org.apache.druid.query.aggregation.exact.cardinality.bitmap64.sql.Bitmap64ExactCardinalitySqlAggregator; import org.apache.druid.segment.serde.ComplexMetrics; import org.apache.druid.sql.guice.SqlBindings; @@ -34,19 +34,19 @@ public class Bitmap64ExactCardinalityModule implements DruidModule { - public static final String TYPE_NAME = "Bitmap64ExactCount"; // common type name to be associated with segment data - public static final String BUILD_TYPE_NAME = "Bitmap64ExactCountBuild"; - public static final String MERGE_TYPE_NAME = "Bitmap64ExactCountMerge"; + public static final String TYPE_NAME = "Bitmap64ExactCardinality"; // common type name to be associated with segment data + public static final String BUILD_TYPE_NAME = "Bitmap64ExactCardinalityBuild"; + public static final String MERGE_TYPE_NAME = "Bitmap64ExactCardinalityMerge"; @Override public List getJacksonModules() { return Collections.singletonList( - new SimpleModule("Bitmap64ExactCountModule") + new SimpleModule("Bitmap64ExactCardinalityModule") .registerSubtypes( new NamedType(Bitmap64ExactCardinalityMergeAggregatorFactory.class, MERGE_TYPE_NAME), new NamedType(Bitmap64ExactCardinalityBuildAggregatorFactory.class, BUILD_TYPE_NAME), - new NamedType(Bitmap64ExactCardinalityPostAggregator.class, "bitmap64ExactCountCardinality") + new NamedType(Bitmap64ExactCardinalityPostAggregator.class, "bitmap64ExactCardinality") ) .addSerializer( RoaringBitmap64Counter.class, @@ -59,7 +59,7 @@ public List getJacksonModules() public void configure(Binder binder) { registerSerde(); - SqlBindings.addAggregator(binder, Bitmap64ExactCountSqlAggregator.class); + SqlBindings.addAggregator(binder, Bitmap64ExactCardinalitySqlAggregator.class); } @VisibleForTesting diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregator.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregator.java index 757697e6a624..ac5e1c731dae 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregator.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregator.java @@ -37,7 +37,7 @@ import java.util.Objects; import java.util.Set; -@JsonTypeName("bitmap64ExactCountCardinality") +@JsonTypeName("bitmap64ExactCardinality") public class Bitmap64ExactCardinalityPostAggregator implements PostAggregator { private final String name; @@ -102,7 +102,7 @@ public PostAggregator decorate(Map aggregators) @Override public byte[] getCacheKey() { - return new CacheKeyBuilder(PostAggregatorIds.BITMAP64_EXACT_COUNT_MERGE_TYPE_ID) + return new CacheKeyBuilder(PostAggregatorIds.BITMAP64_EXACT_CARDINALITY_TYPE_ID) .appendString(fieldName) .build(); } @@ -110,7 +110,7 @@ public byte[] getCacheKey() @Override public String toString() { - return "Bitmap64ExactCountPostAggregator{" + + return "Bitmap64ExactCardinalityPostAggregator{" + "name='" + name + '\'' + ", field=" + fieldName + '}'; diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCountSqlAggregator.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregator.java similarity index 97% rename from extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCountSqlAggregator.java rename to extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregator.java index c44eba73e781..16f27fa2cbf0 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCountSqlAggregator.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregator.java @@ -52,10 +52,10 @@ import java.util.Collections; import java.util.List; -public class Bitmap64ExactCountSqlAggregator implements SqlAggregator +public class Bitmap64ExactCardinalitySqlAggregator implements SqlAggregator { - private static final String NAME = "BITMAP64_EXACT_COUNT"; + private static final String NAME = "BITMAP64_EXACT_CARDINALITY"; private static final SqlAggFunction FUNCTION_INSTANCE = OperatorConversions.aggregatorBuilder(NAME) .operandNames("column") .operandTypes(SqlTypeFamily.ANY) diff --git a/extensions-contrib/druid-exact-cardinality/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/druid-exact-cardinality/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule index c89f417e8c56..176190537890 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule +++ b/extensions-contrib/druid-exact-cardinality/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -14,4 +14,3 @@ # limitations under the License. org.apache.druid.query.aggregation.exact.cardinality.bitmap64.Bitmap64ExactCardinalityModule -org.apache.druid.query.aggregation.exactcount.bitmap32.Bitmap32ExactCountModule diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java index 992452227c80..414337713999 100755 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java @@ -167,9 +167,9 @@ public class AggregatorUtil // DDSketch aggregator public static final byte DDSKETCH_CACHE_TYPE_ID = 0x50; - // Bitmap64 exact count aggregator - public static final byte BITMAP64_EXACT_COUNT_BUILD_CACHE_TYPE_ID = 0x60; - public static final byte BITMAP64_EXACT_COUNT_MERGE_CACHE_TYPE_ID = 0x61; + // Bitmap64 exact cardinality aggregator + public static final byte BITMAP64_EXACT_CARDINALITY_BUILD_CACHE_TYPE_ID = 0x60; + public static final byte BITMAP64_EXACT_CARDINALITY_MERGE_CACHE_TYPE_ID = 0x61; /** * Given a list of PostAggregators and the name of an output column, returns the minimal list of PostAggregators diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/PostAggregatorIds.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/PostAggregatorIds.java index afd80a78b336..0031049b16dd 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/PostAggregatorIds.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/PostAggregatorIds.java @@ -70,5 +70,5 @@ public class PostAggregatorIds public static final byte SPECTATOR_HISTOGRAM_SKETCH_PERCENTILES_CACHE_TYPE_ID = 46; public static final byte DDSKETCH_QUANTILES_TYPE_ID = 51; public static final byte DDSKETCH_QUANTILE_TYPE_ID = 52; - public static final byte BITMAP64_EXACT_COUNT_MERGE_TYPE_ID = 53; + public static final byte BITMAP64_EXACT_CARDINALITY_TYPE_ID = 53; } From 2aec5b64f28324dbf1dfca29be99787cadb088c6 Mon Sep 17 00:00:00 2001 From: GWphua Date: Tue, 20 May 2025 16:55:39 +0800 Subject: [PATCH 05/65] Unit tests for bitmap64 --- .../druid-exact-cardinality/pom.xml | 9 +- ...ap64ExactCardinalityAggregatorFactory.java | 4 +- ...ExactCardinalityAggregatorFactoryTest.java | 267 ++++++++++++++++++ ...CardinalityBuildAggregatorFactoryTest.java | 120 ++++++++ ...CardinalityMergeAggregatorFactoryTest.java | 120 ++++++++ .../Bitmap64ExactCardinalityModuleTest.java | 50 ++++ ...p64ExactCardinalityPostAggregatorTest.java | 141 +++++++++ 7 files changed, 707 insertions(+), 4 deletions(-) create mode 100644 extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactoryTest.java create mode 100644 extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java create mode 100644 extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactoryTest.java create mode 100644 extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java create mode 100644 extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregatorTest.java diff --git a/extensions-contrib/druid-exact-cardinality/pom.xml b/extensions-contrib/druid-exact-cardinality/pom.xml index 816eb5c12c2c..4ba395cef328 100644 --- a/extensions-contrib/druid-exact-cardinality/pom.xml +++ b/extensions-contrib/druid-exact-cardinality/pom.xml @@ -131,8 +131,13 @@ - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine test diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java index 5b4a63f8c26a..bb898a3011ff 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java @@ -97,9 +97,9 @@ public Bitmap64Counter combine(final Object objectA, final Object objectB) } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner() { - return new ObjectAggregateCombiner() + return new ObjectAggregateCombiner<>() { private Bitmap64Counter union = new RoaringBitmap64Counter(); diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactoryTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactoryTest.java new file mode 100644 index 000000000000..b8b2516f7b51 --- /dev/null +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactoryTest.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; + +import org.apache.druid.query.aggregation.AggregateCombiner; +import org.apache.druid.query.aggregation.Aggregator; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.column.ColumnType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.easymock.EasyMock; + +import java.util.Arrays; +import java.util.Collections; + +public class Bitmap64ExactCardinalityAggregatorFactoryTest +{ + private static final String NAME = "testName"; + private static final String FIELD_NAME = "testFieldName"; + + private TestBitmap64ExactCardinalityAggregatorFactory factory; + + // Concrete implementation for testing the abstract class + private static class TestBitmap64ExactCardinalityAggregatorFactory extends Bitmap64ExactCardinalityAggregatorFactory + { + private static final byte CACHE_TYPE_ID = 0x1A; // Using a distinct byte for test + + TestBitmap64ExactCardinalityAggregatorFactory(String name, String fieldName) + { + super(name, fieldName); + } + + @Override + protected byte getCacheTypeId() + { + return CACHE_TYPE_ID; + } + + @Override + public Aggregator factorize(ColumnSelectorFactory metricFactory) + { + return null; + } + + @Override + public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + { + return null; + } + + @Override + public ColumnType getIntermediateType() + { + return Bitmap64ExactCardinalityBuildAggregatorFactory.TYPE; + } + + @Override + public AggregatorFactory withName(String newName) + { + return new TestBitmap64ExactCardinalityAggregatorFactory(newName, getFieldName()); + } + } + + @BeforeEach + public void setUp() + { + factory = new TestBitmap64ExactCardinalityAggregatorFactory(NAME, FIELD_NAME); + } + + @Test + public void testConstructor() + { + Assertions.assertEquals(NAME, factory.getName()); + Assertions.assertEquals(FIELD_NAME, factory.getFieldName()); + } + + @Test + public void testRequiredFields() + { + Assertions.assertEquals(Collections.singletonList(FIELD_NAME), factory.requiredFields()); + } + + @Test + public void testFinalizeComputationNull() + { + Assertions.assertNull(factory.finalizeComputation(null)); + } + + @Test + public void testFinalizeComputation() + { + Bitmap64Counter counter = new RoaringBitmap64Counter(); + counter.add(1L); + counter.add(2L); + Assertions.assertEquals(2L, factory.finalizeComputation(counter)); + } + + @Test + public void testGetComparator() + { + Bitmap64Counter counter1 = new RoaringBitmap64Counter(); + counter1.add(1L); + + Bitmap64Counter counter2 = new RoaringBitmap64Counter(); + counter2.add(1L); + counter2.add(2L); + + Assertions.assertTrue(factory.getComparator().compare(counter1, counter2) < 0); + Assertions.assertTrue(factory.getComparator().compare(counter2, counter1) > 0); + Assertions.assertTrue(factory.getComparator().compare(null, counter1) < 0); + Assertions.assertTrue(factory.getComparator().compare(counter1, null) > 0); + } + + @Test + public void testGetCombiningFactory() + { + AggregatorFactory combiningFactory = factory.getCombiningFactory(); + Assertions.assertInstanceOf(Bitmap64ExactCardinalityMergeAggregatorFactory.class, combiningFactory); + Assertions.assertEquals(NAME, combiningFactory.getName()); + Assertions.assertEquals(NAME, ((Bitmap64ExactCardinalityMergeAggregatorFactory) combiningFactory).getFieldName()); + } + + @Test + public void testGetMaxIntermediateSize() + { + Assertions.assertEquals(Bitmap64ExactCardinalityAggregatorFactory.MAX_INTERMEDIATE_SIZE, factory.getMaxIntermediateSize()); + } + + @Test + public void testEqualsAndHashCode() + { + // Test symmetry + TestBitmap64ExactCardinalityAggregatorFactory factory2 = new TestBitmap64ExactCardinalityAggregatorFactory(NAME, FIELD_NAME); + Assertions.assertEquals(factory, factory2); + Assertions.assertEquals(factory.hashCode(), factory2.hashCode()); + + // Test different name + TestBitmap64ExactCardinalityAggregatorFactory factoryDiffName = new TestBitmap64ExactCardinalityAggregatorFactory(NAME + "_diff", FIELD_NAME); + Assertions.assertNotEquals(factory, factoryDiffName); + + // Test different fieldName + TestBitmap64ExactCardinalityAggregatorFactory factoryDiffFieldName = new TestBitmap64ExactCardinalityAggregatorFactory(NAME, FIELD_NAME + "_diff"); + Assertions.assertNotEquals(factory, factoryDiffFieldName); + + // Test different class (even if fields match, if getEffectiveClass is used in parent equals) + // For Bitmap64ExactCardinalityAggregatorFactory, getClass() is used in equals. + Bitmap64ExactCardinalityAggregatorFactory anotherConcreteFactory = new Bitmap64ExactCardinalityBuildAggregatorFactory(NAME, FIELD_NAME); + Assertions.assertNotEquals(factory, anotherConcreteFactory, "Test factory should not be equal to Build factory due to different class"); + } + + @Test + public void testToString() + { + String expected = "TestBitmap64ExactCardinalityAggregatorFactory { name=" + NAME + ", fieldName=" + FIELD_NAME + " }"; + Assertions.assertEquals(expected, factory.toString()); + } + + @Test + public void testGetCacheKey() + { + byte[] cacheKey1 = factory.getCacheKey(); + TestBitmap64ExactCardinalityAggregatorFactory factory2 = new TestBitmap64ExactCardinalityAggregatorFactory(NAME, FIELD_NAME); + byte[] cacheKey2 = factory2.getCacheKey(); + Assertions.assertArrayEquals(cacheKey1, cacheKey2); + + TestBitmap64ExactCardinalityAggregatorFactory factoryDiffName = new TestBitmap64ExactCardinalityAggregatorFactory(NAME + "_diff", FIELD_NAME); + byte[] cacheKeyDiffName = factoryDiffName.getCacheKey(); + Assertions.assertFalse(Arrays.equals(cacheKey1, cacheKeyDiffName)); + + TestBitmap64ExactCardinalityAggregatorFactory factoryDiffFieldName = new TestBitmap64ExactCardinalityAggregatorFactory(NAME, FIELD_NAME + "_diff"); + byte[] cacheKeyDiffFieldName = factoryDiffFieldName.getCacheKey(); + Assertions.assertFalse(Arrays.equals(cacheKey1, cacheKeyDiffFieldName)); + } + + @Test + public void testCombine() + { + Bitmap64Counter counter1 = new RoaringBitmap64Counter(); + counter1.add(1L); + counter1.add(2L); + + Bitmap64Counter counter2 = new RoaringBitmap64Counter(); + counter2.add(2L); + counter2.add(3L); + + Bitmap64Counter result = factory.combine(counter1, counter2); + Assertions.assertEquals(3L, result.getCardinality()); + Assertions.assertSame(counter1, result); + + Bitmap64Counter counter3 = new RoaringBitmap64Counter(); + counter3.add(4L); + Bitmap64Counter resultNullB = factory.combine(counter3, null); + Assertions.assertSame(counter3, resultNullB); + Assertions.assertEquals(1L, resultNullB.getCardinality()); + + Bitmap64Counter counter4 = new RoaringBitmap64Counter(); + counter4.add(5L); + Bitmap64Counter resultNullA = factory.combine(null, counter4); + Assertions.assertSame(counter4, resultNullA); + Assertions.assertEquals(1L, resultNullA.getCardinality()); + + Assertions.assertNull(factory.combine(null, null)); + } + + @Test + public void testMakeAggregateCombiner() + { + AggregateCombiner combiner = factory.makeAggregateCombiner(); + Assertions.assertNotNull(combiner); + + ColumnValueSelector selector = + EasyMock.createMock(ColumnValueSelector.class); + + Bitmap64Counter counter1 = new RoaringBitmap64Counter(); + counter1.add(10L); + counter1.add(20L); + + Bitmap64Counter counter2 = new RoaringBitmap64Counter(); + counter2.add(20L); + counter2.add(30L); + + EasyMock.expect(selector.getObject()).andReturn(counter1).times(1); + EasyMock.replay(selector); + combiner.fold(selector); + EasyMock.verify(selector); + Assertions.assertEquals(2L, combiner.getObject().getCardinality()); + + EasyMock.reset(selector); + EasyMock.expect(selector.getObject()).andReturn(counter2).times(1); + EasyMock.replay(selector); + combiner.fold(selector); + EasyMock.verify(selector); + Assertions.assertEquals(3L, combiner.getObject().getCardinality()); + + EasyMock.reset(selector); + Bitmap64Counter counter3 = new RoaringBitmap64Counter(); + counter3.add(40L); + EasyMock.expect(selector.getObject()).andReturn(counter3).times(1); + EasyMock.replay(selector); + combiner.reset(selector); + EasyMock.verify(selector); + Assertions.assertEquals(1L, combiner.getObject().getCardinality()); + + Assertions.assertEquals(Bitmap64Counter.class, combiner.classOfObject()); + } +} \ No newline at end of file diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java new file mode 100644 index 000000000000..4fc0a097bede --- /dev/null +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; + +import org.apache.druid.query.aggregation.AggregatorUtil; +import org.apache.druid.query.aggregation.TestObjectColumnSelector; +import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnType; +import org.easymock.EasyMock; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class Bitmap64ExactCardinalityBuildAggregatorFactoryTest +{ + private static final String NAME = "exactCardinalityBuildTestName"; + private static final String FIELD_NAME = "exactCardinalityBuildTestFieldName"; + + private Bitmap64ExactCardinalityBuildAggregatorFactory factory; + + @BeforeEach + public void setUp() + { + factory = new Bitmap64ExactCardinalityBuildAggregatorFactory(NAME, FIELD_NAME); + } + + @Test + public void testConstructor() + { + Assertions.assertEquals(NAME, factory.getName()); + Assertions.assertEquals(FIELD_NAME, factory.getFieldName()); + } + + @Test + public void testGetCacheTypeId() + { + Assertions.assertEquals(AggregatorUtil.BITMAP64_EXACT_CARDINALITY_BUILD_CACHE_TYPE_ID, factory.getCacheTypeId()); + } + + @Test + public void testFactorize() + { + ColumnSelectorFactory selectorFactory = EasyMock.createMock(ColumnSelectorFactory.class); + EasyMock.expect(selectorFactory.makeColumnValueSelector(FIELD_NAME)) + .andReturn(new TestObjectColumnSelector(null)); // Return a dummy selector + EasyMock.replay(selectorFactory); + + Assertions.assertInstanceOf(Bitmap64ExactCardinalityBuildAggregator.class, factory.factorize(selectorFactory)); + EasyMock.verify(selectorFactory); + } + + @Test + public void testFactorizeBuffered() + { + ColumnSelectorFactory selectorFactory = EasyMock.createMock(ColumnSelectorFactory.class); + EasyMock.expect(selectorFactory.makeColumnValueSelector(FIELD_NAME)) + .andReturn(new TestObjectColumnSelector(null)); // Return a dummy selector + EasyMock.replay(selectorFactory); + + Assertions.assertInstanceOf( + Bitmap64ExactCardinalityBuildBufferAggregator.class, + factory.factorizeBuffered(selectorFactory) + ); + EasyMock.verify(selectorFactory); + } + + @Test + public void testGetIntermediateType() + { + Assertions.assertEquals(Bitmap64ExactCardinalityBuildAggregatorFactory.TYPE, factory.getIntermediateType()); + } + + @Test + public void testGetResultType() + { + Assertions.assertEquals(ColumnType.LONG, factory.getResultType()); + } + + @Test + public void testEqualsAndHashCode() + { + Bitmap64ExactCardinalityBuildAggregatorFactory factory1 = new Bitmap64ExactCardinalityBuildAggregatorFactory(NAME, FIELD_NAME); + Bitmap64ExactCardinalityBuildAggregatorFactory factory2 = new Bitmap64ExactCardinalityBuildAggregatorFactory(NAME, FIELD_NAME); + Bitmap64ExactCardinalityBuildAggregatorFactory factoryDiffName = new Bitmap64ExactCardinalityBuildAggregatorFactory(NAME + "_diff", FIELD_NAME); + Bitmap64ExactCardinalityBuildAggregatorFactory factoryDiffFieldName = new Bitmap64ExactCardinalityBuildAggregatorFactory(NAME, FIELD_NAME + "_diff"); + + Assertions.assertEquals(factory1, factory2); + Assertions.assertEquals(factory1.hashCode(), factory2.hashCode()); + + Assertions.assertNotEquals(factory1, factoryDiffName); + Assertions.assertNotEquals(factory1.hashCode(), factoryDiffName.hashCode()); + + Assertions.assertNotEquals(factory1, factoryDiffFieldName); + Assertions.assertNotEquals(factory1.hashCode(), factoryDiffFieldName.hashCode()); + } + + @Test + public void testToString() + { + String expected = "Bitmap64ExactCardinalityBuildAggregatorFactory { name=" + NAME + ", fieldName=" + FIELD_NAME + " }"; + Assertions.assertEquals(expected, factory.toString()); + } +} \ No newline at end of file diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactoryTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactoryTest.java new file mode 100644 index 000000000000..6734184fe009 --- /dev/null +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactoryTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; + +import org.apache.druid.query.aggregation.AggregatorUtil; +import org.apache.druid.query.aggregation.TestObjectColumnSelector; +import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnType; +import org.easymock.EasyMock; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class Bitmap64ExactCardinalityMergeAggregatorFactoryTest +{ + private static final String NAME = "exactCardinalityMergeTestName"; + private static final String FIELD_NAME = "exactCardinalityMergeTestFieldName"; + + private Bitmap64ExactCardinalityMergeAggregatorFactory factory; + + @BeforeEach + public void setUp() + { + factory = new Bitmap64ExactCardinalityMergeAggregatorFactory(NAME, FIELD_NAME); + } + + @Test + public void testConstructor() + { + Assertions.assertEquals(NAME, factory.getName()); + Assertions.assertEquals(FIELD_NAME, factory.getFieldName()); + } + + @Test + public void testGetCacheTypeId() + { + Assertions.assertEquals(AggregatorUtil.BITMAP64_EXACT_CARDINALITY_MERGE_CACHE_TYPE_ID, factory.getCacheTypeId()); + } + + @Test + public void testFactorize() + { + ColumnSelectorFactory selectorFactory = EasyMock.createMock(ColumnSelectorFactory.class); + EasyMock.expect(selectorFactory.makeColumnValueSelector(FIELD_NAME)) + .andReturn(new TestObjectColumnSelector(null)); // Return a dummy selector + EasyMock.replay(selectorFactory); + + Assertions.assertInstanceOf(Bitmap64ExactCardinalityMergeAggregator.class, factory.factorize(selectorFactory)); + EasyMock.verify(selectorFactory); + } + + @Test + public void testFactorizeBuffered() + { + ColumnSelectorFactory selectorFactory = EasyMock.createMock(ColumnSelectorFactory.class); + EasyMock.expect(selectorFactory.makeColumnValueSelector(FIELD_NAME)) + .andReturn(new TestObjectColumnSelector(null)); // Return a dummy selector + EasyMock.replay(selectorFactory); + + Assertions.assertInstanceOf( + Bitmap64ExactCardinalityMergeBufferAggregator.class, + factory.factorizeBuffered(selectorFactory) + ); + EasyMock.verify(selectorFactory); + } + + @Test + public void testGetIntermediateType() + { + Assertions.assertEquals(Bitmap64ExactCardinalityMergeAggregatorFactory.TYPE, factory.getIntermediateType()); + } + + @Test + public void testGetResultType() + { + Assertions.assertEquals(ColumnType.LONG, factory.getResultType()); + } + + @Test + public void testEqualsAndHashCode() + { + Bitmap64ExactCardinalityMergeAggregatorFactory factory1 = new Bitmap64ExactCardinalityMergeAggregatorFactory(NAME, FIELD_NAME); + Bitmap64ExactCardinalityMergeAggregatorFactory factory2 = new Bitmap64ExactCardinalityMergeAggregatorFactory(NAME, FIELD_NAME); + Bitmap64ExactCardinalityMergeAggregatorFactory factoryDiffName = new Bitmap64ExactCardinalityMergeAggregatorFactory(NAME + "_diff", FIELD_NAME); + Bitmap64ExactCardinalityMergeAggregatorFactory factoryDiffFieldName = new Bitmap64ExactCardinalityMergeAggregatorFactory(NAME, FIELD_NAME + "_diff"); + + Assertions.assertEquals(factory1, factory2); + Assertions.assertEquals(factory1.hashCode(), factory2.hashCode()); + + Assertions.assertNotEquals(factory1, factoryDiffName); + Assertions.assertNotEquals(factory1.hashCode(), factoryDiffName.hashCode()); + + Assertions.assertNotEquals(factory1, factoryDiffFieldName); + Assertions.assertNotEquals(factory1.hashCode(), factoryDiffFieldName.hashCode()); + } + + @Test + public void testToString() + { + String expected = "Bitmap64ExactCardinalityMergeAggregatorFactory { name=" + NAME + ", fieldName=" + FIELD_NAME + " }"; + Assertions.assertEquals(expected, factory.toString()); + } +} \ No newline at end of file diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java new file mode 100644 index 000000000000..f6208dabc540 --- /dev/null +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; + +import org.apache.druid.segment.serde.ComplexMetricSerde; +import org.apache.druid.segment.serde.ComplexMetrics; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class Bitmap64ExactCardinalityModuleTest +{ + @Test + public void testRegisterSerde() + { + ComplexMetrics.unregisterSerde(Bitmap64ExactCardinalityModule.TYPE_NAME); + ComplexMetrics.unregisterSerde(Bitmap64ExactCardinalityModule.BUILD_TYPE_NAME); + ComplexMetrics.unregisterSerde(Bitmap64ExactCardinalityModule.MERGE_TYPE_NAME); + + Bitmap64ExactCardinalityModule.registerSerde(); + + ComplexMetricSerde typeNameSerde = ComplexMetrics.getSerdeForType(Bitmap64ExactCardinalityModule.TYPE_NAME); + Assertions.assertNotNull(typeNameSerde); + Assertions.assertInstanceOf(Bitmap64ExactCardinalityMergeComplexMetricSerde.class, typeNameSerde); + + ComplexMetricSerde buildTypeNameSerde = ComplexMetrics.getSerdeForType(Bitmap64ExactCardinalityModule.BUILD_TYPE_NAME); + Assertions.assertNotNull(buildTypeNameSerde); + Assertions.assertInstanceOf(Bitmap64ExactCardinalityBuildComplexMetricSerde.class, buildTypeNameSerde); + + ComplexMetricSerde mergeTypeNameSerde = ComplexMetrics.getSerdeForType(Bitmap64ExactCardinalityModule.MERGE_TYPE_NAME); + Assertions.assertNotNull(mergeTypeNameSerde); + Assertions.assertInstanceOf(Bitmap64ExactCardinalityMergeComplexMetricSerde.class, mergeTypeNameSerde); + } +} \ No newline at end of file diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregatorTest.java new file mode 100644 index 000000000000..6c1a83de56a5 --- /dev/null +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregatorTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; + +import com.google.common.collect.ImmutableMap; +import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator; +import org.apache.druid.query.aggregation.post.PostAggregatorIds; +import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ColumnType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; + +public class Bitmap64ExactCardinalityPostAggregatorTest +{ + private static final String NAME = "postAggTestName"; + private static final String FIELD_NAME = "postAggTestFieldName"; + + private Bitmap64ExactCardinalityPostAggregator postAggregator; + + @BeforeEach + public void setUp() + { + postAggregator = new Bitmap64ExactCardinalityPostAggregator(NAME, FIELD_NAME); + } + + @Test + public void testConstructor() + { + Assertions.assertEquals(NAME, postAggregator.getName()); + Assertions.assertEquals(FIELD_NAME, postAggregator.getFieldName()); + } + + @Test + public void testGetDependentFields() + { + Assertions.assertEquals(Collections.singleton(FIELD_NAME), postAggregator.getDependentFields()); + } + + @Test + public void testGetComparator() + { + Assertions.assertEquals(ArithmeticPostAggregator.DEFAULT_COMPARATOR, postAggregator.getComparator()); + } + + @Test + public void testCompute() + { + Bitmap64Counter counter = new RoaringBitmap64Counter(); + counter.add(1L); + counter.add(2L); + counter.add(3L); + + Map combinedAggregators = ImmutableMap.of(FIELD_NAME, counter); + Assertions.assertEquals(3L, postAggregator.compute(combinedAggregators)); + + Bitmap64Counter emptyCounter = new RoaringBitmap64Counter(); + Map combinedAggregatorsEmpty = ImmutableMap.of(FIELD_NAME, emptyCounter); + Assertions.assertEquals(0L, postAggregator.compute(combinedAggregatorsEmpty)); + } + + @Test + public void testGetType() + { + Assertions.assertEquals(ColumnType.LONG, postAggregator.getType(null)); + } + + @Test + public void testDecorate() + { + Assertions.assertSame(postAggregator, postAggregator.decorate(Collections.emptyMap())); + } + + @Test + public void testGetCacheKey() + { + byte[] expectedKey = new CacheKeyBuilder(PostAggregatorIds.BITMAP64_EXACT_CARDINALITY_TYPE_ID) + .appendString(FIELD_NAME) + .build(); + Assertions.assertArrayEquals(expectedKey, postAggregator.getCacheKey()); + + Bitmap64ExactCardinalityPostAggregator postAggregator2 = new Bitmap64ExactCardinalityPostAggregator(NAME, FIELD_NAME); + Assertions.assertArrayEquals(postAggregator.getCacheKey(), postAggregator2.getCacheKey()); + + Bitmap64ExactCardinalityPostAggregator postAggregatorDiffFieldName = new Bitmap64ExactCardinalityPostAggregator(NAME, FIELD_NAME + "_diff"); + Assertions.assertFalse(Arrays.equals(postAggregator.getCacheKey(), postAggregatorDiffFieldName.getCacheKey())); + + Bitmap64ExactCardinalityPostAggregator postAggregatorDiffName = new Bitmap64ExactCardinalityPostAggregator(NAME + "_diff", FIELD_NAME); + // Cache key for PostAggregator does not include its own name, only dependent fieldName + Assertions.assertArrayEquals(postAggregator.getCacheKey(), postAggregatorDiffName.getCacheKey()); + } + + @Test + public void testEqualsAndHashCode() + { + Bitmap64ExactCardinalityPostAggregator pa1 = new Bitmap64ExactCardinalityPostAggregator(NAME, FIELD_NAME); + Bitmap64ExactCardinalityPostAggregator pa2 = new Bitmap64ExactCardinalityPostAggregator(NAME, FIELD_NAME); + Bitmap64ExactCardinalityPostAggregator paDiffName = new Bitmap64ExactCardinalityPostAggregator(NAME + "_diff", FIELD_NAME); + Bitmap64ExactCardinalityPostAggregator paDiffFieldName = new Bitmap64ExactCardinalityPostAggregator(NAME, FIELD_NAME + "_diff"); + + Assertions.assertEquals(pa1, pa2); + Assertions.assertEquals(pa1.hashCode(), pa2.hashCode()); + + Assertions.assertNotEquals(null, pa1); // Null comparison + Assertions.assertNotEquals(new Object(), pa1); // Different type + + Assertions.assertNotEquals(pa1, paDiffName); + Assertions.assertNotEquals(pa1.hashCode(), paDiffName.hashCode()); + + Assertions.assertNotEquals(pa1, paDiffFieldName); + Assertions.assertNotEquals(pa1.hashCode(), paDiffFieldName.hashCode()); + } + + @Test + public void testToString() + { + String expected = "Bitmap64ExactCardinalityPostAggregator{name='" + NAME + "', field=" + FIELD_NAME + "}"; + Assertions.assertEquals(expected, postAggregator.toString()); + } +} \ No newline at end of file From bb28202ece93bf733293c3bca60febc206d3be19 Mon Sep 17 00:00:00 2001 From: GWphua Date: Tue, 20 May 2025 17:07:00 +0800 Subject: [PATCH 06/65] Unit tests for bitmap aggregator --- ...ExactCardinalityAggregatorFactoryTest.java | 66 +++-- ...CardinalityBuildAggregatorFactoryTest.java | 22 +- ...64ExactCardinalityBuildAggregatorTest.java | 133 ++++++++++ ...tCardinalityBuildBufferAggregatorTest.java | 223 +++++++++++++++++ ...CardinalityMergeAggregatorFactoryTest.java | 28 ++- ...64ExactCardinalityMergeAggregatorTest.java | 161 +++++++++++++ ...tCardinalityMergeBufferAggregatorTest.java | 228 ++++++++++++++++++ .../Bitmap64ExactCardinalityModuleTest.java | 4 +- ...p64ExactCardinalityPostAggregatorTest.java | 38 ++- 9 files changed, 861 insertions(+), 42 deletions(-) create mode 100644 extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorTest.java create mode 100644 extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregatorTest.java create mode 100644 extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorTest.java create mode 100644 extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeBufferAggregatorTest.java diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactoryTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactoryTest.java index b8b2516f7b51..f920be781c81 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactoryTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactoryTest.java @@ -60,13 +60,13 @@ protected byte getCacheTypeId() @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return null; + return null; } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return null; + return null; } @Override @@ -100,7 +100,7 @@ public void testRequiredFields() { Assertions.assertEquals(Collections.singletonList(FIELD_NAME), factory.requiredFields()); } - + @Test public void testFinalizeComputationNull() { @@ -115,7 +115,7 @@ public void testFinalizeComputation() counter.add(2L); Assertions.assertEquals(2L, factory.finalizeComputation(counter)); } - + @Test public void testGetComparator() { @@ -125,7 +125,7 @@ public void testGetComparator() Bitmap64Counter counter2 = new RoaringBitmap64Counter(); counter2.add(1L); counter2.add(2L); - + Assertions.assertTrue(factory.getComparator().compare(counter1, counter2) < 0); Assertions.assertTrue(factory.getComparator().compare(counter2, counter1) > 0); Assertions.assertTrue(factory.getComparator().compare(null, counter1) < 0); @@ -140,33 +140,52 @@ public void testGetCombiningFactory() Assertions.assertEquals(NAME, combiningFactory.getName()); Assertions.assertEquals(NAME, ((Bitmap64ExactCardinalityMergeAggregatorFactory) combiningFactory).getFieldName()); } - + @Test public void testGetMaxIntermediateSize() { - Assertions.assertEquals(Bitmap64ExactCardinalityAggregatorFactory.MAX_INTERMEDIATE_SIZE, factory.getMaxIntermediateSize()); + Assertions.assertEquals( + Bitmap64ExactCardinalityAggregatorFactory.MAX_INTERMEDIATE_SIZE, + factory.getMaxIntermediateSize() + ); } - + @Test public void testEqualsAndHashCode() { // Test symmetry - TestBitmap64ExactCardinalityAggregatorFactory factory2 = new TestBitmap64ExactCardinalityAggregatorFactory(NAME, FIELD_NAME); + TestBitmap64ExactCardinalityAggregatorFactory factory2 = new TestBitmap64ExactCardinalityAggregatorFactory( + NAME, + FIELD_NAME + ); Assertions.assertEquals(factory, factory2); Assertions.assertEquals(factory.hashCode(), factory2.hashCode()); // Test different name - TestBitmap64ExactCardinalityAggregatorFactory factoryDiffName = new TestBitmap64ExactCardinalityAggregatorFactory(NAME + "_diff", FIELD_NAME); + TestBitmap64ExactCardinalityAggregatorFactory factoryDiffName = new TestBitmap64ExactCardinalityAggregatorFactory( + NAME + "_diff", + FIELD_NAME + ); Assertions.assertNotEquals(factory, factoryDiffName); // Test different fieldName - TestBitmap64ExactCardinalityAggregatorFactory factoryDiffFieldName = new TestBitmap64ExactCardinalityAggregatorFactory(NAME, FIELD_NAME + "_diff"); + TestBitmap64ExactCardinalityAggregatorFactory factoryDiffFieldName = new TestBitmap64ExactCardinalityAggregatorFactory( + NAME, + FIELD_NAME + "_diff" + ); Assertions.assertNotEquals(factory, factoryDiffFieldName); // Test different class (even if fields match, if getEffectiveClass is used in parent equals) // For Bitmap64ExactCardinalityAggregatorFactory, getClass() is used in equals. - Bitmap64ExactCardinalityAggregatorFactory anotherConcreteFactory = new Bitmap64ExactCardinalityBuildAggregatorFactory(NAME, FIELD_NAME); - Assertions.assertNotEquals(factory, anotherConcreteFactory, "Test factory should not be equal to Build factory due to different class"); + Bitmap64ExactCardinalityAggregatorFactory anotherConcreteFactory = new Bitmap64ExactCardinalityBuildAggregatorFactory( + NAME, + FIELD_NAME + ); + Assertions.assertNotEquals( + factory, + anotherConcreteFactory, + "Test factory should not be equal to Build factory due to different class" + ); } @Test @@ -180,15 +199,24 @@ public void testToString() public void testGetCacheKey() { byte[] cacheKey1 = factory.getCacheKey(); - TestBitmap64ExactCardinalityAggregatorFactory factory2 = new TestBitmap64ExactCardinalityAggregatorFactory(NAME, FIELD_NAME); + TestBitmap64ExactCardinalityAggregatorFactory factory2 = new TestBitmap64ExactCardinalityAggregatorFactory( + NAME, + FIELD_NAME + ); byte[] cacheKey2 = factory2.getCacheKey(); Assertions.assertArrayEquals(cacheKey1, cacheKey2); - TestBitmap64ExactCardinalityAggregatorFactory factoryDiffName = new TestBitmap64ExactCardinalityAggregatorFactory(NAME + "_diff", FIELD_NAME); + TestBitmap64ExactCardinalityAggregatorFactory factoryDiffName = new TestBitmap64ExactCardinalityAggregatorFactory( + NAME + "_diff", + FIELD_NAME + ); byte[] cacheKeyDiffName = factoryDiffName.getCacheKey(); Assertions.assertFalse(Arrays.equals(cacheKey1, cacheKeyDiffName)); - TestBitmap64ExactCardinalityAggregatorFactory factoryDiffFieldName = new TestBitmap64ExactCardinalityAggregatorFactory(NAME, FIELD_NAME + "_diff"); + TestBitmap64ExactCardinalityAggregatorFactory factoryDiffFieldName = new TestBitmap64ExactCardinalityAggregatorFactory( + NAME, + FIELD_NAME + "_diff" + ); byte[] cacheKeyDiffFieldName = factoryDiffFieldName.getCacheKey(); Assertions.assertFalse(Arrays.equals(cacheKey1, cacheKeyDiffFieldName)); } @@ -205,8 +233,8 @@ public void testCombine() counter2.add(3L); Bitmap64Counter result = factory.combine(counter1, counter2); - Assertions.assertEquals(3L, result.getCardinality()); - Assertions.assertSame(counter1, result); + Assertions.assertEquals(3L, result.getCardinality()); + Assertions.assertSame(counter1, result); Bitmap64Counter counter3 = new RoaringBitmap64Counter(); counter3.add(4L); @@ -261,7 +289,7 @@ public void testMakeAggregateCombiner() combiner.reset(selector); EasyMock.verify(selector); Assertions.assertEquals(1L, combiner.getObject().getCardinality()); - + Assertions.assertEquals(Bitmap64Counter.class, combiner.classOfObject()); } } \ No newline at end of file diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java index 4fc0a097bede..99b234db3124 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java @@ -92,14 +92,26 @@ public void testGetResultType() { Assertions.assertEquals(ColumnType.LONG, factory.getResultType()); } - + @Test public void testEqualsAndHashCode() { - Bitmap64ExactCardinalityBuildAggregatorFactory factory1 = new Bitmap64ExactCardinalityBuildAggregatorFactory(NAME, FIELD_NAME); - Bitmap64ExactCardinalityBuildAggregatorFactory factory2 = new Bitmap64ExactCardinalityBuildAggregatorFactory(NAME, FIELD_NAME); - Bitmap64ExactCardinalityBuildAggregatorFactory factoryDiffName = new Bitmap64ExactCardinalityBuildAggregatorFactory(NAME + "_diff", FIELD_NAME); - Bitmap64ExactCardinalityBuildAggregatorFactory factoryDiffFieldName = new Bitmap64ExactCardinalityBuildAggregatorFactory(NAME, FIELD_NAME + "_diff"); + Bitmap64ExactCardinalityBuildAggregatorFactory factory1 = new Bitmap64ExactCardinalityBuildAggregatorFactory( + NAME, + FIELD_NAME + ); + Bitmap64ExactCardinalityBuildAggregatorFactory factory2 = new Bitmap64ExactCardinalityBuildAggregatorFactory( + NAME, + FIELD_NAME + ); + Bitmap64ExactCardinalityBuildAggregatorFactory factoryDiffName = new Bitmap64ExactCardinalityBuildAggregatorFactory( + NAME + "_diff", + FIELD_NAME + ); + Bitmap64ExactCardinalityBuildAggregatorFactory factoryDiffFieldName = new Bitmap64ExactCardinalityBuildAggregatorFactory( + NAME, + FIELD_NAME + "_diff" + ); Assertions.assertEquals(factory1, factory2); Assertions.assertEquals(factory1.hashCode(), factory2.hashCode()); diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorTest.java new file mode 100644 index 000000000000..f39b16e5f071 --- /dev/null +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; + +import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.easymock.EasyMock; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class Bitmap64ExactCardinalityBuildAggregatorTest +{ + private BaseLongColumnValueSelector mockSelector; + private Bitmap64ExactCardinalityBuildAggregator aggregator; + + @BeforeEach + public void setUp() + { + mockSelector = EasyMock.createMock(BaseLongColumnValueSelector.class); + aggregator = new Bitmap64ExactCardinalityBuildAggregator(mockSelector); + } + + @Test + public void testAggregateSingleValue() + { + EasyMock.expect(mockSelector.getLong()).andReturn(123L).once(); + EasyMock.replay(mockSelector); + + aggregator.aggregate(); + + Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(); + Assertions.assertEquals(1, counter.getCardinality()); + + EasyMock.verify(mockSelector); + } + + @Test + public void testAggregateMultipleDistinctValues() + { + EasyMock.expect(mockSelector.getLong()).andReturn(10L).once(); + EasyMock.expect(mockSelector.getLong()).andReturn(20L).once(); + EasyMock.expect(mockSelector.getLong()).andReturn(30L).once(); + EasyMock.replay(mockSelector); + + aggregator.aggregate(); + aggregator.aggregate(); + aggregator.aggregate(); + + Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(); + Assertions.assertEquals(3, counter.getCardinality()); + + EasyMock.verify(mockSelector); + } + + @Test + public void testAggregateMultipleValuesWithDuplicates() + { + EasyMock.expect(mockSelector.getLong()).andReturn(10L).once(); + EasyMock.expect(mockSelector.getLong()).andReturn(20L).once(); + EasyMock.expect(mockSelector.getLong()).andReturn(10L).once(); // Duplicate + EasyMock.replay(mockSelector); + + aggregator.aggregate(); + aggregator.aggregate(); + aggregator.aggregate(); + + Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(); + Assertions.assertEquals(2, counter.getCardinality()); + + EasyMock.verify(mockSelector); + } + + @Test + public void testGetInitialState() + { + Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(); + Assertions.assertNotNull(counter); + Assertions.assertEquals(0, counter.getCardinality()); + } + + @Test + public void testClose() + { + aggregator.close(); + Assertions.assertNull(aggregator.get(), "Bitmap should be null after close"); + } + + @Test + public void testCloseIsIdempotent() + { + aggregator.close(); + Assertions.assertNull(aggregator.get(), "Bitmap should be null after first close"); + aggregator.close(); // Call close again + Assertions.assertNull(aggregator.get(), "Bitmap should still be null after second close"); + } + + @Test + public void testGetFloatThrowsException() + { + Assertions.assertThrows( + UnsupportedOperationException.class, () -> { + aggregator.getFloat(); + } + ); + } + + @Test + public void testGetLongThrowsException() + { + Assertions.assertThrows( + UnsupportedOperationException.class, () -> { + aggregator.getLong(); + } + ); + } +} \ No newline at end of file diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregatorTest.java new file mode 100644 index 000000000000..66b24f134cb0 --- /dev/null +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregatorTest.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; + +import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.easymock.EasyMock; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; + +public class Bitmap64ExactCardinalityBuildBufferAggregatorTest +{ + private BaseLongColumnValueSelector mockSelector; + private Bitmap64ExactCardinalityBuildBufferAggregator aggregator; + private ByteBuffer buffer; + private static final int BUFFER_CAPACITY = 1024; + private static final int POSITION_1 = 0; + private static final int POSITION_2 = 100; // Another distinct position + + @BeforeEach + public void setUp() + { + mockSelector = EasyMock.createMock(BaseLongColumnValueSelector.class); + aggregator = new Bitmap64ExactCardinalityBuildBufferAggregator(mockSelector); + buffer = ByteBuffer.allocate(BUFFER_CAPACITY); + } + + @Test + public void testInit() + { + aggregator.init(buffer, POSITION_1); + Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertNotNull(counter); + Assertions.assertEquals(0, counter.getCardinality()); + } + + @Test + public void testAggregateSingleValue() + { + aggregator.init(buffer, POSITION_1); + EasyMock.expect(mockSelector.getLong()).andReturn(123L).once(); + EasyMock.replay(mockSelector); + + aggregator.aggregate(buffer, POSITION_1); + + Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertEquals(1, counter.getCardinality()); + + EasyMock.verify(mockSelector); + } + + @Test + public void testAggregateCreatesCollectorIfNotExists() + { + // No init call, aggregate should create it + EasyMock.expect(mockSelector.getLong()).andReturn(456L).once(); + EasyMock.replay(mockSelector); + + aggregator.aggregate(buffer, POSITION_1); + + Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertNotNull(counter); + Assertions.assertEquals(1, counter.getCardinality()); + + EasyMock.verify(mockSelector); + } + + @Test + public void testAggregateMultipleDistinctValues() + { + aggregator.init(buffer, POSITION_1); + EasyMock.expect(mockSelector.getLong()).andReturn(10L).once(); + EasyMock.expect(mockSelector.getLong()).andReturn(20L).once(); + EasyMock.replay(mockSelector); + + aggregator.aggregate(buffer, POSITION_1); + aggregator.aggregate(buffer, POSITION_1); + + Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertEquals(2, counter.getCardinality()); + + EasyMock.verify(mockSelector); + } + + @Test + public void testAggregateWithDuplicates() + { + aggregator.init(buffer, POSITION_1); + EasyMock.expect(mockSelector.getLong()).andReturn(10L).once(); + EasyMock.expect(mockSelector.getLong()).andReturn(20L).once(); + EasyMock.expect(mockSelector.getLong()).andReturn(10L).once(); // Duplicate + EasyMock.replay(mockSelector); + + aggregator.aggregate(buffer, POSITION_1); + aggregator.aggregate(buffer, POSITION_1); + aggregator.aggregate(buffer, POSITION_1); + + Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertEquals(2, counter.getCardinality()); + + EasyMock.verify(mockSelector); + } + + @Test + public void testAggregateAtDifferentPositions() + { + aggregator.init(buffer, POSITION_1); + aggregator.init(buffer, POSITION_2); + + EasyMock.expect(mockSelector.getLong()).andReturn(10L).once(); // For POSITION_1 + EasyMock.expect(mockSelector.getLong()).andReturn(20L).once(); // For POSITION_2 + EasyMock.replay(mockSelector); + + aggregator.aggregate(buffer, POSITION_1); + aggregator.aggregate(buffer, POSITION_2); + + Bitmap64Counter counter1 = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertEquals(1, counter1.getCardinality()); + + Bitmap64Counter counter2 = (Bitmap64Counter) aggregator.get(buffer, POSITION_2); + Assertions.assertEquals(1, counter2.getCardinality()); + + Assertions.assertNotSame(counter1, counter2); + + EasyMock.verify(mockSelector); + } + + @Test + public void testGetWithoutInitOrAggregateReturnsNewCounter() + { + Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertNotNull(counter); + Assertions.assertEquals(0, counter.getCardinality(), "Getting for a new position should return an empty counter"); + } + + @Test + public void testCloseIsNoOp() + { + aggregator.init(buffer, POSITION_1); + EasyMock.expect(mockSelector.getLong()).andReturn(10L).once(); + EasyMock.replay(mockSelector); + aggregator.aggregate(buffer, POSITION_1); + + aggregator.close(); // Should be a no-op + + Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertNotNull(counter, "Counter should still exist after close"); + Assertions.assertEquals(1, counter.getCardinality()); + EasyMock.verify(mockSelector); + } + + @Test + public void testRelocateSameBuffer() + { + aggregator.init(buffer, POSITION_1); + EasyMock.expect(mockSelector.getLong()).andReturn(123L).times(1); + EasyMock.replay(mockSelector); + aggregator.aggregate(buffer, POSITION_1); + EasyMock.verify(mockSelector); + + Bitmap64Counter originalCounter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertEquals(1, originalCounter.getCardinality()); + + aggregator.relocate(POSITION_1, POSITION_2, buffer, buffer); + + Bitmap64Counter newCounter = (Bitmap64Counter) aggregator.get(buffer, POSITION_2); + Assertions.assertNotNull(newCounter); + Assertions.assertEquals(1, newCounter.getCardinality()); + Assertions.assertSame(originalCounter, newCounter, "Relocate in same buffer should move the same counter instance"); + } + + @Test + public void testRelocateDifferentBuffers() + { + aggregator.init(buffer, POSITION_1); + EasyMock.expect(mockSelector.getLong()).andReturn(456L).times(1); + EasyMock.replay(mockSelector); + aggregator.aggregate(buffer, POSITION_1); + EasyMock.verify(mockSelector); + + Bitmap64Counter originalCounter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertEquals(1, originalCounter.getCardinality()); + + ByteBuffer newBuffer = ByteBuffer.allocate(BUFFER_CAPACITY); + aggregator.relocate(POSITION_1, POSITION_1, buffer, newBuffer); // new buffer, same position offset + + Bitmap64Counter newCounter = (Bitmap64Counter) aggregator.get(newBuffer, POSITION_1); + Assertions.assertNotNull(newCounter); + Assertions.assertEquals(1, newCounter.getCardinality()); + Assertions.assertSame( + originalCounter, + newCounter, + "Relocate to different buffer should move the same counter instance" + ); + } + + @Test + public void testUnsupportedGetOperations() + { + Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getLong(buffer, POSITION_1)); + Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getDouble(buffer, POSITION_1)); + Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getFloat(buffer, POSITION_1)); + } +} \ No newline at end of file diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactoryTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactoryTest.java index 6734184fe009..133c541b11d2 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactoryTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactoryTest.java @@ -92,14 +92,26 @@ public void testGetResultType() { Assertions.assertEquals(ColumnType.LONG, factory.getResultType()); } - + @Test public void testEqualsAndHashCode() { - Bitmap64ExactCardinalityMergeAggregatorFactory factory1 = new Bitmap64ExactCardinalityMergeAggregatorFactory(NAME, FIELD_NAME); - Bitmap64ExactCardinalityMergeAggregatorFactory factory2 = new Bitmap64ExactCardinalityMergeAggregatorFactory(NAME, FIELD_NAME); - Bitmap64ExactCardinalityMergeAggregatorFactory factoryDiffName = new Bitmap64ExactCardinalityMergeAggregatorFactory(NAME + "_diff", FIELD_NAME); - Bitmap64ExactCardinalityMergeAggregatorFactory factoryDiffFieldName = new Bitmap64ExactCardinalityMergeAggregatorFactory(NAME, FIELD_NAME + "_diff"); + Bitmap64ExactCardinalityMergeAggregatorFactory factory1 = new Bitmap64ExactCardinalityMergeAggregatorFactory( + NAME, + FIELD_NAME + ); + Bitmap64ExactCardinalityMergeAggregatorFactory factory2 = new Bitmap64ExactCardinalityMergeAggregatorFactory( + NAME, + FIELD_NAME + ); + Bitmap64ExactCardinalityMergeAggregatorFactory factoryDiffName = new Bitmap64ExactCardinalityMergeAggregatorFactory( + NAME + "_diff", + FIELD_NAME + ); + Bitmap64ExactCardinalityMergeAggregatorFactory factoryDiffFieldName = new Bitmap64ExactCardinalityMergeAggregatorFactory( + NAME, + FIELD_NAME + "_diff" + ); Assertions.assertEquals(factory1, factory2); Assertions.assertEquals(factory1.hashCode(), factory2.hashCode()); @@ -114,7 +126,11 @@ public void testEqualsAndHashCode() @Test public void testToString() { - String expected = "Bitmap64ExactCardinalityMergeAggregatorFactory { name=" + NAME + ", fieldName=" + FIELD_NAME + " }"; + String expected = "Bitmap64ExactCardinalityMergeAggregatorFactory { name=" + + NAME + + ", fieldName=" + + FIELD_NAME + + " }"; Assertions.assertEquals(expected, factory.toString()); } } \ No newline at end of file diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorTest.java new file mode 100644 index 000000000000..dcee6ebb6b44 --- /dev/null +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; + +import org.apache.druid.segment.ColumnValueSelector; +import org.easymock.EasyMock; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class Bitmap64ExactCardinalityMergeAggregatorTest +{ + private ColumnValueSelector mockSelector; + private Bitmap64ExactCardinalityMergeAggregator aggregator; + + @BeforeEach + public void setUp() + { + mockSelector = EasyMock.createMock(ColumnValueSelector.class); + aggregator = new Bitmap64ExactCardinalityMergeAggregator(mockSelector); + } + + @Test + public void testAggregateWithNonNullCounter() + { + RoaringBitmap64Counter inputCounter = new RoaringBitmap64Counter(); + inputCounter.add(123L); + inputCounter.add(456L); + + EasyMock.expect(mockSelector.getObject()).andReturn(inputCounter).once(); + EasyMock.replay(mockSelector); + + aggregator.aggregate(); + + Bitmap64Counter resultCounter = (Bitmap64Counter) aggregator.get(); + Assertions.assertEquals(2, resultCounter.getCardinality()); + + EasyMock.verify(mockSelector); + } + + @Test + public void testAggregateWithNullCounter() + { + EasyMock.expect(mockSelector.getObject()).andReturn(null).once(); + EasyMock.replay(mockSelector); + + aggregator.aggregate(); // Should not throw NPE, RoaringBitmap64Counter.fold handles null + + Bitmap64Counter resultCounter = (Bitmap64Counter) aggregator.get(); + Assertions.assertEquals(0, resultCounter.getCardinality(), "Aggregating null should not change cardinality"); + + EasyMock.verify(mockSelector); + } + + @Test + public void testAggregateMultipleCounters() + { + RoaringBitmap64Counter inputCounter1 = new RoaringBitmap64Counter(); + inputCounter1.add(10L); + inputCounter1.add(20L); + + RoaringBitmap64Counter inputCounter2 = new RoaringBitmap64Counter(); + inputCounter2.add(20L); // Duplicate with counter1 + inputCounter2.add(30L); + + EasyMock.expect(mockSelector.getObject()).andReturn(inputCounter1).once(); + EasyMock.expect(mockSelector.getObject()).andReturn(inputCounter2).once(); + EasyMock.replay(mockSelector); + + aggregator.aggregate(); + aggregator.aggregate(); + + Bitmap64Counter resultCounter = (Bitmap64Counter) aggregator.get(); + Assertions.assertEquals(3, resultCounter.getCardinality()); // 10, 20, 30 + + EasyMock.verify(mockSelector); + } + + @Test + public void testAggregateMultipleCountersIncludingNull() + { + RoaringBitmap64Counter inputCounter1 = new RoaringBitmap64Counter(); + inputCounter1.add(10L); + inputCounter1.add(20L); + + RoaringBitmap64Counter inputCounter3 = new RoaringBitmap64Counter(); + inputCounter3.add(20L); + inputCounter3.add(30L); + + EasyMock.expect(mockSelector.getObject()).andReturn(inputCounter1).once(); + EasyMock.expect(mockSelector.getObject()).andReturn(null).once(); // Null counter + EasyMock.expect(mockSelector.getObject()).andReturn(inputCounter3).once(); + EasyMock.replay(mockSelector); + + aggregator.aggregate(); // counter1 + aggregator.aggregate(); // null + aggregator.aggregate(); // counter3 + + Bitmap64Counter resultCounter = (Bitmap64Counter) aggregator.get(); + Assertions.assertEquals(3, resultCounter.getCardinality()); // 10, 20, 30 + + EasyMock.verify(mockSelector); + } + + @Test + public void testGetInitialState() + { + Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(); + Assertions.assertNotNull(counter); + Assertions.assertEquals(0, counter.getCardinality()); + } + + @Test + public void testClose() + { + aggregator.close(); + Assertions.assertNull(aggregator.get(), "Bitmap should be null after close"); + } + + @Test + public void testCloseIsIdempotent() + { + aggregator.close(); + Assertions.assertNull(aggregator.get(), "Bitmap should be null after first close"); + aggregator.close(); + Assertions.assertNull(aggregator.get(), "Bitmap should still be null after second close"); + } + + @Test + public void testGetFloatThrowsException() + { + Assertions.assertThrows(UnsupportedOperationException.class, () -> { + aggregator.getFloat(); + }); + } + + @Test + public void testGetLongThrowsException() + { + Assertions.assertThrows(UnsupportedOperationException.class, () -> { + aggregator.getLong(); + }); + } +} \ No newline at end of file diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeBufferAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeBufferAggregatorTest.java new file mode 100644 index 000000000000..9307117a0aa4 --- /dev/null +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeBufferAggregatorTest.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; + +import org.apache.druid.segment.ColumnValueSelector; +import org.easymock.EasyMock; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; + +public class Bitmap64ExactCardinalityMergeBufferAggregatorTest +{ + private ColumnValueSelector mockSelector; + private Bitmap64ExactCardinalityMergeBufferAggregator aggregator; + private ByteBuffer buffer; + private static final int BUFFER_CAPACITY = 1024; + private static final int POSITION_1 = 0; + private static final int POSITION_2 = 100; + + @BeforeEach + public void setUp() + { + mockSelector = EasyMock.createMock(ColumnValueSelector.class); + aggregator = new Bitmap64ExactCardinalityMergeBufferAggregator(mockSelector); + buffer = ByteBuffer.allocate(BUFFER_CAPACITY); + } + + @Test + public void testInit() + { + aggregator.init(buffer, POSITION_1); + Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertNotNull(counter); + Assertions.assertEquals(0, counter.getCardinality()); + } + + @Test + public void testAggregateWithNonNullCounter() + { + aggregator.init(buffer, POSITION_1); + RoaringBitmap64Counter inputCounter = new RoaringBitmap64Counter(); + inputCounter.add(123L); + + EasyMock.expect(mockSelector.getObject()).andReturn(inputCounter).once(); + EasyMock.replay(mockSelector); + + aggregator.aggregate(buffer, POSITION_1); + + Bitmap64Counter resultCounter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertEquals(1, resultCounter.getCardinality()); + + EasyMock.verify(mockSelector); + } + + @Test + public void testAggregateWithNullCounterFromSelector() + { + aggregator.init(buffer, POSITION_1); + Bitmap64Counter initialCounter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + long initialCardinality = initialCounter.getCardinality(); + + EasyMock.expect(mockSelector.getObject()).andReturn(null).once(); + EasyMock.replay(mockSelector); + + aggregator.aggregate(buffer, POSITION_1); // Should not throw NPE, and not change the counter + + Bitmap64Counter resultCounter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertEquals( + initialCardinality, + resultCounter.getCardinality(), + "Aggregating null from selector should not change cardinality" + ); + + EasyMock.verify(mockSelector); + } + + @Test + public void testAggregateMultipleCounters() + { + aggregator.init(buffer, POSITION_1); + RoaringBitmap64Counter inputCounter1 = new RoaringBitmap64Counter(); + inputCounter1.add(10L); + RoaringBitmap64Counter inputCounter2 = new RoaringBitmap64Counter(); + inputCounter2.add(20L); + inputCounter2.add(10L); // duplicate + + EasyMock.expect(mockSelector.getObject()).andReturn(inputCounter1).once(); + EasyMock.expect(mockSelector.getObject()).andReturn(inputCounter2).once(); + EasyMock.replay(mockSelector); + + aggregator.aggregate(buffer, POSITION_1); + aggregator.aggregate(buffer, POSITION_1); + + Bitmap64Counter resultCounter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertEquals(2, resultCounter.getCardinality()); // 10, 20 + + EasyMock.verify(mockSelector); + } + + @Test + public void testAggregateAtDifferentPositions() + { + aggregator.init(buffer, POSITION_1); + aggregator.init(buffer, POSITION_2); + + RoaringBitmap64Counter counterForPos1 = new RoaringBitmap64Counter(); + counterForPos1.add(1L); + RoaringBitmap64Counter counterForPos2 = new RoaringBitmap64Counter(); + counterForPos2.add(2L); + + EasyMock.expect(mockSelector.getObject()).andReturn(counterForPos1).once(); // For POSITION_1 + EasyMock.expect(mockSelector.getObject()).andReturn(counterForPos2).once(); // For POSITION_2 + EasyMock.replay(mockSelector); + + aggregator.aggregate(buffer, POSITION_1); + aggregator.aggregate(buffer, POSITION_2); + + Bitmap64Counter result1 = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertEquals(1, result1.getCardinality()); + + Bitmap64Counter result2 = (Bitmap64Counter) aggregator.get(buffer, POSITION_2); + Assertions.assertEquals(1, result2.getCardinality()); + + Assertions.assertNotSame(result1, result2); + EasyMock.verify(mockSelector); + } + + @Test + public void testCloseClearsCache() + { + aggregator.init(buffer, POSITION_1); + RoaringBitmap64Counter inputCounter = new RoaringBitmap64Counter(); + inputCounter.add(1L); + EasyMock.expect(mockSelector.getObject()).andReturn(inputCounter).once(); + EasyMock.replay(mockSelector); + aggregator.aggregate(buffer, POSITION_1); + EasyMock.verify(mockSelector); + + Bitmap64Counter counterBeforeClose = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertNotNull(counterBeforeClose); + + aggregator.close(); + + // After close, accessing the same buffer and position might throw NPE or return null + // depending on how IdentityHashMap and Int2ObjectMap behave after clearing. + // The current implementation of get() would lead to NPE if buffer is not in counterCache. + Assertions.assertThrows( + NullPointerException.class, + () -> aggregator.get(buffer, POSITION_1), + "Accessing counter from cleared cache should fail or return null" + ); + } + + @Test + public void testRelocateSameBuffer() + { + aggregator.init(buffer, POSITION_1); + RoaringBitmap64Counter initialCounterVal = new RoaringBitmap64Counter(); + initialCounterVal.add(123L); + + EasyMock.expect(mockSelector.getObject()).andReturn(initialCounterVal).once(); + EasyMock.replay(mockSelector); + aggregator.aggregate(buffer, POSITION_1); + EasyMock.verify(mockSelector); + + Bitmap64Counter originalCounter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertEquals(1, originalCounter.getCardinality()); + + aggregator.relocate(POSITION_1, POSITION_2, buffer, buffer); + + Bitmap64Counter newCounter = (Bitmap64Counter) aggregator.get(buffer, POSITION_2); + Assertions.assertNotNull(newCounter); + Assertions.assertEquals(1, newCounter.getCardinality()); + Assertions.assertSame(originalCounter, newCounter); + } + + @Test + public void testRelocateDifferentBuffers() + { + aggregator.init(buffer, POSITION_1); + RoaringBitmap64Counter initialCounterVal = new RoaringBitmap64Counter(); + initialCounterVal.add(456L); + + EasyMock.expect(mockSelector.getObject()).andReturn(initialCounterVal).once(); + EasyMock.replay(mockSelector); + aggregator.aggregate(buffer, POSITION_1); + EasyMock.verify(mockSelector); + + Bitmap64Counter originalCounter = (Bitmap64Counter) aggregator.get(buffer, POSITION_1); + Assertions.assertEquals(1, originalCounter.getCardinality()); + + ByteBuffer newBuffer = ByteBuffer.allocate(BUFFER_CAPACITY); + aggregator.relocate(POSITION_1, POSITION_1, buffer, newBuffer); + + Bitmap64Counter newCounter = (Bitmap64Counter) aggregator.get(newBuffer, POSITION_1); + Assertions.assertNotNull(newCounter); + Assertions.assertEquals(1, newCounter.getCardinality()); + Assertions.assertSame(originalCounter, newCounter); + } + + @Test + public void testUnsupportedGetOperations() + { + aggregator.init(buffer, POSITION_1); // Ensure the entry exists in cache to avoid NPE on get() + Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getLong(buffer, POSITION_1)); + Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getDouble(buffer, POSITION_1)); + Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getFloat(buffer, POSITION_1)); + } +} \ No newline at end of file diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java index f6208dabc540..7745753478d5 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java @@ -33,7 +33,7 @@ public void testRegisterSerde() ComplexMetrics.unregisterSerde(Bitmap64ExactCardinalityModule.BUILD_TYPE_NAME); ComplexMetrics.unregisterSerde(Bitmap64ExactCardinalityModule.MERGE_TYPE_NAME); - Bitmap64ExactCardinalityModule.registerSerde(); + Bitmap64ExactCardinalityModule.registerSerde(); ComplexMetricSerde typeNameSerde = ComplexMetrics.getSerdeForType(Bitmap64ExactCardinalityModule.TYPE_NAME); Assertions.assertNotNull(typeNameSerde); @@ -42,7 +42,7 @@ public void testRegisterSerde() ComplexMetricSerde buildTypeNameSerde = ComplexMetrics.getSerdeForType(Bitmap64ExactCardinalityModule.BUILD_TYPE_NAME); Assertions.assertNotNull(buildTypeNameSerde); Assertions.assertInstanceOf(Bitmap64ExactCardinalityBuildComplexMetricSerde.class, buildTypeNameSerde); - + ComplexMetricSerde mergeTypeNameSerde = ComplexMetrics.getSerdeForType(Bitmap64ExactCardinalityModule.MERGE_TYPE_NAME); Assertions.assertNotNull(mergeTypeNameSerde); Assertions.assertInstanceOf(Bitmap64ExactCardinalityMergeComplexMetricSerde.class, mergeTypeNameSerde); diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregatorTest.java index 6c1a83de56a5..e1e4621af1fc 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregatorTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregatorTest.java @@ -74,7 +74,7 @@ public void testCompute() Map combinedAggregators = ImmutableMap.of(FIELD_NAME, counter); Assertions.assertEquals(3L, postAggregator.compute(combinedAggregators)); - + Bitmap64Counter emptyCounter = new RoaringBitmap64Counter(); Map combinedAggregatorsEmpty = ImmutableMap.of(FIELD_NAME, emptyCounter); Assertions.assertEquals(0L, postAggregator.compute(combinedAggregatorsEmpty)); @@ -83,7 +83,7 @@ public void testCompute() @Test public void testGetType() { - Assertions.assertEquals(ColumnType.LONG, postAggregator.getType(null)); + Assertions.assertEquals(ColumnType.LONG, postAggregator.getType(null)); } @Test @@ -100,24 +100,42 @@ public void testGetCacheKey() .build(); Assertions.assertArrayEquals(expectedKey, postAggregator.getCacheKey()); - Bitmap64ExactCardinalityPostAggregator postAggregator2 = new Bitmap64ExactCardinalityPostAggregator(NAME, FIELD_NAME); + Bitmap64ExactCardinalityPostAggregator postAggregator2 = new Bitmap64ExactCardinalityPostAggregator( + NAME, + FIELD_NAME + ); Assertions.assertArrayEquals(postAggregator.getCacheKey(), postAggregator2.getCacheKey()); - Bitmap64ExactCardinalityPostAggregator postAggregatorDiffFieldName = new Bitmap64ExactCardinalityPostAggregator(NAME, FIELD_NAME + "_diff"); + Bitmap64ExactCardinalityPostAggregator postAggregatorDiffFieldName = new Bitmap64ExactCardinalityPostAggregator( + NAME, + FIELD_NAME + + "_diff" + ); Assertions.assertFalse(Arrays.equals(postAggregator.getCacheKey(), postAggregatorDiffFieldName.getCacheKey())); - - Bitmap64ExactCardinalityPostAggregator postAggregatorDiffName = new Bitmap64ExactCardinalityPostAggregator(NAME + "_diff", FIELD_NAME); + + Bitmap64ExactCardinalityPostAggregator postAggregatorDiffName = new Bitmap64ExactCardinalityPostAggregator( + NAME + + "_diff", + FIELD_NAME + ); // Cache key for PostAggregator does not include its own name, only dependent fieldName - Assertions.assertArrayEquals(postAggregator.getCacheKey(), postAggregatorDiffName.getCacheKey()); + Assertions.assertArrayEquals(postAggregator.getCacheKey(), postAggregatorDiffName.getCacheKey()); } - + @Test public void testEqualsAndHashCode() { Bitmap64ExactCardinalityPostAggregator pa1 = new Bitmap64ExactCardinalityPostAggregator(NAME, FIELD_NAME); Bitmap64ExactCardinalityPostAggregator pa2 = new Bitmap64ExactCardinalityPostAggregator(NAME, FIELD_NAME); - Bitmap64ExactCardinalityPostAggregator paDiffName = new Bitmap64ExactCardinalityPostAggregator(NAME + "_diff", FIELD_NAME); - Bitmap64ExactCardinalityPostAggregator paDiffFieldName = new Bitmap64ExactCardinalityPostAggregator(NAME, FIELD_NAME + "_diff"); + Bitmap64ExactCardinalityPostAggregator paDiffName = new Bitmap64ExactCardinalityPostAggregator( + NAME + "_diff", + FIELD_NAME + ); + Bitmap64ExactCardinalityPostAggregator paDiffFieldName = new Bitmap64ExactCardinalityPostAggregator( + NAME, + FIELD_NAME + + "_diff" + ); Assertions.assertEquals(pa1, pa2); Assertions.assertEquals(pa1.hashCode(), pa2.hashCode()); From 8aeb38d8c17c4720d7a66e032d510a25d9005366 Mon Sep 17 00:00:00 2001 From: GWphua Date: Tue, 20 May 2025 17:20:04 +0800 Subject: [PATCH 07/65] tidy up for tests and Counter --- ...ap64ExactCardinalityAggregatorFactory.java | 2 +- .../bitmap64/RoaringBitmap64Counter.java | 5 +++- ...64ExactCardinalityMergeAggregatorTest.java | 28 +++++-------------- .../Bitmap64ExactCardinalityModuleTest.java | 4 --- 4 files changed, 12 insertions(+), 27 deletions(-) diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java index bb898a3011ff..f97960f64437 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java @@ -167,7 +167,7 @@ public int getMaxIntermediateSize() } @Override - public boolean equals(final Object object) + public boolean equals(Object object) { if (this == object) { return true; diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/RoaringBitmap64Counter.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/RoaringBitmap64Counter.java index 1d278e10336c..e0c0b31d25fa 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/RoaringBitmap64Counter.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/RoaringBitmap64Counter.java @@ -75,7 +75,10 @@ public long getCardinality() @Override public Bitmap64Counter fold(Bitmap64Counter rhs) { - bitmap.or(((RoaringBitmap64Counter) rhs).bitmap); + if (rhs != null) { + bitmap.or(((RoaringBitmap64Counter) rhs).bitmap); + } + return this; } diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorTest.java index dcee6ebb6b44..81fde53aa6a8 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorTest.java @@ -50,6 +50,7 @@ public void testAggregateWithNonNullCounter() aggregator.aggregate(); Bitmap64Counter resultCounter = (Bitmap64Counter) aggregator.get(); + Assertions.assertNotNull(resultCounter); Assertions.assertEquals(2, resultCounter.getCardinality()); EasyMock.verify(mockSelector); @@ -64,6 +65,7 @@ public void testAggregateWithNullCounter() aggregator.aggregate(); // Should not throw NPE, RoaringBitmap64Counter.fold handles null Bitmap64Counter resultCounter = (Bitmap64Counter) aggregator.get(); + Assertions.assertNotNull(resultCounter); Assertions.assertEquals(0, resultCounter.getCardinality(), "Aggregating null should not change cardinality"); EasyMock.verify(mockSelector); @@ -88,6 +90,7 @@ public void testAggregateMultipleCounters() aggregator.aggregate(); Bitmap64Counter resultCounter = (Bitmap64Counter) aggregator.get(); + Assertions.assertNotNull(resultCounter); Assertions.assertEquals(3, resultCounter.getCardinality()); // 10, 20, 30 EasyMock.verify(mockSelector); @@ -114,6 +117,7 @@ public void testAggregateMultipleCountersIncludingNull() aggregator.aggregate(); // counter3 Bitmap64Counter resultCounter = (Bitmap64Counter) aggregator.get(); + Assertions.assertNotNull(resultCounter); Assertions.assertEquals(3, resultCounter.getCardinality()); // 10, 20, 30 EasyMock.verify(mockSelector); @@ -133,29 +137,11 @@ public void testClose() aggregator.close(); Assertions.assertNull(aggregator.get(), "Bitmap should be null after close"); } - - @Test - public void testCloseIsIdempotent() - { - aggregator.close(); - Assertions.assertNull(aggregator.get(), "Bitmap should be null after first close"); - aggregator.close(); - Assertions.assertNull(aggregator.get(), "Bitmap should still be null after second close"); - } - - @Test - public void testGetFloatThrowsException() - { - Assertions.assertThrows(UnsupportedOperationException.class, () -> { - aggregator.getFloat(); - }); - } @Test - public void testGetLongThrowsException() + public void testUnsupported() { - Assertions.assertThrows(UnsupportedOperationException.class, () -> { - aggregator.getLong(); - }); + Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getFloat()); + Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getLong()); } } \ No newline at end of file diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java index 7745753478d5..f0c9522a8e66 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java @@ -29,10 +29,6 @@ public class Bitmap64ExactCardinalityModuleTest @Test public void testRegisterSerde() { - ComplexMetrics.unregisterSerde(Bitmap64ExactCardinalityModule.TYPE_NAME); - ComplexMetrics.unregisterSerde(Bitmap64ExactCardinalityModule.BUILD_TYPE_NAME); - ComplexMetrics.unregisterSerde(Bitmap64ExactCardinalityModule.MERGE_TYPE_NAME); - Bitmap64ExactCardinalityModule.registerSerde(); ComplexMetricSerde typeNameSerde = ComplexMetrics.getSerdeForType(Bitmap64ExactCardinalityModule.TYPE_NAME); From 703c7a9f701e603eb8bf5a60188d583e6db9c2c9 Mon Sep 17 00:00:00 2001 From: GWphua Date: Tue, 20 May 2025 17:36:31 +0800 Subject: [PATCH 08/65] Fix checkstyle --- ...ExactCardinalityAggregatorFactoryTest.java | 10 ++++-- ...CardinalityBuildAggregatorFactoryTest.java | 2 +- ...64ExactCardinalityBuildAggregatorTest.java | 33 ++++--------------- ...tCardinalityBuildBufferAggregatorTest.java | 2 +- ...CardinalityMergeAggregatorFactoryTest.java | 2 +- ...64ExactCardinalityMergeAggregatorTest.java | 6 ++-- ...tCardinalityMergeBufferAggregatorTest.java | 2 +- .../Bitmap64ExactCardinalityModuleTest.java | 2 +- ...p64ExactCardinalityPostAggregatorTest.java | 2 +- 9 files changed, 23 insertions(+), 38 deletions(-) diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactoryTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactoryTest.java index f920be781c81..24c18f57a298 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactoryTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactoryTest.java @@ -26,10 +26,10 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.column.ColumnType; +import org.easymock.EasyMock; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.easymock.EasyMock; import java.util.Arrays; import java.util.Collections; @@ -191,7 +191,11 @@ public void testEqualsAndHashCode() @Test public void testToString() { - String expected = "TestBitmap64ExactCardinalityAggregatorFactory { name=" + NAME + ", fieldName=" + FIELD_NAME + " }"; + String expected = "TestBitmap64ExactCardinalityAggregatorFactory { name=" + + NAME + + ", fieldName=" + + FIELD_NAME + + " }"; Assertions.assertEquals(expected, factory.toString()); } @@ -292,4 +296,4 @@ public void testMakeAggregateCombiner() Assertions.assertEquals(Bitmap64Counter.class, combiner.classOfObject()); } -} \ No newline at end of file +} diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java index 99b234db3124..202171e08058 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java @@ -129,4 +129,4 @@ public void testToString() String expected = "Bitmap64ExactCardinalityBuildAggregatorFactory { name=" + NAME + ", fieldName=" + FIELD_NAME + " }"; Assertions.assertEquals(expected, factory.toString()); } -} \ No newline at end of file +} diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorTest.java index f39b16e5f071..01c67419aac7 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorTest.java @@ -46,6 +46,7 @@ public void testAggregateSingleValue() aggregator.aggregate(); Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(); + Assertions.assertNotNull(counter); Assertions.assertEquals(1, counter.getCardinality()); EasyMock.verify(mockSelector); @@ -64,6 +65,7 @@ public void testAggregateMultipleDistinctValues() aggregator.aggregate(); Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(); + Assertions.assertNotNull(counter); Assertions.assertEquals(3, counter.getCardinality()); EasyMock.verify(mockSelector); @@ -82,6 +84,7 @@ public void testAggregateMultipleValuesWithDuplicates() aggregator.aggregate(); Bitmap64Counter counter = (Bitmap64Counter) aggregator.get(); + Assertions.assertNotNull(counter); Assertions.assertEquals(2, counter.getCardinality()); EasyMock.verify(mockSelector); @@ -103,31 +106,9 @@ public void testClose() } @Test - public void testCloseIsIdempotent() - { - aggregator.close(); - Assertions.assertNull(aggregator.get(), "Bitmap should be null after first close"); - aggregator.close(); // Call close again - Assertions.assertNull(aggregator.get(), "Bitmap should still be null after second close"); - } - - @Test - public void testGetFloatThrowsException() - { - Assertions.assertThrows( - UnsupportedOperationException.class, () -> { - aggregator.getFloat(); - } - ); - } - - @Test - public void testGetLongThrowsException() + public void testUnsupportedGetOperations() { - Assertions.assertThrows( - UnsupportedOperationException.class, () -> { - aggregator.getLong(); - } - ); + Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getFloat()); + Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getLong()); } -} \ No newline at end of file +} diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregatorTest.java index 66b24f134cb0..141c0c0540f9 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregatorTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregatorTest.java @@ -220,4 +220,4 @@ public void testUnsupportedGetOperations() Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getDouble(buffer, POSITION_1)); Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getFloat(buffer, POSITION_1)); } -} \ No newline at end of file +} diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactoryTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactoryTest.java index 133c541b11d2..260a1d9c8691 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactoryTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactoryTest.java @@ -133,4 +133,4 @@ public void testToString() + " }"; Assertions.assertEquals(expected, factory.toString()); } -} \ No newline at end of file +} diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorTest.java index 81fde53aa6a8..5c1e9bda9f52 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorTest.java @@ -96,7 +96,7 @@ public void testAggregateMultipleCounters() EasyMock.verify(mockSelector); } - @Test + @Test public void testAggregateMultipleCountersIncludingNull() { RoaringBitmap64Counter inputCounter1 = new RoaringBitmap64Counter(); @@ -139,9 +139,9 @@ public void testClose() } @Test - public void testUnsupported() + public void testUnsupportedGetOperations() { Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getFloat()); Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getLong()); } -} \ No newline at end of file +} diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeBufferAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeBufferAggregatorTest.java index 9307117a0aa4..7ca2a070e216 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeBufferAggregatorTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeBufferAggregatorTest.java @@ -225,4 +225,4 @@ public void testUnsupportedGetOperations() Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getDouble(buffer, POSITION_1)); Assertions.assertThrows(UnsupportedOperationException.class, () -> aggregator.getFloat(buffer, POSITION_1)); } -} \ No newline at end of file +} diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java index f0c9522a8e66..c7d7396cb198 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java @@ -43,4 +43,4 @@ public void testRegisterSerde() Assertions.assertNotNull(mergeTypeNameSerde); Assertions.assertInstanceOf(Bitmap64ExactCardinalityMergeComplexMetricSerde.class, mergeTypeNameSerde); } -} \ No newline at end of file +} diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregatorTest.java index e1e4621af1fc..1e8628a78ffd 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregatorTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregatorTest.java @@ -156,4 +156,4 @@ public void testToString() String expected = "Bitmap64ExactCardinalityPostAggregator{name='" + NAME + "', field=" + FIELD_NAME + "}"; Assertions.assertEquals(expected, postAggregator.toString()); } -} \ No newline at end of file +} From cafa112e3e0de882e452a40420364bc0ab97219b Mon Sep 17 00:00:00 2001 From: GWphua Date: Tue, 20 May 2025 17:48:53 +0800 Subject: [PATCH 09/65] Tidy README --- .../druid-exact-cardinality/README.md | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/extensions-contrib/druid-exact-cardinality/README.md b/extensions-contrib/druid-exact-cardinality/README.md index eee2c5ebe77e..cdb4c4b16949 100644 --- a/extensions-contrib/druid-exact-cardinality/README.md +++ b/extensions-contrib/druid-exact-cardinality/README.md @@ -19,7 +19,38 @@ This module provides exact cardinality count for long type column. -Usage for `Bitmap64ExactCardinalityBuild` and `Bitmap64ExactCardinalityMerge`: +## How it Works + +This extension calculates the exact cardinality (distinct count) of values in a column of `LONG` data type. It uses [Roaring Bitmaps](https://roaringbitmap.org/), specifically the `Roaring64NavigableMap` variant, as its underlying data structure. + +Here's a brief overview of its operation: + +1. **Data Ingestion (`Bitmap64ExactCardinalityBuild`)**: + * When new data is ingested, and this aggregator is applied to a `LONG` column (specified by `fieldName`), each unique long value from the input column is added to a `Roaring64NavigableMap`. + * This bitmap efficiently stores the set of unique 64-bit integers encountered. + +2. **Aggregation and Rollup (`Bitmap64ExactCardinalityMerge`)**: + * During rollup or when combining results from different segments/queries, the aggregator merges multiple `Roaring64NavigableMap` instances. + * This merge operation is a bitwise `OR` on the bitmaps, which correctly computes the union of the distinct values from each source. + * The intermediate object stored and transferred is the serialized form of the `Roaring64NavigableMap`. + +3. **Query Time**: + * At query time, the `Bitmap64ExactCardinalityMerge` aggregator is typically used to combine the bitmaps from the selected segments. + * The final computation involves calling `getLongCardinality()` on the resulting `Roaring64NavigableMap`, which returns the exact count of distinct long values. + * A `Bitmap64ExactCardinalityPostAggregator` is also available for post-processing, allowing the cardinality result to be used in further calculations. + +**Key Characteristics**: + +* **Exactness**: Provides precise distinct counts, unlike approximation algorithms (e.g., HyperLogLog). +* **Input Type**: Designed for `LONG` (64-bit integer) columns. If your data is in string format, it must be converted to longs first (e.g., via dictionary encoding or hashing) to use this aggregator. +* **Data Structure**: Leverages Roaring Bitmaps for memory efficiency and fast set operations (add, union, cardinality check) compared to traditional `HashSet` approaches, especially for certain data distributions. +* **Trade-offs**: + * **Memory**: While Roaring Bitmaps are efficient, storing exact unique values will generally consume more memory than sketch-based approximate algorithms, especially for very high cardinality dimensions with uniformly distributed values. + * **Performance**: The performance can be very good, but for extremely high cardinalities, the cost of merging large bitmaps might be higher than that of merging smaller sketches. + +This aggregator is suitable for use cases where the precision of the distinct count is paramount and the input data is or can be represented as long values. + +## Usage Examples Kafka ingestion task spec: ``` { @@ -183,8 +214,8 @@ WHERE "__time" >= CURRENT_TIMESTAMP - INTERVAL '1' DAY GROUP BY key ``` -Note: this `longExactCardinality` aggregator is recommended to use in `timeseries` type query though it also works for `topN` -and `groupBy` query. eg: +Note: this `longExactCardinality` aggregator is recommended for use in `timeseries` type queries, though it also works for `topN` +and `groupBy` queries. e.g.: ``` { "queryType": "timeseries", From 53d30b37ba160f98d212db375fdecce3f8e15d3b Mon Sep 17 00:00:00 2001 From: GWphua Date: Wed, 21 May 2025 10:20:39 +0800 Subject: [PATCH 10/65] Tidy review suggestions --- .../Bitmap64ExactCardinalityObjectStrategy.java | 14 ++++++++++---- .../bitmap64/RoaringBitmap64Counter.java | 17 +++++++++-------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityObjectStrategy.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityObjectStrategy.java index 9fcbeb7e29d8..df3e5c4eccae 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityObjectStrategy.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityObjectStrategy.java @@ -19,10 +19,12 @@ package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; -import org.apache.druid.segment.data.ObjectStrategy; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; import javax.annotation.Nullable; -import java.nio.ByteBuffer; + +import org.apache.druid.segment.data.ObjectStrategy; public class Bitmap64ExactCardinalityObjectStrategy implements ObjectStrategy { @@ -40,8 +42,12 @@ public Class getClazz() public Bitmap64Counter fromByteBuffer(ByteBuffer buffer, int numBytes) { final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes); - byte[] bytes = new byte[readOnlyBuffer.remaining()]; + + if (readOnlyBuffer.remaining() < numBytes) { + throw new BufferUnderflowException(); + } + + byte[] bytes = new byte[numBytes]; readOnlyBuffer.get(bytes, 0, numBytes); return RoaringBitmap64Counter.fromBytes(bytes); } diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/RoaringBitmap64Counter.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/RoaringBitmap64Counter.java index e0c0b31d25fa..ed82265c11d5 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/RoaringBitmap64Counter.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/RoaringBitmap64Counter.java @@ -19,7 +19,6 @@ package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; -import com.google.common.base.Throwables; import org.apache.druid.java.util.common.logger.Logger; import org.roaringbitmap.longlong.Roaring64NavigableMap; @@ -31,9 +30,9 @@ public class RoaringBitmap64Counter implements Bitmap64Counter { - private static Logger logger = new Logger(RoaringBitmap64Counter.class); + private static final Logger log = new Logger(RoaringBitmap64Counter.class); - private Roaring64NavigableMap bitmap; + private final Roaring64NavigableMap bitmap; public RoaringBitmap64Counter() { @@ -47,17 +46,20 @@ private RoaringBitmap64Counter(Roaring64NavigableMap bitmap) public static RoaringBitmap64Counter fromBytes(byte[] bytes) { - ByteArrayInputStream byteIn = new ByteArrayInputStream(bytes); try { + ByteArrayInputStream byteIn = new ByteArrayInputStream(bytes); DataInputStream in = new DataInputStream(byteIn); Roaring64NavigableMap bitmap = new Roaring64NavigableMap(); bitmap.deserialize(in); return new RoaringBitmap64Counter(bitmap); } catch (Exception e) { - logger.info(e.getMessage(), e); + log.error(e, "Failed to deserialize RoaringBitmap64Counter from bytes"); + throw new RuntimeException( + "Failed to deserialize RoaringBitmap64Counter from bytes. Error: " + e.getMessage(), + e + ); } - return null; } @Override @@ -92,8 +94,7 @@ public ByteBuffer toByteBuffer() return ByteBuffer.wrap(out.toByteArray()); } catch (Exception e) { - throw Throwables.propagate(e); + throw new RuntimeException(e); } } - } From 9b2876641a0c1433f96c66f75f6936b8681de329 Mon Sep 17 00:00:00 2001 From: GWphua Date: Wed, 21 May 2025 10:28:02 +0800 Subject: [PATCH 11/65] Fix checkstyle --- .../bitmap64/Bitmap64ExactCardinalityObjectStrategy.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityObjectStrategy.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityObjectStrategy.java index df3e5c4eccae..309f4250cf86 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityObjectStrategy.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityObjectStrategy.java @@ -19,12 +19,11 @@ package org.apache.druid.query.aggregation.exact.cardinality.bitmap64; -import java.nio.BufferUnderflowException; -import java.nio.ByteBuffer; +import org.apache.druid.segment.data.ObjectStrategy; import javax.annotation.Nullable; - -import org.apache.druid.segment.data.ObjectStrategy; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; public class Bitmap64ExactCardinalityObjectStrategy implements ObjectStrategy { From cb52b82bacf3ff16003e741b1422f506e6f48d3f Mon Sep 17 00:00:00 2001 From: GWphua Date: Wed, 21 May 2025 10:37:39 +0800 Subject: [PATCH 12/65] Clean up dependencies in pom file --- .../druid-exact-cardinality/pom.xml | 62 ++----------------- ...ap64ExactCardinalityAggregatorFactory.java | 4 -- 2 files changed, 5 insertions(+), 61 deletions(-) diff --git a/extensions-contrib/druid-exact-cardinality/pom.xml b/extensions-contrib/druid-exact-cardinality/pom.xml index 4ba395cef328..2c96578562d6 100644 --- a/extensions-contrib/druid-exact-cardinality/pom.xml +++ b/extensions-contrib/druid-exact-cardinality/pom.xml @@ -40,16 +40,6 @@ calcite-core provided - - org.apache.commons - commons-math3 - - - org.apache.druid - druid-server - ${project.parent.version} - provided - org.apache.druid druid-processing @@ -78,13 +68,15 @@ provided - commons-codec - commons-codec + org.roaringbitmap + RoaringBitmap + 0.9.49 provided it.unimi.dsi - fastutil + fastutil-core + 8.5.4 provided @@ -103,31 +95,6 @@ jackson-databind provided - - com.fasterxml.jackson.datatype - jackson-datatype-guava - provided - - - com.fasterxml.jackson.datatype - jackson-datatype-joda - provided - - - com.fasterxml.jackson.dataformat - jackson-dataformat-smile - provided - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - provided - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-smile-provider - provided - @@ -135,11 +102,6 @@ junit-jupiter-api test - - org.junit.jupiter - junit-jupiter-engine - test - org.easymock easymock @@ -152,20 +114,6 @@ test-jar test - - org.apache.druid - druid-server - ${project.parent.version} - test - test-jar - - - org.apache.druid - druid-sql - ${project.parent.version} - test-jar - test - diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java index f97960f64437..f27eaad6e34f 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java @@ -25,7 +25,6 @@ import org.apache.druid.query.aggregation.ObjectAggregateCombiner; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnValueSelector; -import org.jetbrains.annotations.NotNull; import javax.annotation.Nullable; import java.util.Collections; @@ -42,10 +41,8 @@ public abstract class Bitmap64ExactCardinalityAggregatorFactory extends Aggregat static final Comparator COMPARATOR = Comparator.nullsFirst(Comparator.comparingLong(Bitmap64Counter::getCardinality)); - @NotNull private final String name; - @NotNull private final String fieldName; Bitmap64ExactCardinalityAggregatorFactory( @@ -71,7 +68,6 @@ public String getFieldName() } @Override - @NotNull public List requiredFields() { return Collections.singletonList(fieldName); From 1db8df4ed4a6e3608c718d9eb78c4e209191fef7 Mon Sep 17 00:00:00 2001 From: GWphua Date: Wed, 21 May 2025 11:00:49 +0800 Subject: [PATCH 13/65] Docs for extension --- docs/configuration/extensions.md | 1 + .../druid-exact-cardinality.md | 252 ++++++++++++------ 2 files changed, 165 insertions(+), 88 deletions(-) rename extensions-contrib/druid-exact-cardinality/README.md => docs/development/extensions-contrib/druid-exact-cardinality.md (51%) diff --git a/docs/configuration/extensions.md b/docs/configuration/extensions.md index fb040188ba38..97e5c368efe9 100644 --- a/docs/configuration/extensions.md +++ b/docs/configuration/extensions.md @@ -86,6 +86,7 @@ All of these community extensions can be downloaded using [pull-deps](../operati |druid-ddsketch|Support for DDSketch approximate quantiles based on [DDSketch](https://github.com/datadog/sketches-java) | [link](../development/extensions-contrib/ddsketch-quantiles.md)| |druid-deltalake-extensions|Support for ingesting Delta Lake tables.|[link](../development/extensions-contrib/delta-lake.md)| |druid-distinctcount|DistinctCount aggregator|[link](../development/extensions-contrib/distinctcount.md)| +|druid-exact-cardinality|Support for exact cardinality counting using Roaring Bitmaps.|[link](../development/extensions-contrib/druid-exact-cardinality.md)| |druid-iceberg-extensions|Support for ingesting Iceberg tables.|[link](../development/extensions-contrib/iceberg.md)| |druid-redis-cache|A cache implementation for Druid based on Redis.|[link](../development/extensions-contrib/redis-cache.md)| |druid-time-min-max|Min/Max aggregator for timestamp.|[link](../development/extensions-contrib/time-min-max.md)| diff --git a/extensions-contrib/druid-exact-cardinality/README.md b/docs/development/extensions-contrib/druid-exact-cardinality.md similarity index 51% rename from extensions-contrib/druid-exact-cardinality/README.md rename to docs/development/extensions-contrib/druid-exact-cardinality.md index cdb4c4b16949..d5cc3efe000f 100644 --- a/extensions-contrib/druid-exact-cardinality/README.md +++ b/docs/development/extensions-contrib/druid-exact-cardinality.md @@ -1,3 +1,8 @@ +--- +id: druid-exact-cardinality +title: "Exact Cardinality" +--- + -This module provides exact cardinality count for long type column. +This extension provides exact cardinality counting functionality for LONG type columns using [Roaring Bitmaps](https://roaringbitmap.org/). Unlike approximate cardinality aggregators like HyperLogLog, this aggregator provides precise distinct counts. + +## Installation + +To use this Apache Druid extension, [include](../../configuration/extensions.md#loading-extensions) `druid-exact-cardinality` in the extensions load list. ## How it Works -This extension calculates the exact cardinality (distinct count) of values in a column of `LONG` data type. It uses [Roaring Bitmaps](https://roaringbitmap.org/), specifically the `Roaring64NavigableMap` variant, as its underlying data structure. +The extension uses `Roaring64NavigableMap` as its underlying data structure to efficiently store and compute exact cardinality of 64-bit integers. It provides two types of aggregators that serve different purposes: + +### Build Aggregator (Bitmap64ExactCardinalityBuild) + +The BUILD aggregator is used when you want to compute cardinality directly from raw LONG values: +- Used during ingestion or when querying raw data +- Must be used on columns of type LONG, otherwise the output will be 1. + +Example: +```json +{ + "type": "Bitmap64ExactCardinalityBuild", + "name": "unique_values", + "fieldName": "id" +} +``` + +### Merge Aggregator (Bitmap64ExactCardinalityMerge) + +The MERGE aggregator is used when working with pre-computed bitmaps: +- Used for querying pre-aggregated data (columns that were previously aggregated using BUILD) +- Combines multiple bitmaps using bitwise operations +- Must be used on columns that are aggregated using BUILD +- `Bitmap64ExactCardinalityMerge` aggregator is recommended for use in `timeseries` type queries, though it also works for `topN` and `groupBy` queries. + +Example: +```json +{ + "type": "Bitmap64ExactCardinalityMerge", + "name": "total_unique_values", + "fieldName": "unique_values" // Must be a pre-computed bitmap +} +``` + +### Typical Workflow + +1. During ingestion, use BUILD to create the initial bitmap: + ```json + { + "type": "index", + "spec": { + "dataSchema": { + "metricsSpec": [ + { + "type": "Bitmap64ExactCardinalityBuild", + "name": "unique_users", + "fieldName": "user_id" + } + ] + } + } + } + ``` + +2. When querying the aggregated data, use MERGE to combine bitmaps: + ```json + { + "queryType": "timeseries", + "aggregations": [ + { + "type": "Bitmap64ExactCardinalityMerge", + "name": "total_unique_users", + "fieldName": "unique_users" + } + ] + } + ``` + +## Usage + +### SQL Query -Here's a brief overview of its operation: +You can use the `BITMAP64_EXACT_CARDINALITY` function in SQL queries: -1. **Data Ingestion (`Bitmap64ExactCardinalityBuild`)**: - * When new data is ingested, and this aggregator is applied to a `LONG` column (specified by `fieldName`), each unique long value from the input column is added to a `Roaring64NavigableMap`. - * This bitmap efficiently stores the set of unique 64-bit integers encountered. +```sql +SELECT BITMAP64_EXACT_CARDINALITY(column_name) +FROM datasource +WHERE ... +GROUP BY ... +``` -2. **Aggregation and Rollup (`Bitmap64ExactCardinalityMerge`)**: - * During rollup or when combining results from different segments/queries, the aggregator merges multiple `Roaring64NavigableMap` instances. - * This merge operation is a bitwise `OR` on the bitmaps, which correctly computes the union of the distinct values from each source. - * The intermediate object stored and transferred is the serialized form of the `Roaring64NavigableMap`. +### Post-Aggregator -3. **Query Time**: - * At query time, the `Bitmap64ExactCardinalityMerge` aggregator is typically used to combine the bitmaps from the selected segments. - * The final computation involves calling `getLongCardinality()` on the resulting `Roaring64NavigableMap`, which returns the exact count of distinct long values. - * A `Bitmap64ExactCardinalityPostAggregator` is also available for post-processing, allowing the cardinality result to be used in further calculations. +You can also use the post-aggregator for further processing: -**Key Characteristics**: +```json +{ + "type": "bitmap64ExactCardinality", + "name": "", + "fieldName": "" +} +``` -* **Exactness**: Provides precise distinct counts, unlike approximation algorithms (e.g., HyperLogLog). -* **Input Type**: Designed for `LONG` (64-bit integer) columns. If your data is in string format, it must be converted to longs first (e.g., via dictionary encoding or hashing) to use this aggregator. -* **Data Structure**: Leverages Roaring Bitmaps for memory efficiency and fast set operations (add, union, cardinality check) compared to traditional `HashSet` approaches, especially for certain data distributions. -* **Trade-offs**: - * **Memory**: While Roaring Bitmaps are efficient, storing exact unique values will generally consume more memory than sketch-based approximate algorithms, especially for very high cardinality dimensions with uniformly distributed values. - * **Performance**: The performance can be very good, but for extremely high cardinalities, the cost of merging large bitmaps might be higher than that of merging smaller sketches. +## Considerations -This aggregator is suitable for use cases where the precision of the distinct count is paramount and the input data is or can be represented as long values. +- **Memory Usage**: While Roaring Bitmaps are efficient, storing exact unique values will generally consume more memory than approximate algorithms like HyperLogLog. +- **Input Type**: This aggregator only works with LONG (64-bit integer) columns. String or other data types must be converted to longs before using this aggregator. +- **Build vs Merge**: Always use BUILD for raw data and MERGE for pre-aggregated data. Using BUILD on pre-aggregated data or MERGE on raw data will not work correctly. + +## Example Use Cases + +1. **User Analytics**: Count unique users over time +```sql +-- First ingest with BUILD aggregator +-- Then query with: +SELECT + TIME_FLOOR(__time, 'PT1H') AS hour, + BITMAP64_EXACT_CARDINALITY(unique_users) as distinct_users +FROM user_metrics +GROUP BY 1 +``` -## Usage Examples -Kafka ingestion task spec: +2. **High-Precision Metrics**: When exact counts are required +```json +{ + "type": "groupBy", + "dimensions": ["country"], + "aggregations": [ + { + "type": "Bitmap64ExactCardinalityMerge", + "name": "exact_user_count", + "fieldName": "unique_users" + } + ] +} ``` + +## More Examples + +### Kafka ingestion task spec +```json { "type": "kafka", "spec": { @@ -122,8 +230,36 @@ Kafka ingestion task spec: } } ``` -Query from rollup datasource: + +### Query from datasource with raw bytes +```json +{ + "queryType": "timeseries", + "dataSource": { + "type": "table", + "name": "wikipedia" + }, + "intervals": { + "type": "intervals", + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" + ] + }, + "granularity": { + "type": "all" + }, + "aggregations": [ + { + "type": "Bitmap64ExactCardinalityBuild", + "name": "a0", + "fieldName": "added" + } + ] +} ``` + +### Query from datasource with pre-aggregated bitmap +```json { "queryType": "timeseries", "dataSource": { @@ -153,8 +289,9 @@ Query from rollup datasource: "limit": 2147483647 } ``` -query with post aggregator: -``` + +### Query with Post-aggregator: +```json { "queryType": "timeseries", "dataSource": { @@ -206,64 +343,3 @@ query with post aggregator: "limit": 2147483647 } ``` -sql query: -``` -SELECT "key", BITMAP64_EXACT_CARDINALITY("cardinality") -FROM "ticker_event_bitmap64_exact_cardinality_rollup" -WHERE "__time" >= CURRENT_TIMESTAMP - INTERVAL '1' DAY -GROUP BY key -``` - -Note: this `longExactCardinality` aggregator is recommended for use in `timeseries` type queries, though it also works for `topN` -and `groupBy` queries. e.g.: -``` -{ - "queryType": "timeseries", - "dataSource": "ticker_event", - "intervals": { - "type": "intervals", - "intervals": [ - "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" - ] - }, - "descending": false, - "virtualColumns": [], - "filter": null, - "granularity": { - "type": "all" - }, - "aggregations": [ - { - "type": "longExactCardinality", - "name": "ExactCardinality", - "fieldName": "value", - "expression": null - } - ] -} -``` -``` -{ - "queryType": "groupBy", - "dataSource": "ticker_event", - "dimensions": ["key"], - "intervals": { - "type": "intervals", - "intervals": [ - "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" - ] - }, - "granularity": { - "type": "all" - }, - "aggregations": [ - { - "type": "longExactCardinality", - "name": "a0", - "fieldName": "value" - } - ] -} -``` - - From b7b5a049443caaec1f04e64813321cd8fdf3c7a8 Mon Sep 17 00:00:00 2001 From: GWphua Date: Wed, 21 May 2025 11:45:54 +0800 Subject: [PATCH 14/65] Fix Spelling check --- website/.spelling | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/website/.spelling b/website/.spelling index e4f99df4b210..f1a3df68002d 100644 --- a/website/.spelling +++ b/website/.spelling @@ -470,6 +470,7 @@ pre-aggregating pre-aggregation pre-computation pre-compute +pre-computed pre-computing preconfigured pre-existing @@ -2434,6 +2435,10 @@ quantileFromDDSketch collapsingLowestDense snapshotVersion +- ../docs/development/extensions-contrib/druid-exact-cardinality.md +Bitmap64ExactCardinalityBuild +Bitmap64ExactCardinalityMerge + - ../docs/development/extensions-core/catalog.md ColumnSpec PropertyKeyName From 83dfef9e477d83f1e30ddafce0ea1d0850cbeff5 Mon Sep 17 00:00:00 2001 From: GWphua Date: Wed, 21 May 2025 12:02:15 +0800 Subject: [PATCH 15/65] Add wikipedia datasource walkthrough --- .../druid-exact-cardinality.md | 203 ++++++++++++------ 1 file changed, 143 insertions(+), 60 deletions(-) diff --git a/docs/development/extensions-contrib/druid-exact-cardinality.md b/docs/development/extensions-contrib/druid-exact-cardinality.md index d5cc3efe000f..632e2a312a8f 100644 --- a/docs/development/extensions-contrib/druid-exact-cardinality.md +++ b/docs/development/extensions-contrib/druid-exact-cardinality.md @@ -157,7 +157,149 @@ GROUP BY 1 } ``` -## More Examples +## Walkthrough Using Wikipedia datasource + +### Batch Ingestion Task Spec +```json +{ + "type": "index", + "spec": { + "dataSchema": { + "dataSource": "wikipedia_metrics", + "timestampSpec": { + "column": "__time", + "format": "auto" + }, + "dimensionsSpec": { + "dimensions": [ + "channel", + "namespace", + "page", + "user", + "cityName", + "countryName", + "regionName", + "isRobot", + "isUnpatrolled", + "isNew", + "isAnonymous" + ] + }, + "metricsSpec": [ + { + "type": "Bitmap64ExactCardinalityBuild", + "name": "unique_added_values", + "fieldName": "added" + }, + { + "type": "Bitmap64ExactCardinalityBuild", + "name": "unique_delta_values", + "fieldName": "delta" + }, + { + "type": "Bitmap64ExactCardinalityBuild", + "name": "unique_comment_lengths", + "fieldName": "commentLength" + }, + { + "name": "count", + "type": "count" + }, + { + "name": "sum_added", + "type": "longSum", + "fieldName": "added" + }, + { + "name": "sum_delta", + "type": "longSum", + "fieldName": "delta" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "HOUR", + "rollup": true, + "intervals": ["2016-06-27/2016-06-28"] + } + }, + "ioConfig": { + "type": "index", + "inputSource": { + "type": "druid", + "dataSource": "wikipedia", + "interval": "2016-06-27/2016-06-28" + }, + "inputFormat": { + "type": "tsv", + "findColumnsFromHeader": true + } + }, + "tuningConfig": { + "type": "index", + "maxRowsPerSegment": 5000000, + "maxRowsInMemory": 25000 + } + } +} +``` + +### Query from datasource with raw bytes +```json +{ + "queryType": "timeseries", + "dataSource": { + "type": "table", + "name": "wikipedia_metrics" + }, + "intervals": { + "type": "intervals", + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" + ] + }, + "granularity": { + "type": "all" + }, + "aggregations": [ + { + "type": "Bitmap64ExactCardinalityBuild", + "name": "a0", + "fieldName": "unique_added_values" + } + ] +} +``` + +### Query from datasource with pre-aggregated bitmap +```json +{ + "queryType": "timeseries", + "dataSource": { + "type": "table", + "name": "wikipedia_metrics" + }, + "intervals": { + "type": "intervals", + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" + ] + }, + "granularity": { + "type": "all" + }, + "aggregations": [ + { + "type": "Bitmap64ExactCardinalityMerge", + "name": "a0", + "fieldName": "unique_added_values" + } + ] +} +``` + +## Other Examples ### Kafka ingestion task spec ```json @@ -231,65 +373,6 @@ GROUP BY 1 } ``` -### Query from datasource with raw bytes -```json -{ - "queryType": "timeseries", - "dataSource": { - "type": "table", - "name": "wikipedia" - }, - "intervals": { - "type": "intervals", - "intervals": [ - "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" - ] - }, - "granularity": { - "type": "all" - }, - "aggregations": [ - { - "type": "Bitmap64ExactCardinalityBuild", - "name": "a0", - "fieldName": "added" - } - ] -} -``` - -### Query from datasource with pre-aggregated bitmap -```json -{ - "queryType": "timeseries", - "dataSource": { - "type": "table", - "name": "ticker_event_bitmap64_exact_cardinality_rollup" - }, - "intervals": { - "type": "intervals", - "intervals": [ - "2020-09-13T06:35:35.000Z/146140482-04-24T15:36:27.903Z" - ] - }, - "descending": false, - "virtualColumns": [], - "filter": null, - "granularity": { - "type": "all" - }, - "aggregations": [ - { - "type": "Bitmap64ExactCardinalityMerge", - "name": "a0", - "fieldName": "cardinality" - } - ], - "postAggregations": [], - "limit": 2147483647 -} -``` - ### Query with Post-aggregator: ```json { From 9310334b3d48a198352942738c52c961ce60d6fd Mon Sep 17 00:00:00 2001 From: GWphua Date: Thu, 22 May 2025 16:00:27 +0800 Subject: [PATCH 16/65] Add SQL Test Revert "Add wikipedia datasource walkthrough" This reverts commit 83dfef9e477d83f1e30ddafce0ea1d0850cbeff5. Revert "Add SQL Test" This reverts commit e81a0fdc2f07b71958bc32734abe16bacbac920d. --- .../druid-exact-cardinality.md | 63 +++++--- .../druid-exact-cardinality/pom.xml | 19 +++ ...ap64ExactCardinalitySqlAggregatorTest.java | 150 ++++++++++++++++++ 3 files changed, 208 insertions(+), 24 deletions(-) create mode 100644 extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java diff --git a/docs/development/extensions-contrib/druid-exact-cardinality.md b/docs/development/extensions-contrib/druid-exact-cardinality.md index 632e2a312a8f..01171b8ee693 100644 --- a/docs/development/extensions-contrib/druid-exact-cardinality.md +++ b/docs/development/extensions-contrib/druid-exact-cardinality.md @@ -35,10 +35,12 @@ The extension uses `Roaring64NavigableMap` as its underlying data structure to e ### Build Aggregator (Bitmap64ExactCardinalityBuild) The BUILD aggregator is used when you want to compute cardinality directly from raw LONG values: + - Used during ingestion or when querying raw data - Must be used on columns of type LONG, otherwise the output will be 1. Example: + ```json { "type": "Bitmap64ExactCardinalityBuild", @@ -50,17 +52,19 @@ Example: ### Merge Aggregator (Bitmap64ExactCardinalityMerge) The MERGE aggregator is used when working with pre-computed bitmaps: + - Used for querying pre-aggregated data (columns that were previously aggregated using BUILD) - Combines multiple bitmaps using bitwise operations - Must be used on columns that are aggregated using BUILD - `Bitmap64ExactCardinalityMerge` aggregator is recommended for use in `timeseries` type queries, though it also works for `topN` and `groupBy` queries. Example: + ```json { "type": "Bitmap64ExactCardinalityMerge", "name": "total_unique_values", - "fieldName": "unique_values" // Must be a pre-computed bitmap + "fieldName": "unique_values" // Must be a pre-computed bitmap } ``` @@ -132,6 +136,7 @@ You can also use the post-aggregator for further processing: ## Example Use Cases 1. **User Analytics**: Count unique users over time + ```sql -- First ingest with BUILD aggregator -- Then query with: @@ -143,10 +148,13 @@ GROUP BY 1 ``` 2. **High-Precision Metrics**: When exact counts are required + ```json { "type": "groupBy", - "dimensions": ["country"], + "dimensions": [ + "country" + ], "aggregations": [ { "type": "Bitmap64ExactCardinalityMerge", @@ -160,6 +168,7 @@ GROUP BY 1 ## Walkthrough Using Wikipedia datasource ### Batch Ingestion Task Spec + ```json { "type": "index", @@ -221,7 +230,9 @@ GROUP BY 1 "segmentGranularity": "DAY", "queryGranularity": "HOUR", "rollup": true, - "intervals": ["2016-06-27/2016-06-28"] + "intervals": [ + "2016-06-27/2016-06-28" + ] } }, "ioConfig": { @@ -246,6 +257,7 @@ GROUP BY 1 ``` ### Query from datasource with raw bytes + ```json { "queryType": "timeseries", @@ -273,35 +285,37 @@ GROUP BY 1 ``` ### Query from datasource with pre-aggregated bitmap + ```json { - "queryType": "timeseries", - "dataSource": { - "type": "table", - "name": "wikipedia_metrics" - }, - "intervals": { - "type": "intervals", - "intervals": [ - "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" - ] - }, - "granularity": { - "type": "all" - }, - "aggregations": [ - { - "type": "Bitmap64ExactCardinalityMerge", - "name": "a0", - "fieldName": "unique_added_values" - } - ] + "queryType": "timeseries", + "dataSource": { + "type": "table", + "name": "wikipedia_metrics" + }, + "intervals": { + "type": "intervals", + "intervals": [ + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" + ] + }, + "granularity": { + "type": "all" + }, + "aggregations": [ + { + "type": "Bitmap64ExactCardinalityMerge", + "name": "a0", + "fieldName": "unique_added_values" + } + ] } ``` ## Other Examples ### Kafka ingestion task spec + ```json { "type": "kafka", @@ -374,6 +388,7 @@ GROUP BY 1 ``` ### Query with Post-aggregator: + ```json { "queryType": "timeseries", diff --git a/extensions-contrib/druid-exact-cardinality/pom.xml b/extensions-contrib/druid-exact-cardinality/pom.xml index 2c96578562d6..fd32845434f0 100644 --- a/extensions-contrib/druid-exact-cardinality/pom.xml +++ b/extensions-contrib/druid-exact-cardinality/pom.xml @@ -114,6 +114,25 @@ test-jar test + + org.apache.druid + druid-server + ${project.parent.version} + test-jar + test + + + org.apache.druid + druid-sql + ${project.parent.version} + test-jar + test + + + org.hamcrest + hamcrest-core + test + diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java new file mode 100644 index 000000000000..e8852bf283bf --- /dev/null +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.exact.cardinality.bitmap64.sql; + +import com.google.common.collect.ImmutableList; +import org.apache.druid.initialization.DruidModule; +import org.apache.druid.query.aggregation.exact.cardinality.bitmap64.Bitmap64ExactCardinalityBuildAggregatorFactory; +import org.apache.druid.query.aggregation.exact.cardinality.bitmap64.Bitmap64ExactCardinalityModule; +import org.apache.druid.query.aggregation.exact.cardinality.bitmap64.sql.Bitmap64ExactCardinalitySqlAggregatorTest.Bitmap64ExactCardinalitySqlAggComponentSupplier; +import org.apache.druid.segment.IndexBuilder; +import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.incremental.IncrementalIndexSchema; +import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.sql.calcite.BaseCalciteQueryTest; +import org.apache.druid.sql.calcite.SqlTestFrameworkConfig; +import org.apache.druid.sql.calcite.TempDirProducer; +import org.apache.druid.sql.calcite.util.DruidModuleCollection; +import org.apache.druid.sql.calcite.util.SqlTestFramework.StandardComponentSupplier; +import org.apache.druid.sql.calcite.util.TestDataBuilder; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.LinearShardSpec; +import org.junit.jupiter.api.Test; + +@SqlTestFrameworkConfig.ComponentSupplier(Bitmap64ExactCardinalitySqlAggComponentSupplier.class) +public class Bitmap64ExactCardinalitySqlAggregatorTest extends BaseCalciteQueryTest +{ + private static final String DATA_SOURCE = "numfoo"; + + static class Bitmap64ExactCardinalitySqlAggComponentSupplier extends StandardComponentSupplier + { + public Bitmap64ExactCardinalitySqlAggComponentSupplier(TempDirProducer tempFolderProducer) + { + super(tempFolderProducer); + } + + @Override + public DruidModule getCoreModule() + { + return DruidModuleCollection.of(super.getCoreModule(), new Bitmap64ExactCardinalityModule()); + } + + @Override + public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQuerySegmentWalker walker) + { + final QueryableIndex index = + IndexBuilder.create() + .tmpDir(tempDirProducer.newTempFolder()) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema(new IncrementalIndexSchema.Builder() + .withMetrics( + new Bitmap64ExactCardinalityBuildAggregatorFactory( + "unique_m1_values", + "m1" + ) + ) + .withRollup(false) + .build()) + .rows(TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS) + .buildMMappedIndex(); + + return walker.add( + DataSegment.builder() + .dataSource(DATA_SOURCE) + .interval(index.getDataInterval()) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(), + index + ); + } + } + + @Test + public void testExactCardinalityOnLongColumn() + { + testBuilder() + .sql("SELECT BITMAP64_EXACT_CARDINALITY(m1) FROM " + DATA_SOURCE) + .expectedResults(ImmutableList.of( + new Object[]{4L} + )) + .run(); + } + + @Test + public void testExactCardinalityOnPreAggregatedColumn() + { + testBuilder() + .sql("SELECT BITMAP64_EXACT_CARDINALITY(unique_m1_values) FROM " + DATA_SOURCE) + .expectedResults(ImmutableList.of( + new Object[]{4L} // Expecting 4, as 10, 20, 30, 40 are the unique values + )) + .run(); + } + + @Test + public void testExactCardinalityWithGroupBy() + { + testBuilder() + .sql("SELECT __time, BITMAP64_EXACT_CARDINALITY(m1) FROM " + DATA_SOURCE + " GROUP BY __time ORDER BY __time") + .expectedResults(ImmutableList.of( + new Object[]{1466985600000L, 3L}, + new Object[]{1467072000000L, 2L} + )) + .run(); + } + + @Test + public void testExactCardinalityOnPreAggregatedWithGroupBy() + { + testBuilder() + .sql("SELECT __time, BITMAP64_EXACT_CARDINALITY(unique_m1_values) FROM " + + DATA_SOURCE + + " GROUP BY __time ORDER BY __time") + .expectedResults(ImmutableList.of( + new Object[]{1466985600000L, 3L}, + new Object[]{1467072000000L, 2L} + )) + .run(); + } + + @Test + public void testExactCardinalityWithFilter() + { + testBuilder() + .sql("SELECT BITMAP64_EXACT_CARDINALITY(m1) FROM " + DATA_SOURCE + " WHERE m1 > 20") + .expectedResults(ImmutableList.of( + new Object[]{2L} // 30, 40 + )) + .run(); + } +} From 39b7ce519a6f0776a47ed8c1ee8d1e10c22a2506 Mon Sep 17 00:00:00 2001 From: GWphua Date: Thu, 22 May 2025 18:33:42 +0800 Subject: [PATCH 17/65] Fix Cannot find class error --- extensions-contrib/druid-exact-cardinality/pom.xml | 5 +++++ .../sql/Bitmap64ExactCardinalitySqlAggregatorTest.java | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/extensions-contrib/druid-exact-cardinality/pom.xml b/extensions-contrib/druid-exact-cardinality/pom.xml index fd32845434f0..2d505f33a34b 100644 --- a/extensions-contrib/druid-exact-cardinality/pom.xml +++ b/extensions-contrib/druid-exact-cardinality/pom.xml @@ -133,6 +133,11 @@ hamcrest-core test + + org.reflections + reflections + test + diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java index e8852bf283bf..baefc6c5109c 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java @@ -44,7 +44,7 @@ public class Bitmap64ExactCardinalitySqlAggregatorTest extends BaseCalciteQueryT { private static final String DATA_SOURCE = "numfoo"; - static class Bitmap64ExactCardinalitySqlAggComponentSupplier extends StandardComponentSupplier + public static class Bitmap64ExactCardinalitySqlAggComponentSupplier extends StandardComponentSupplier { public Bitmap64ExactCardinalitySqlAggComponentSupplier(TempDirProducer tempFolderProducer) { From f552150b47667301268024633156421c91dd67ff Mon Sep 17 00:00:00 2001 From: GWphua Date: Thu, 22 May 2025 18:49:19 +0800 Subject: [PATCH 18/65] Fix Assert cannot be found --- extensions-contrib/druid-exact-cardinality/pom.xml | 5 +++++ .../Bitmap64ExactCardinalitySqlAggregatorTest.java | 14 +++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/extensions-contrib/druid-exact-cardinality/pom.xml b/extensions-contrib/druid-exact-cardinality/pom.xml index 2d505f33a34b..ab2e0040e7f7 100644 --- a/extensions-contrib/druid-exact-cardinality/pom.xml +++ b/extensions-contrib/druid-exact-cardinality/pom.xml @@ -97,6 +97,11 @@ + + junit + junit + test + org.junit.jupiter junit-jupiter-api diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java index baefc6c5109c..e2ea15171592 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java @@ -23,7 +23,6 @@ import org.apache.druid.initialization.DruidModule; import org.apache.druid.query.aggregation.exact.cardinality.bitmap64.Bitmap64ExactCardinalityBuildAggregatorFactory; import org.apache.druid.query.aggregation.exact.cardinality.bitmap64.Bitmap64ExactCardinalityModule; -import org.apache.druid.query.aggregation.exact.cardinality.bitmap64.sql.Bitmap64ExactCardinalitySqlAggregatorTest.Bitmap64ExactCardinalitySqlAggComponentSupplier; import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.incremental.IncrementalIndexSchema; @@ -32,14 +31,16 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.SqlTestFrameworkConfig; import org.apache.druid.sql.calcite.TempDirProducer; +import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.DruidModuleCollection; import org.apache.druid.sql.calcite.util.SqlTestFramework.StandardComponentSupplier; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.junit.jupiter.api.Test; +import org.junit.Assert.*; -@SqlTestFrameworkConfig.ComponentSupplier(Bitmap64ExactCardinalitySqlAggComponentSupplier.class) +@SqlTestFrameworkConfig.ComponentSupplier(Bitmap64ExactCardinalitySqlAggregatorTest.Bitmap64ExactCardinalitySqlAggComponentSupplier.class) public class Bitmap64ExactCardinalitySqlAggregatorTest extends BaseCalciteQueryTest { private static final String DATA_SOURCE = "numfoo"; @@ -60,8 +61,10 @@ public DruidModule getCoreModule() @Override public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQuerySegmentWalker walker) { + Bitmap64ExactCardinalityModule.registerSerde(); + final QueryableIndex index = - IndexBuilder.create() + IndexBuilder.create(CalciteTests.getJsonMapper()) .tmpDir(tempDirProducer.newTempFolder()) .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) .schema(new IncrementalIndexSchema.Builder() @@ -92,6 +95,7 @@ public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQu @Test public void testExactCardinalityOnLongColumn() { + cannotVectorize(); testBuilder() .sql("SELECT BITMAP64_EXACT_CARDINALITY(m1) FROM " + DATA_SOURCE) .expectedResults(ImmutableList.of( @@ -103,6 +107,7 @@ public void testExactCardinalityOnLongColumn() @Test public void testExactCardinalityOnPreAggregatedColumn() { + cannotVectorize(); testBuilder() .sql("SELECT BITMAP64_EXACT_CARDINALITY(unique_m1_values) FROM " + DATA_SOURCE) .expectedResults(ImmutableList.of( @@ -114,6 +119,7 @@ public void testExactCardinalityOnPreAggregatedColumn() @Test public void testExactCardinalityWithGroupBy() { + cannotVectorize(); testBuilder() .sql("SELECT __time, BITMAP64_EXACT_CARDINALITY(m1) FROM " + DATA_SOURCE + " GROUP BY __time ORDER BY __time") .expectedResults(ImmutableList.of( @@ -126,6 +132,7 @@ public void testExactCardinalityWithGroupBy() @Test public void testExactCardinalityOnPreAggregatedWithGroupBy() { + cannotVectorize(); testBuilder() .sql("SELECT __time, BITMAP64_EXACT_CARDINALITY(unique_m1_values) FROM " + DATA_SOURCE @@ -140,6 +147,7 @@ public void testExactCardinalityOnPreAggregatedWithGroupBy() @Test public void testExactCardinalityWithFilter() { + cannotVectorize(); testBuilder() .sql("SELECT BITMAP64_EXACT_CARDINALITY(m1) FROM " + DATA_SOURCE + " WHERE m1 > 20") .expectedResults(ImmutableList.of( From ddfc4e56b21070cce0df013140c6f4bad7d69520 Mon Sep 17 00:00:00 2001 From: GWphua Date: Thu, 22 May 2025 19:10:59 +0800 Subject: [PATCH 19/65] Fix test case --- ...tmap64ExactCardinalityBuildAggregator.java | 4 +- ...ExactCardinalityBuildBufferAggregator.java | 4 +- ...ap64ExactCardinalitySqlAggregatorTest.java | 54 +++++++++++-------- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregator.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregator.java index 941628108f1e..a26d7c4b7c20 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregator.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregator.java @@ -38,7 +38,9 @@ public Bitmap64ExactCardinalityBuildAggregator(BaseLongColumnValueSelector selec @Override public void aggregate() { - bitmap.add(selector.getLong()); + if (!selector.isNull()) { + bitmap.add(selector.getLong()); + } } @Nullable diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregator.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregator.java index c40635d0c376..43604a8b97fb 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregator.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregator.java @@ -50,7 +50,9 @@ public void aggregate(ByteBuffer buf, int position) try { buf.position(position); Bitmap64Counter bitmap64Counter = getOrCreateCollector(buf, position); - bitmap64Counter.add(selector.getLong()); + if (!selector.isNull()) { + bitmap64Counter.add(selector.getLong()); + } } finally { buf.position(oldPosition); diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java index e2ea15171592..89890a1157ed 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java @@ -19,6 +19,7 @@ package org.apache.druid.query.aggregation.exact.cardinality.bitmap64.sql; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.initialization.DruidModule; import org.apache.druid.query.aggregation.exact.cardinality.bitmap64.Bitmap64ExactCardinalityBuildAggregatorFactory; @@ -35,10 +36,10 @@ import org.apache.druid.sql.calcite.util.DruidModuleCollection; import org.apache.druid.sql.calcite.util.SqlTestFramework.StandardComponentSupplier; import org.apache.druid.sql.calcite.util.TestDataBuilder; +import org.apache.druid.sql.calcite.util.datasets.TestDataSet; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.junit.jupiter.api.Test; -import org.junit.Assert.*; @SqlTestFrameworkConfig.ComponentSupplier(Bitmap64ExactCardinalitySqlAggregatorTest.Bitmap64ExactCardinalitySqlAggComponentSupplier.class) public class Bitmap64ExactCardinalitySqlAggregatorTest extends BaseCalciteQueryTest @@ -63,19 +64,22 @@ public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQu { Bitmap64ExactCardinalityModule.registerSerde(); + ObjectMapper jsonMapper = CalciteTests.getJsonMapper(); + new Bitmap64ExactCardinalityModule().getJacksonModules().forEach(jsonMapper::registerModule); + final QueryableIndex index = - IndexBuilder.create(CalciteTests.getJsonMapper()) + IndexBuilder.create(jsonMapper) .tmpDir(tempDirProducer.newTempFolder()) .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema(new IncrementalIndexSchema.Builder() - .withMetrics( - new Bitmap64ExactCardinalityBuildAggregatorFactory( - "unique_m1_values", - "m1" - ) - ) - .withRollup(false) - .build()) + .schema( + IncrementalIndexSchema.builder() + .withDimensionsSpec(TestDataSet.NUMFOO.getInputRowSchema().getDimensionsSpec()) + .withMetrics( + new Bitmap64ExactCardinalityBuildAggregatorFactory("unique_m1_values", "m1") + ) + .withRollup(false) + .build() + ) .rows(TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS) .buildMMappedIndex(); @@ -97,9 +101,9 @@ public void testExactCardinalityOnLongColumn() { cannotVectorize(); testBuilder() - .sql("SELECT BITMAP64_EXACT_CARDINALITY(m1) FROM " + DATA_SOURCE) + .sql("SELECT BITMAP64_EXACT_CARDINALITY(l1) FROM " + DATA_SOURCE) .expectedResults(ImmutableList.of( - new Object[]{4L} + new Object[]{3L} // l1 values: 7, 325323, 0 (distinct count = 3) )) .run(); } @@ -111,7 +115,7 @@ public void testExactCardinalityOnPreAggregatedColumn() testBuilder() .sql("SELECT BITMAP64_EXACT_CARDINALITY(unique_m1_values) FROM " + DATA_SOURCE) .expectedResults(ImmutableList.of( - new Object[]{4L} // Expecting 4, as 10, 20, 30, 40 are the unique values + new Object[]{6L} // m1 string inputs: "1.0"-"6.0" are 6 unique values )) .run(); } @@ -121,10 +125,14 @@ public void testExactCardinalityWithGroupBy() { cannotVectorize(); testBuilder() - .sql("SELECT __time, BITMAP64_EXACT_CARDINALITY(m1) FROM " + DATA_SOURCE + " GROUP BY __time ORDER BY __time") + .sql("SELECT __time, BITMAP64_EXACT_CARDINALITY(l1) FROM " + DATA_SOURCE + " GROUP BY __time ORDER BY __time") .expectedResults(ImmutableList.of( - new Object[]{1466985600000L, 3L}, - new Object[]{1467072000000L, 2L} + new Object[]{946684800000L, 1L}, // 2000-01-01, l1=7L + new Object[]{946771200000L, 1L}, // 2000-01-02, l1=325323L + new Object[]{946857600000L, 1L}, // 2000-01-03, l1=0L + new Object[]{978307200000L, 0L}, // 2001-01-01, l1 is null + new Object[]{978393600000L, 0L}, // 2001-01-02, l1 is null + new Object[]{978480000000L, 0L} // 2001-01-03, l1 is null )) .run(); } @@ -138,8 +146,12 @@ public void testExactCardinalityOnPreAggregatedWithGroupBy() + DATA_SOURCE + " GROUP BY __time ORDER BY __time") .expectedResults(ImmutableList.of( - new Object[]{1466985600000L, 3L}, - new Object[]{1467072000000L, 2L} + new Object[]{946684800000L, 1L}, // 2000-01-01, m1="1.0" + new Object[]{946771200000L, 1L}, // 2000-01-02, m1="2.0" + new Object[]{946857600000L, 1L}, // 2000-01-03, m1="3.0" + new Object[]{978307200000L, 1L}, // 2001-01-01, m1="4.0" + new Object[]{978393600000L, 1L}, // 2001-01-02, m1="5.0" + new Object[]{978480000000L, 1L} // 2001-01-03, m1="6.0" )) .run(); } @@ -149,9 +161,9 @@ public void testExactCardinalityWithFilter() { cannotVectorize(); testBuilder() - .sql("SELECT BITMAP64_EXACT_CARDINALITY(m1) FROM " + DATA_SOURCE + " WHERE m1 > 20") + .sql("SELECT BITMAP64_EXACT_CARDINALITY(l1) FROM " + DATA_SOURCE + " WHERE l1 > 20") .expectedResults(ImmutableList.of( - new Object[]{2L} // 30, 40 + new Object[]{1L} // l1 values > 20: 325323L (distinct count = 1) )) .run(); } From 074b2baa9be3312fd67c2cf82dd6b4a08da15d03 Mon Sep 17 00:00:00 2001 From: GWphua Date: Fri, 23 May 2025 10:52:44 +0800 Subject: [PATCH 20/65] Fix SqlAggregator test cases --- ...ap64ExactCardinalitySqlAggregatorTest.java | 133 +++++++++++++----- 1 file changed, 98 insertions(+), 35 deletions(-) diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java index 89890a1157ed..283edc48e5f9 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java @@ -19,19 +19,27 @@ package org.apache.druid.query.aggregation.exact.cardinality.bitmap64.sql; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableList; +import java.util.Collections; + import org.apache.druid.initialization.DruidModule; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.exact.cardinality.bitmap64.Bitmap64ExactCardinalityBuildAggregatorFactory; +import org.apache.druid.query.aggregation.exact.cardinality.bitmap64.Bitmap64ExactCardinalityMergeAggregatorFactory; import org.apache.druid.query.aggregation.exact.cardinality.bitmap64.Bitmap64ExactCardinalityModule; +import org.apache.druid.query.dimension.DefaultDimensionSpec; +import org.apache.druid.query.groupby.GroupByQuery; +import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.SqlTestFrameworkConfig; import org.apache.druid.sql.calcite.TempDirProducer; +import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.DruidModuleCollection; import org.apache.druid.sql.calcite.util.SqlTestFramework.StandardComponentSupplier; @@ -41,6 +49,9 @@ import org.apache.druid.timeline.partition.LinearShardSpec; import org.junit.jupiter.api.Test; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; + @SqlTestFrameworkConfig.ComponentSupplier(Bitmap64ExactCardinalitySqlAggregatorTest.Bitmap64ExactCardinalitySqlAggComponentSupplier.class) public class Bitmap64ExactCardinalitySqlAggregatorTest extends BaseCalciteQueryTest { @@ -100,71 +111,123 @@ public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQu public void testExactCardinalityOnLongColumn() { cannotVectorize(); - testBuilder() - .sql("SELECT BITMAP64_EXACT_CARDINALITY(l1) FROM " + DATA_SOURCE) - .expectedResults(ImmutableList.of( + testQuery( + "SELECT BITMAP64_EXACT_CARDINALITY(l1) FROM " + DATA_SOURCE, + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(DATA_SOURCE) + .intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))) + .granularity(Granularities.ALL) + .aggregators( + ImmutableList.of( + new Bitmap64ExactCardinalityBuildAggregatorFactory( + "a0", + "l1" + ) + ) + ) + .context(Collections.emptyMap()) + .build() + ), + ImmutableList.of( new Object[]{3L} // l1 values: 7, 325323, 0 (distinct count = 3) - )) - .run(); + ) + ); } + @Test public void testExactCardinalityOnPreAggregatedColumn() { cannotVectorize(); - testBuilder() - .sql("SELECT BITMAP64_EXACT_CARDINALITY(unique_m1_values) FROM " + DATA_SOURCE) - .expectedResults(ImmutableList.of( + String sql = "SELECT BITMAP64_EXACT_CARDINALITY(unique_m1_values) FROM " + DATA_SOURCE; + testQuery( + sql, + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(DATA_SOURCE) + .intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))) + .granularity(Granularities.ALL) + .aggregators( + ImmutableList.of( + new Bitmap64ExactCardinalityMergeAggregatorFactory( + "a0", + "unique_m1_values" + ) + ) + ) + .context(Collections.emptyMap()) + .build() + ), + ImmutableList.of( new Object[]{6L} // m1 string inputs: "1.0"-"6.0" are 6 unique values - )) - .run(); + ) + ); } @Test public void testExactCardinalityWithGroupBy() { cannotVectorize(); - testBuilder() - .sql("SELECT __time, BITMAP64_EXACT_CARDINALITY(l1) FROM " + DATA_SOURCE + " GROUP BY __time ORDER BY __time") - .expectedResults(ImmutableList.of( + String sql = "SELECT __time, BITMAP64_EXACT_CARDINALITY(l1) FROM " + DATA_SOURCE + " GROUP BY __time ORDER BY __time"; + testQuery( + sql, + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(DATA_SOURCE) + .setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))) + .setGranularity(Granularities.ALL) + .setDimensions(ImmutableList.of(new DefaultDimensionSpec("__time", "d0", ColumnType.LONG))) + .setAggregatorSpecs( + ImmutableList.of( + new Bitmap64ExactCardinalityBuildAggregatorFactory("a0", "l1") + ) + ) + .setContext(Collections.emptyMap()) + .build() + ), + ImmutableList.of( new Object[]{946684800000L, 1L}, // 2000-01-01, l1=7L new Object[]{946771200000L, 1L}, // 2000-01-02, l1=325323L new Object[]{946857600000L, 1L}, // 2000-01-03, l1=0L new Object[]{978307200000L, 0L}, // 2001-01-01, l1 is null new Object[]{978393600000L, 0L}, // 2001-01-02, l1 is null new Object[]{978480000000L, 0L} // 2001-01-03, l1 is null - )) - .run(); + ) + ); } @Test public void testExactCardinalityOnPreAggregatedWithGroupBy() { cannotVectorize(); - testBuilder() - .sql("SELECT __time, BITMAP64_EXACT_CARDINALITY(unique_m1_values) FROM " + String sql = "SELECT __time, BITMAP64_EXACT_CARDINALITY(unique_m1_values) FROM " + DATA_SOURCE - + " GROUP BY __time ORDER BY __time") - .expectedResults(ImmutableList.of( + + " GROUP BY __time ORDER BY __time"; + testQuery( + sql, + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(DATA_SOURCE) + .setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))) + .setGranularity(Granularities.ALL) + .setDimensions(ImmutableList.of(new DefaultDimensionSpec("__time", "d0", ColumnType.LONG))) + .setAggregatorSpecs( + ImmutableList.of( + new Bitmap64ExactCardinalityMergeAggregatorFactory("a0", "unique_m1_values") + ) + ) + .setContext(Collections.emptyMap()) + .build() + ), + ImmutableList.of( new Object[]{946684800000L, 1L}, // 2000-01-01, m1="1.0" new Object[]{946771200000L, 1L}, // 2000-01-02, m1="2.0" new Object[]{946857600000L, 1L}, // 2000-01-03, m1="3.0" new Object[]{978307200000L, 1L}, // 2001-01-01, m1="4.0" new Object[]{978393600000L, 1L}, // 2001-01-02, m1="5.0" new Object[]{978480000000L, 1L} // 2001-01-03, m1="6.0" - )) - .run(); - } - - @Test - public void testExactCardinalityWithFilter() - { - cannotVectorize(); - testBuilder() - .sql("SELECT BITMAP64_EXACT_CARDINALITY(l1) FROM " + DATA_SOURCE + " WHERE l1 > 20") - .expectedResults(ImmutableList.of( - new Object[]{1L} // l1 values > 20: 325323L (distinct count = 1) - )) - .run(); + ) + ); } } From e2a1afe67c8c85a2092885d20e74783d3c3f7390 Mon Sep 17 00:00:00 2001 From: GWphua Date: Fri, 23 May 2025 11:09:00 +0800 Subject: [PATCH 21/65] Fix checkstyle --- .../sql/Bitmap64ExactCardinalitySqlAggregatorTest.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java index 283edc48e5f9..d63eb8a2cdae 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java @@ -19,8 +19,8 @@ package org.apache.druid.query.aggregation.exact.cardinality.bitmap64.sql; -import java.util.Collections; - +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; import org.apache.druid.initialization.DruidModule; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.query.Druids; @@ -49,8 +49,7 @@ import org.apache.druid.timeline.partition.LinearShardSpec; import org.junit.jupiter.api.Test; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableList; +import java.util.Collections; @SqlTestFrameworkConfig.ComponentSupplier(Bitmap64ExactCardinalitySqlAggregatorTest.Bitmap64ExactCardinalitySqlAggComponentSupplier.class) public class Bitmap64ExactCardinalitySqlAggregatorTest extends BaseCalciteQueryTest From 683a06a9bf2246b9f5bac8119ab660c33553ac1f Mon Sep 17 00:00:00 2001 From: GWphua Date: Fri, 23 May 2025 11:15:04 +0800 Subject: [PATCH 22/65] Fix registerSerde problem --- ...ap64ExactCardinalitySqlAggregatorTest.java | 129 +++++++++--------- 1 file changed, 65 insertions(+), 64 deletions(-) diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java index d63eb8a2cdae..d3eabaeb79f7 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java @@ -19,7 +19,6 @@ package org.apache.druid.query.aggregation.exact.cardinality.bitmap64.sql; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.initialization.DruidModule; import org.apache.druid.java.util.common.granularity.Granularities; @@ -73,22 +72,22 @@ public DruidModule getCoreModule() public SpecificSegmentsQuerySegmentWalker addSegmentsToWalker(SpecificSegmentsQuerySegmentWalker walker) { Bitmap64ExactCardinalityModule.registerSerde(); - - ObjectMapper jsonMapper = CalciteTests.getJsonMapper(); - new Bitmap64ExactCardinalityModule().getJacksonModules().forEach(jsonMapper::registerModule); - final QueryableIndex index = - IndexBuilder.create(jsonMapper) + IndexBuilder.create(CalciteTests.getJsonMapper()) .tmpDir(tempDirProducer.newTempFolder()) .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) .schema( IncrementalIndexSchema.builder() - .withDimensionsSpec(TestDataSet.NUMFOO.getInputRowSchema().getDimensionsSpec()) - .withMetrics( - new Bitmap64ExactCardinalityBuildAggregatorFactory("unique_m1_values", "m1") - ) - .withRollup(false) - .build() + .withDimensionsSpec(TestDataSet.NUMFOO.getInputRowSchema() + .getDimensionsSpec()) + .withMetrics( + new Bitmap64ExactCardinalityBuildAggregatorFactory( + "unique_m1_values", + "m1" + ) + ) + .withRollup(false) + .build() ) .rows(TestDataBuilder.ROWS1_WITH_NUMERIC_DIMS) .buildMMappedIndex(); @@ -114,19 +113,19 @@ public void testExactCardinalityOnLongColumn() "SELECT BITMAP64_EXACT_CARDINALITY(l1) FROM " + DATA_SOURCE, ImmutableList.of( Druids.newTimeseriesQueryBuilder() - .dataSource(DATA_SOURCE) - .intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))) - .granularity(Granularities.ALL) - .aggregators( - ImmutableList.of( - new Bitmap64ExactCardinalityBuildAggregatorFactory( - "a0", - "l1" - ) - ) - ) - .context(Collections.emptyMap()) - .build() + .dataSource(DATA_SOURCE) + .intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))) + .granularity(Granularities.ALL) + .aggregators( + ImmutableList.of( + new Bitmap64ExactCardinalityBuildAggregatorFactory( + "a0", + "l1" + ) + ) + ) + .context(Collections.emptyMap()) + .build() ), ImmutableList.of( new Object[]{3L} // l1 values: 7, 325323, 0 (distinct count = 3) @@ -144,19 +143,19 @@ public void testExactCardinalityOnPreAggregatedColumn() sql, ImmutableList.of( Druids.newTimeseriesQueryBuilder() - .dataSource(DATA_SOURCE) - .intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))) - .granularity(Granularities.ALL) - .aggregators( - ImmutableList.of( - new Bitmap64ExactCardinalityMergeAggregatorFactory( - "a0", - "unique_m1_values" - ) - ) - ) - .context(Collections.emptyMap()) - .build() + .dataSource(DATA_SOURCE) + .intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))) + .granularity(Granularities.ALL) + .aggregators( + ImmutableList.of( + new Bitmap64ExactCardinalityMergeAggregatorFactory( + "a0", + "unique_m1_values" + ) + ) + ) + .context(Collections.emptyMap()) + .build() ), ImmutableList.of( new Object[]{6L} // m1 string inputs: "1.0"-"6.0" are 6 unique values @@ -168,22 +167,24 @@ public void testExactCardinalityOnPreAggregatedColumn() public void testExactCardinalityWithGroupBy() { cannotVectorize(); - String sql = "SELECT __time, BITMAP64_EXACT_CARDINALITY(l1) FROM " + DATA_SOURCE + " GROUP BY __time ORDER BY __time"; + String sql = "SELECT __time, BITMAP64_EXACT_CARDINALITY(l1) FROM " + DATA_SOURCE + " " + + "GROUP BY __time " + + "ORDER BY __time"; testQuery( sql, ImmutableList.of( GroupByQuery.builder() - .setDataSource(DATA_SOURCE) - .setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))) - .setGranularity(Granularities.ALL) - .setDimensions(ImmutableList.of(new DefaultDimensionSpec("__time", "d0", ColumnType.LONG))) - .setAggregatorSpecs( - ImmutableList.of( - new Bitmap64ExactCardinalityBuildAggregatorFactory("a0", "l1") - ) - ) - .setContext(Collections.emptyMap()) - .build() + .setDataSource(DATA_SOURCE) + .setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))) + .setGranularity(Granularities.ALL) + .setDimensions(ImmutableList.of(new DefaultDimensionSpec("__time", "d0", ColumnType.LONG))) + .setAggregatorSpecs( + ImmutableList.of( + new Bitmap64ExactCardinalityBuildAggregatorFactory("a0", "l1") + ) + ) + .setContext(Collections.emptyMap()) + .build() ), ImmutableList.of( new Object[]{946684800000L, 1L}, // 2000-01-01, l1=7L @@ -200,24 +201,24 @@ public void testExactCardinalityWithGroupBy() public void testExactCardinalityOnPreAggregatedWithGroupBy() { cannotVectorize(); - String sql = "SELECT __time, BITMAP64_EXACT_CARDINALITY(unique_m1_values) FROM " - + DATA_SOURCE - + " GROUP BY __time ORDER BY __time"; + String sql = "SELECT __time, BITMAP64_EXACT_CARDINALITY(unique_m1_values) FROM " + DATA_SOURCE + " " + + "GROUP BY __time " + + "ORDER BY __time"; testQuery( sql, ImmutableList.of( GroupByQuery.builder() - .setDataSource(DATA_SOURCE) - .setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))) - .setGranularity(Granularities.ALL) - .setDimensions(ImmutableList.of(new DefaultDimensionSpec("__time", "d0", ColumnType.LONG))) - .setAggregatorSpecs( - ImmutableList.of( - new Bitmap64ExactCardinalityMergeAggregatorFactory("a0", "unique_m1_values") - ) - ) - .setContext(Collections.emptyMap()) - .build() + .setDataSource(DATA_SOURCE) + .setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))) + .setGranularity(Granularities.ALL) + .setDimensions(ImmutableList.of(new DefaultDimensionSpec("__time", "d0", ColumnType.LONG))) + .setAggregatorSpecs( + ImmutableList.of( + new Bitmap64ExactCardinalityMergeAggregatorFactory("a0", "unique_m1_values") + ) + ) + .setContext(Collections.emptyMap()) + .build() ), ImmutableList.of( new Object[]{946684800000L, 1L}, // 2000-01-01, m1="1.0" From df0725e0cbbb14960a82855b85cea2025430b82e Mon Sep 17 00:00:00 2001 From: GWphua Date: Fri, 23 May 2025 11:30:06 +0800 Subject: [PATCH 23/65] Add JavaDocs for interface --- .../cardinality/bitmap64/Bitmap64Counter.java | 30 +++++++++++++++++++ ...ap64ExactCardinalityAggregatorFactory.java | 3 +- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64Counter.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64Counter.java index d02377c80d26..74e4315b1163 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64Counter.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64Counter.java @@ -21,13 +21,43 @@ import java.nio.ByteBuffer; +/** + * Interface for a 64-bit bitmap counter used for exact cardinality estimation. + *

+ * Implementations of this interface provide methods to add values, merge counters, retrieve the cardinality, + * and serialize the counter state. This is typically used in scenarios where exact distinct counting is required + * for large datasets, such as analytics or aggregation queries in Druid. + *

+ */ public interface Bitmap64Counter { + /** + * Adds a value to the bitmap counter. The value is treated as a unique element to be tracked for cardinality. + * + * @param value the value to add to the counter + */ void add(long value); + /** + * Returns the exact cardinality (number of unique values) currently tracked by this counter. + * + * @return the number of unique values added to the counter + */ long getCardinality(); + /** + * Merges the unique values from another Bitmap64Counter into this one, resulting in a new counter that represents the union of both counters. + * This method modifies and returns the current Bitmap64Counter instance. + * + * @param rhs the other Bitmap64Counter to merge with + * @return a new Bitmap64Counter representing the union of both counters + */ Bitmap64Counter fold(Bitmap64Counter rhs); + /** + * Serializes the current state of the counter into a ByteBuffer. + * + * @return a ByteBuffer containing the serialized state of the counter + */ ByteBuffer toByteBuffer(); } diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java index f27eaad6e34f..f0e7e30af74b 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java @@ -88,8 +88,7 @@ public Bitmap64Counter combine(final Object objectA, final Object objectB) if (objectA == null) { return (Bitmap64Counter) objectB; } - ((Bitmap64Counter) objectA).fold((Bitmap64Counter) objectB); - return (Bitmap64Counter) objectA; + return ((Bitmap64Counter) objectA).fold((Bitmap64Counter) objectB); } @Override From 3c675801c2be9a5bdd65bc353589e5cb1e6a18f7 Mon Sep 17 00:00:00 2001 From: GWphua Date: Fri, 23 May 2025 15:36:08 +0800 Subject: [PATCH 24/65] Fix null check not passing --- .../bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java | 6 ++++++ .../Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java | 2 +- .../Bitmap64ExactCardinalityBuildAggregatorTest.java | 1 + .../Bitmap64ExactCardinalityBuildBufferAggregatorTest.java | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java index f0e7e30af74b..cc21250da2a5 100644 --- a/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java +++ b/extensions-contrib/druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java @@ -26,6 +26,7 @@ import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnValueSelector; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.Collections; import java.util.Comparator; @@ -41,8 +42,10 @@ public abstract class Bitmap64ExactCardinalityAggregatorFactory extends Aggregat static final Comparator COMPARATOR = Comparator.nullsFirst(Comparator.comparingLong(Bitmap64Counter::getCardinality)); + @Nonnull private final String name; + @Nonnull private final String fieldName; Bitmap64ExactCardinalityAggregatorFactory( @@ -55,18 +58,21 @@ public abstract class Bitmap64ExactCardinalityAggregatorFactory extends Aggregat } @Override + @Nonnull @JsonProperty public String getName() { return name; } + @Nonnull @JsonProperty public String getFieldName() { return fieldName; } + @Nonnull @Override public List requiredFields() { diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java index 202171e08058..bca09a7d32bc 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java @@ -71,7 +71,7 @@ public void testFactorizeBuffered() { ColumnSelectorFactory selectorFactory = EasyMock.createMock(ColumnSelectorFactory.class); EasyMock.expect(selectorFactory.makeColumnValueSelector(FIELD_NAME)) - .andReturn(new TestObjectColumnSelector(null)); // Return a dummy selector + .andReturn(new TestObjectColumnSelector<>(null)); // Return a dummy selector EasyMock.replay(selectorFactory); Assertions.assertInstanceOf( diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorTest.java index 01c67419aac7..0b4ca5955b39 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorTest.java @@ -34,6 +34,7 @@ public class Bitmap64ExactCardinalityBuildAggregatorTest public void setUp() { mockSelector = EasyMock.createMock(BaseLongColumnValueSelector.class); + EasyMock.expect(mockSelector.isNull()).andReturn(false).anyTimes(); aggregator = new Bitmap64ExactCardinalityBuildAggregator(mockSelector); } diff --git a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregatorTest.java b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregatorTest.java index 141c0c0540f9..38520ff425af 100644 --- a/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregatorTest.java +++ b/extensions-contrib/druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregatorTest.java @@ -40,6 +40,7 @@ public class Bitmap64ExactCardinalityBuildBufferAggregatorTest public void setUp() { mockSelector = EasyMock.createMock(BaseLongColumnValueSelector.class); + EasyMock.expect(mockSelector.isNull()).andReturn(false).anyTimes(); aggregator = new Bitmap64ExactCardinalityBuildBufferAggregator(mockSelector); buffer = ByteBuffer.allocate(BUFFER_CAPACITY); } From 033db629091b4c87783ecea632ce851988e339ec Mon Sep 17 00:00:00 2001 From: GWphua Date: Tue, 27 May 2025 10:48:54 +0800 Subject: [PATCH 25/65] Add dockerfiles for DruidExactCardinalityIT --- .../DruidExactCardinality/docker-compose.py | 35 +++++++++ .../categories/DruidExactCardinality.java | 24 ++++++ .../DruidExactCardinalityTest.java | 30 ++++++++ .../ITDruidExactCardinalityTest.java | 31 ++++++++ .../cluster/DruidExactCardinality/docker.yaml | 76 +++++++++++++++++++ 5 files changed, 196 insertions(+) create mode 100644 integration-tests-ex/cases/cluster/DruidExactCardinality/docker-compose.py create mode 100644 integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/DruidExactCardinality.java create mode 100644 integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/extensionDruidExactCardinality/DruidExactCardinalityTest.java create mode 100644 integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/extensionDruidExactCardinality/ITDruidExactCardinalityTest.java create mode 100644 integration-tests-ex/cases/src/test/resources/cluster/DruidExactCardinality/docker.yaml diff --git a/integration-tests-ex/cases/cluster/DruidExactCardinality/docker-compose.py b/integration-tests-ex/cases/cluster/DruidExactCardinality/docker-compose.py new file mode 100644 index 000000000000..74e22e8cdd4b --- /dev/null +++ b/integration-tests-ex/cases/cluster/DruidExactCardinality/docker-compose.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from template import BaseTemplate, generate + +class Template(BaseTemplate): + def extend_druid_service(self, service): + self.add_env(service, 'druid_test_loadList', 'druid-exact-cardinality') + + def define_coordinator(self): + service_name = COORDINATOR + service = self.define_master_service(service_name, COORDINATOR) + self.add_env(service, 'druid_host', service_name) + self.add_env(service, 'druid_manager_segments_pollDuration', 'PT5S') + self.add_env(service, 'druid_coordinator_period', 'PT10S') + + def define_indexer(self): + ''' + Override the indexer to MIDDLE_MANAGER + ''' + return self.define_std_indexer(MIDDLE_MANAGER) + +generate(__file__, Template()) diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/DruidExactCardinality.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/DruidExactCardinality.java new file mode 100644 index 000000000000..deab79311a7d --- /dev/null +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/DruidExactCardinality.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testsEx.categories; + +public class DruidExactCardinality +{ +} diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/extensionDruidExactCardinality/DruidExactCardinalityTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/extensionDruidExactCardinality/DruidExactCardinalityTest.java new file mode 100644 index 000000000000..edaebfecb5eb --- /dev/null +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/extensionDruidExactCardinality/DruidExactCardinalityTest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testsEx.extensionDruidExactCardinality; + +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.testsEx.indexer.AbstractIndexerTest; + +public class DruidExactCardinalityTest extends AbstractIndexerTest +{ + private static final Logger LOG = new Logger(DruidExactCardinalityTest.class); + + +} diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/extensionDruidExactCardinality/ITDruidExactCardinalityTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/extensionDruidExactCardinality/ITDruidExactCardinalityTest.java new file mode 100644 index 000000000000..67ca81f0ce69 --- /dev/null +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/extensionDruidExactCardinality/ITDruidExactCardinalityTest.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testsEx.extensionDruidExactCardinality; + +import org.apache.druid.testsEx.categories.DruidExactCardinality; +import org.apache.druid.testsEx.config.DruidTestRunner; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +@RunWith(DruidTestRunner.class) +@Category(DruidExactCardinality.class) +public class ITDruidExactCardinalityTest extends DruidExactCardinalityTest +{ +} diff --git a/integration-tests-ex/cases/src/test/resources/cluster/DruidExactCardinality/docker.yaml b/integration-tests-ex/cases/src/test/resources/cluster/DruidExactCardinality/docker.yaml new file mode 100644 index 000000000000..e230e17b94d0 --- /dev/null +++ b/integration-tests-ex/cases/src/test/resources/cluster/DruidExactCardinality/docker.yaml @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#------------------------------------------------------------------------- + +# Definition of the query test cluster. +# See https://yaml.org/spec/1.2.2 for more about YAML +include: + - /cluster/Common/zk-metastore.yaml + - /cluster/Common/kafka.yaml + +druid: + coordinator: + instances: + - port: 8081 + overlord: + instances: + - port: 8090 + broker: + instances: + - port: 8082 + router: + instances: + - port: 8888 + historical: + instances: + - port: 8083 + middlemanager: + instances: + - port: 8091 + +# Metastore initialization queries. +# REPLACE is used so that the statements are idempotent +# The fancy formatting is for human consumption, it is compacted internally +metastoreInit: + - sql: | + REPLACE INTO druid_segments ( + id, dataSource, created_date, start, end, partitioned, version, used, payload + ) + VALUES ( + 'wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z', + 'wikipedia', + '2013-08-08T21:26:23.799Z', + '2013-08-01T00:00:00.000Z', + '2013-08-02T00:00:00.000Z', + 0, + '2013-08-08T21:22:48.989Z', + 1, + '{"dataSource": "wikipedia", + "interval": "2013-08-01T00:00:00.000Z/2013-08-02T00:00:00.000Z", + "version": "2013-08-08T21:22:48.989Z", + "loadSpec": { + "type": "s3_zip", + "bucket": "static.druid.io", + "key": "data/segments/wikipedia/20130801T000000.000Z_20130802T000000.000Z/2013-08-08T21_22_48.989Z/0/index.zip" + }, + "dimensions": "dma_code,continent_code,geo,area_code,robot,country_name,network,city,namespace, + anonymous,unpatrolled,page,postal_code,language,newpage,user,region_lookup", + "metrics": "count,delta,variation,added,deleted", + "shardSpec": {"type": "none"}, + "binaryVersion": 9, + "size": 24664730, + "identifier": "wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z" + }' + ) \ No newline at end of file From 2285590e15ec8d0f0cb849ea2f4f4ce0a3d238f9 Mon Sep 17 00:00:00 2001 From: GWphua Date: Tue, 27 May 2025 14:38:58 +0800 Subject: [PATCH 26/65] Add integration tests for Druid Exact Cardinality --- ...ruid-exact-cardinality-34.0.0-SNAPSHOT.jar | Bin 0 -> 38784 bytes integration-tests-ex/cases/pom.xml | 5 + .../DruidExactCardinalityTest.java | 208 ++++++++++++++++++ .../ITDruidExactCardinalityTest.java | 2 +- .../DruidExactCardinalityTest.java | 30 --- .../druid-exact-cardinality/data.json | 10 + .../kafka_supervisor_template.json | 69 ++++++ .../druid-exact-cardinality/queries.json | 49 +++++ 8 files changed, 342 insertions(+), 31 deletions(-) create mode 100644 apache-druid-34.0.0-SNAPSHOT/extensions/druid-exact-cardinality/druid-exact-cardinality-34.0.0-SNAPSHOT.jar create mode 100644 integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java rename integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/{extensionDruidExactCardinality => DruidExactCardinality}/ITDruidExactCardinalityTest.java (94%) delete mode 100644 integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/extensionDruidExactCardinality/DruidExactCardinalityTest.java create mode 100644 integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/data.json create mode 100644 integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/kafka_supervisor_template.json create mode 100644 integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json diff --git a/apache-druid-34.0.0-SNAPSHOT/extensions/druid-exact-cardinality/druid-exact-cardinality-34.0.0-SNAPSHOT.jar b/apache-druid-34.0.0-SNAPSHOT/extensions/druid-exact-cardinality/druid-exact-cardinality-34.0.0-SNAPSHOT.jar new file mode 100644 index 0000000000000000000000000000000000000000..34bb42a227192dd93d936be08d89d49f7a3ec209 GIT binary patch literal 38784 zcmce-1C*>$vZz_MZQJ%K+qP}nt~zDA>XdDpr)=9gWm{9XyXW5SzW4Q-H#7Zmtz4=2 zBX`D**ctKd*z!`qAW#4R5C8z+bqdM=e|LlYel8=TEI=zED@rf_Er#+f2JsiMJCFTt zc);(MKmY*W@BWvVjDW0!sECp>os4L7{I*;W1H$mO#54Y-f{`TH0RC?DB!*5F4K-tT z&c!aBAYqa8)AqRO5Ydf+oXsWgh0p5+5-j=x*NtTJ;Tj1781iD>lvQ z#n8PzL-Z5We(`LR48b-+A#AyCyvuI)3=+>m_AL0sQ%-qtVMG-7w0`iH0tA_0cOmsl z+ToY~@d*s(LtChQyoNGqUuUz74ZQd-O+L!*cI4?clvx{Z51yn^pL-}KNIQS?#UrxN zYc7b`-i+pVEAHxWFU#aZH?Ahf>Q}v?a?a$?;n$q24}kwL;QwrAq`x=3qizx6f|~pu`vD@;^_aAxPyy{qsPCvi1>ef zk%5_+qluY;vxS}QzfeH=|4f02yMdAOzfi;azoW*;z|q*k*1+1r`Ct7s-v75M4K18) z4D8uh{>5+w|3B+?a6(3!Y9o7g&i zPvyVzU7`OyMSsyu`R6)=7TQS)8iiG<2Umz?ZFCr@}A}b^zqEr<# z3k$>uBlN{Bg62Qv{~Lk^K@%UnkW2+jHt^(xkP+~C+PC>2>*TsYamv|YiI5PxG+&6i z5dv&gux>2((pBsyB=DB7f`x+>vf(fGt=t7OeP5&G<%8wpW&7|GhpUfWYc3Z$>Obf3 zWhVvf|F~Q>HtvPmd>dSs1^@uz?*}L?AtWNJB(kZg<+MJ8>ibl);#jO?0al#NVu5}9 zv+f|YO2|UE&6X@NjI@@Z1*~?2jv(UW;YVs$K%ueQtEx;ftHk+g!|`qQRn{Q%Ou zwC}3Z8_A(i%lOrLc3=mux{X1}L!zDT5*%=xM=gF8e{^O!QHN&ZfLOOTA)&cyd5)|T zD)HNx5LZjw|3oqh11Gsijw6+T)bz*P(V!pd1=ba)!m)z|!>zP{BCLYK$5L$Xo!@CO zQtPXqYpgoyB>Bo+IWrh`bRTGQH={Y3jcsN}U>q%<-R%=&rZ*84bJ!EqyH#vZYX*4I zxJIJNhGezPI8xF6fTw05uNduq1Mk#5-63>kbomO=`uhqpW6DPrRy94cU!)&&kI&mbl_ zVA~fkhyb( zgWGih^F!_MwPPb8M-k9j*aoqHE)1W1@O#aJYvVn4Ymow`_lET+Xr6s}`#Ib@ZmW?s z1)rsr=5yV~C|n~PB#^=^r?+CLf4&{oR~?#St%tZaEOvNdDC+K(p zGyfWL@Tpm7+1K7b7xHk&wiQ&ggZ6*!jQm(0=OHQ1cctdi>>-G0i~KM984ZDIPu}Y^ zH2~vy#5YG|kZ<`0ihyQOfNPU@lZ=0d_}l7(xpVYmYeC^H8LM)_sB4V~3TV;q0m0?s znXC^?)gt8L_Gh8*B}R6u)!q@4C>cGbjzX!8or2fy1tFOuuye-%Z0vW!zMpiFGz-Xk z6XSB#6ew5P!$O53@BcVxZ3qr?KsE+*d7|E_!Kd|=4vx)SfK!BTBf z9DY~sYDA$lN9Pi)sI-KO1K%WP+5pQ3-7ie6YKM_s1`uveAw~U3R1rU0N-mk_l2n$s zyiJt!hUK@kKg@}s#I{b4HAAjlTBHDv!gd2WXn&Y49%sD5)(GY6C;37o6#=NQ-Mb{? zZ^Ld2z|G?hvL!Gpl9A}ii)v2Xe}k-v*eQK@xoR1Yj4EuhhPUri(3{4Zwr6pBYqF?2 zv6pVDi?;j*FQI!P z2_s!tz|^J@+kV5*S^3anIOTe7ek3;7y^N)7SU>ngLx`teLAz#rHVQ7GCHWCq@R!Uv zg*@HieKYZSn7lB4yuq|6QQ)FA=wk}e^y14oOGxaq3yhZaagJ_lHw^ zJ$Cq;WUJ+nh0l;N{xA`DI?1UY&2Kh#*0y16HAIswT?*?f-=iR0 z9F!3km&uHls(>x&r!>Ja_EV&CEm2GJP+O`o1COYM=z-ijCJ(&`@ssTWFkX{_79Eg& zU9R^Yh*;?}POpND{$RFUQ(_hq=P^PI>iaB`5-6keU~xPKg_60bPjRp)@nIUhM}Ufh zOD~GZRfu{#9YQlyH%uRT74%%KTqOoncX}juvRvpD?<)fnw)}NoP-Qee*$fxHaod=0A z2o_ok=!^Tlz#HYe;Sidf57C>1pV>Idias2dB;6%USdSaVUbj1B3Kr)DwSLEWF@p$! zj{@{naDGEc#k z+22UWN*SF#X5Mh!48rwFOsC#u4g$1J zI1YmFh*)bz_g=x;Y+E8H&#mUE(rmGsE!4y2x&0GGtuUi7%TrS1N2QE7RcNnTb4$$LA`|kI5>= zGbm6rdXqwGfKK4p=D=wfu}({luFJvEAuW#O#)EfEJGHqzx$~RP@W>FcGzA$$_94-f?mXb%C68BYgOa4LY(&6PMf^ajmQ|OlSryTBxv)ZLxe?^ujK-HkbPvGR4`E62*6Z z?}5b?Sh9KJ;scP&U|yd?WCmaK{`gNPdWUMzI6ke)LB}7+ zd^AhrE)18$haa)*_?LsjgPWU%EXp7|etOl_{+H(6fWE&^N~OqW zgQ->6$Yd~uofI+cPXT1;(j%~lyrAmzC*BS}-p+t3yFJ^qyE?S0KRrL@A5Oaw_;j|r zJ`fwV-)p^lI=|8d|7_m+x^x{agQMvVUF^Z(Jnnw%$!_?zc64dzIW--w_k-D?@qgj5 zEjpj^vomj?$_{|?n910&&N*?}!_J1p%D>h~O6By1BO4UCYTEb{N;jRV?g2XU%a-7WI+*w3Of9l=uq-!rQ*b5? zL@psdrnBX`xxQXeJI$Vr8zbi7(CD<;{zu?Xps9Q8EW5Yf&fq&8?GwMKJ?CylA{l$c$kvy3Y? z5@$;fS{`Bb7ghjEAH|83H2G#eiLhK^J`K#fG=>+M4l}P&zf*Uj&8c~&AHI)9) zV)|@bBQ2oJ%b5r~XhgQwa()%%}%{VX_9J;IkN0wGmm>h$u zv@P$~R6Tha2p-(4{jS6QE-RerLc*}dLpZxiPK-H|J$mrxHwxnD}QRPAoB9{+Sp?M}w%+(P7 z6d{PG=`P1tyw*2MpARJEVp6^SnO}Q&pV`bPiO~zP+xGoZLGJ-27|&ngJ-|0)4OPh* zr3isp=CHqEaH`g05bXIOb7`E1?H*q&B=*iO@3rsHzX1QSG#98Wk9Yen$&0`5|A0hf z<&?jRbDs$TxPAfz{@EWTX+?CwDMcNk`=D#k2&Ob7utNHA?q1Qw>(yW5zv}M-c62p` zUH0GU9AJT5F@_^?zY8v{626QV%1zWj@Rnd%BJ!dX1nPtSl}>(XUn_CcQ`vakMh7ww z^3!L~5`u)9q5G|-H^xa5R2W2%7*pCWe*BRR8vFoZwrnV1MzxYk$E`t>bOf)7+gu>B9tS<|Oe4oADFgfB@N?s< z__twS+Y}=8dOXq=%eb*d};O2 zpx!_F6kum{#p$;%{r>hVrSHf80Lp*E#eb#1f5p0gPiX%NSpOcw{U^p0{2R{{vU9O@ zHgTjgvNmvXs*-1v9pFdEf+h<^00KqQtHu5_NHL-t}h@(s`zfUdHF5b6lN@1eFwU(nKSjY2GPc#O< z$kOC%bcdT?WQe~`q)sfhQt)74yjlI)k(>D~web7-JeDbEl@1jUcu10c^M}{bgxl21 zQC9lL=l-1<0J{B;5R5&EMojVCYyqlerWd4|g8_g%AA`B zJJnXLWVn{19_y4My<@54>5OIK-B32l&-%2eV(KR{SPC}}^QgSd8VS2UD4f}AWHliy zTyc^_Ho3@P1BbNF*|hW#5j{Yp5#_omG4_=24kzM%frL4e$q6fcwbynQ6O@;d!~jU;o`p}!@p*zEwoF=f4Hp1@E_J;Qcmx|%pCMZQC_+gz&8QlTVp z{ASpk>*QwiAK0=?f&XDSds2)>c_wqCz7#FQXx6zro^yTj8{3Jd4& z{JOj>pR{9F+4^u6Zj<6C$q;6xO}Pj#a_iWP+Hve}FtgdgsteFFQzS}U=Kec7%2=r3 z){8hg8t`3%xW^2*SMl&!5pejp(LZ(|bASBEV-i0>B<8~tLGBO3wu27&h|<3TtKaZv zKk(992>DYkPi9I(QAiv~9@xnUV+{zBSU7MoNyss-iG=8hS7e?Rbv;!Bj>Gu$m+9fT zM?mZ;SVCY@R6R~CJWu*4qyvp#vm$fx|Hu*8aKA%(VRRelbA7?3C;no<80czx1~j=1 z&iBFEgJ&~kyLlsiXMxEgnlmJsx5ZUhyq12kcxRW{d6a+Q0b#)cL)=kk@yy79F|%g_ z!usjI*9B~4pj#dIQ;gwQF!L^5!iTCR@(?(t8nny?Ns1`2(?4?e4vB8Q0TxuduSs61 z*SPf$jW)pR2gpB0`?+TB5c&63X%Hp=fX)B8X#ZQRt4C_#ETVqtsHd5v2g8C)45%mM zYoLvhWqj|Sy7cMuqbZ^>lCYiBa?>PEOj(oVxaBoM-ujGhcpYzT|2THTd)n}L+E&{Jx<-6w4yIC;f*2_ksOsgN zAN!q@m5`xoWKoe@WMl?i&Wf$Xgef&i(QJxU*SgGB^L#Uj*jg;*NX7#oHl zoE;n&os z2#F2aFEt>sJ{Kj8+rL7Rryz>V25r2Q8^k4)Kdcy$yaXe%e7?F;XljV@yPsjT3Dw~(_`CxXE z7E`%jdtxG(&`KD*RI65TR?Dv=nwQ-;3uBN%rwCnnv0{)6qM*;;12x&^yc7l4KY;SO zT~%D+}8 zZU^xpUW+k+@}17|vb{Pe4~YqMj!Dv$U|G2N1#CeXR%1}14xX6aGh)IUXy=+HfP1Vz zB9ja=N{e&Q?StTHxVE-pN)n9hok{XePwkQ~V4j=$Co;+I?mNTH)fN9?scTa#=f>*? z%Q7}qWSX<4O3!^ul?@{(Ha2FE$(Nz_@N5*8Rqn-v*kzd#pD{7UEXorm;SzvgFLOvS z-4W$FmBHl9LTx3j$jSq;Ekt=Un9M0w-BN|&O$1*)fKG-{=o{=G8Lw49D1E|(0V;Fh z0Fq3^yZVI0WX@elRWx-~WHvt*49Q;BOA&?Oq1sP$k4hsbSu)%AVBg=syTRX80v+J| zFft#+q#Kl>JFw?s%9?b z7=N-tQzO)gAA?ry&BGA?9h;4Xfyd30!QuVI?|pt5bfIi$Vp$?&o@_c- z@|BwjEfV7(NssH6CQd9B*y(sU(?<87#FkLXOsFl^hrgza&UU?Il%^W*WF>U$ZHQ{S z0+-{*8HpQBmmU3nNXEyjK-#D8-vaJTBWMv(hn6+>SGdeBqeecHpi}A=e2N*p>U8?n zHRcxN#7LczCZz*z(N`3+*6Yr3x|Z%>*bB<<-epazvbzQ~x2YQipKvkT6h}HAX;e4i zVxoF;uO!ZOQ9Dprpj9TV)ztC1RdM=uU?ij9sNT5>9-a##g?P!#PpzjUI2BETGsluP!Ti&Qb^bEQ{_UyOsKRJi# zAPF!}rJE_!%e5AX&!Xl$jGxLltNh5vu+W)oflWV37qopCGhP*>FfNJx5ZvsbW6_QX&XBwB+QLQpfeoGK7(+f zZOx|V$o5qD0kC$ma4z zIjFP+r1b&aXf0}I<8L7o_9X)S#)egt6mH34E6}r4@|(@W@MrIa?-PTU5uYJ_PfunK zF}bd>;F8L(qvS$*nNW3$>!6?0&EkV(SK)$dc4vg0;ht{o@O+Bgr{Avo@p5{n883Hv zb{yLGv0gEcM|eUxn2s3wZN+0bNu&hyZUZhU0zZLz?)f93Y@v1xN7tZHQBb@?o|bg4 zwX4Y2ZkQ~A(7CsU$P&(W;a(?qji&R8Ux_ejvh*prB~CagrU)XXJ|s@{Xs{g6MWC*! z9f#Bi;=ODKNyng4M`v}(DYtv^Cf>B`OUzgBrfaKOlUdx?oJKuC3T5{6R%j4Y1NG5S zyQdpOr6w~u*c2x7Ny&j|6dVC9DM9UCDaDaaeMa=`;;+2o|6|7XB@IvV`Q5lS3igdu z{%skXpo@jI@qZO*e=CWVt^T7n`yqtCo+5%MFbEFSy%td&3uT!sOy$i0ic}$UzwYc- zA7ksbd{N!kvsW_%A~+-5^Hvb$7TToD7@xapjOTXs<9LJH?P&c;P7h$hC@+}M7{*xU zAkGZKXK>qbe81bO%0(BF+jL|*wL1zOgXvJRBOE9KOp+|rfGM27krvPKt*ADOB0lDU z3~`n_vcr}{8*jX~(-IJbCAEayX*Gsv@Xo4Ff z#AJrRsPb_{WFrnq&~-?C^>yk8{zHkMrvqudjKIS(S-m7e7X%>$9`!95?m4{E=uo#w zKj%3H^jnEk6<(+J4YCF~f#=Cg?Q;wcdx`~7HXJbO4~LVR10rEk-LS9Hx25_S!VPyT zLx|>9q|TZl7W4hbVZzjcp0mzFbL3B;N`j$FOW~@Sq?0l`3g-jWNF5G9%tF`H07wr)GB-|1(oBbgXY9nouJHnK4jxfm|;$NQD zUrEN~MxdCV@o1uC+!@U|m+MY534MU_7}kB?$+m^VKUzd_ce2gRqNKL3OVyeJMdvdV za%_~^EdTk6y_OBws2DM{U{z-q9KuJm@Fr4maz}hc^>#=g*}fA?5mp2k?2W(M{pfuS z{EuNczJSdZ{*6gQzA=g8|EVzi8!P$0Ny)k+vKYb-V#skME4n~^0tZ82N|JSRLJWu% z!scMm;^6)MnDWvwp`nfflWX0iSvr&B%v*$n@;E6KUIP4N$!SaC)dYseAom&DY3V0U zAJ1a3x9J|gPuLy2oTGR_uE?kWWDFyFpu@Dz!K&2$Mimm9yD;(6ABo&CNO15bJDG;5 zT6q=@CQ`pTm^PSOO#a*~ZGtu=U{tIXt=*|h^aHCDgz#T5M$t;>g~DT2qo370TiLd6 zxJ^vTOKfZ4WKpu#W0rJ5=|3`5?3HeEMLwZ8t4;6DuK!F!vwtvhhwfKa<09~!Fq>Ugib6)MyrIrxKY>cO(YUoUo`zTgj2I;jkE&eT}EgSgD35NWj~8; zXq~#=G&kmhJ|98dr(l%E)y&{2OJin;#fE9i)qw3xp?1YgiFM-u8y6y~2$0c~TP~ts zBDT{irR~0$lg`6a->|VKec(u+2&Om5?8866Fn~U#`DlHD3UVwXnACOB^~8J0yB8K} zCME6MdlSGg6Ou?hQ{73I*@6pMd(gf^ZE;;oBERBFm$T4T5p+VnP7cz?)@ z4QDE;f)^BkWb7-`>yOgrl&vVg!~O zU|%XF^F48Hhy~*|Q=pJWA+a6N7XJz&^`UJLM#q%$l_%G!+>WKb-gy2AaE@Z$JbKIG z2RAqS7uq>&I%V4Z#IS*gBh4i9OtHEB zFJJhzgLT#`f*=u^S^m52+!ip2UoW2}bWiF$Dls0>Q;z-P*Qj`8d8|NloM@kdmxCMU zb{sF@Q^nV4iF{9lw7SJK@G1De7X)2QO-&sC`A>RK-9iax8TCi323BYafjCi)xH)t}=wub>IFh~w zgFhydzYRzP3c2dEG;`yh6TGKD;jESVYz_sq;Z-9uCsk5>MN*Z)QU;Yez7rv;F9N?o zjZ3dh7GjLjIzg3)^2_C#s~=Zc9i7Vq+g~4j+WgHb2Hn$0tgdM(~{+<<4{C%DzGJsbJMFY>7vV~-h(w|`2`CBqvJ@=6Y8*K zV4zcUlo#qVek~^jUYvj)lFButqBN;DHszn5rj(hpFLqWOsC49JmJ7H`%DPEezF5~V zp}>f?N_cJ>j?72*$FZ}QtOfxlm@t#vrWhdDjk{tznzb0OO>6i{ib6j!EO2LB4cN1F z{H!iOl{%<^DT_&InlM~r>P+CQz{+96eSRQ^u=y3G>Yb5CYG+kmA!ErtXQho1LHR4P zOlxc=q2guYjTtv{*{krnkapUcRhH!EwD=1BKEN8rqq0WQeG0AE@d!~F=A~0OH;$=0 zjYv{C;~`Y+*fWhZY*I^ig3cUsEVaOn2KG$rZrj8hgvHdV&7mflp!zrK&r~su^jGpqE#K%qgcxE@Q0FGj6@_33>34Ivi{wY=`Xh7l+2kNz7%+ zsw{IXotU)u>lu=RK8CdUXpXNyYL7m!(vp^V==8r`{r1m>2&IB!M+GLL1BJW0;u3U zt7P7%A~C{Q*l`Qz0#^kVLp_wKQ#59L(q==g#Fc{?COSpIosDPGRi?wzH$akKSh0Vy z30b~zxyWO^eK)MtX?)*ZiQN`C-eFE^{h{^{a)$OI^kjszLt#L%1^Sx8M$cow5u$u+ z38owdPlPmiRQW2*GLc6ry4cm|2N~>=X0wH~UB83x$C)tPFba2s>|~Ybtc6c&aO#+q z$@}eBox#b$2q+gZDV@bdhy0&)yV4?U0h{gjyd~cVgIZ5tE?x=w6(CSD^}{@;#t#yQ z^kyFJn&?v=$r3(=Ox~&VS+mzTleQSEibKfB>=r0lW;&L^8)RSKFXf?i_XgC?AVuyVg94{zC z$_@JFvR+H|Xq}EJonTn9FLmYwBqQgPG@ycFfdG=-?Hva1dq6g|M6Qah87S3cA$4a%slX$3 z08DieThz2>SNj`b*z&*1_+^4ieeqiU0T{K0ECUUH$!n6fPrH&8oV4 zLF*aidZwaHoYNM~y;v2oR6hJ<{v+;do%Jmh@AlZcZSD%X0)Hp$db&VwW+V36=$!_m zOQ>M`r;h*b(=+jB@PXeyb&a4e9_@2}BLpB&005VNZ;m2lXJc<|;x1$2>}X-6Wa4OS z@}C>M$})EI{0Kgo!DNLNl(c$9@=%n7wD{D8Kw$(VSWWqg5Q>40+bh&&)L!g5HH05( z_Ja01;MWBRM$48`19{q>*AwT@UFoZsv3?()H^{xzJmvZRhL}ZIbOxaY272B|JSbc{ z_@QrePB6?h$*PyILUnD7X(FJhI{(B!Ri`@e6xlpiO| zpb*8Ek&Rh0vJ*jk%#z_Wv9lXEwM-_M)5=!Be>R3`g9Ic8*pHdhk@q`oATOzD29M$1 z<~hb=^la>{x78&Iu91gd2#5Ht3-8OzZb?z~ok^XIKOCFTFm$F#VvQ3dti_X|d_}f0 zJER0+kIP$P>oIAcP|$Wrup~VeV1yoMoMXrZ0~@G#PGys_D>4oU90%5{7^bC)iAeR} zMx;fkTjt>LACaijhQSJ_$@gU+IhTEiW-wP<3eKE&C!B_NV%<8XVhLk>9OSe$oDRFb=?aAj3}S)VnqFh%={dFA~q zL-Ay1^2I!ZnGJzU!&* z@F6Ym7fE1FcS4mSu@VE&9&nk1VYg={=bIcI9PCeJUU;Rnyh-zf9uJl*>o|!Gb3VLD zyrMw2Pw7d&S&m`gwaA6O+Lb#{6~AK=R~8SzUsi4%zm4t7NHq6e8pu^_dO2~l>L@H#Q)ESt)F z$(dsbU=S4vM=VDCPs)++7Q6;Tje-FmCPYZ*X6NFIW_~Y^!`v`BPdXlapXbzKlad9A zdve;YzO$RHJN+DD~KwI`=ihF4N=l@rTMu)@E+`1?lN<_-q790Ia&NT%;vjT>#`Tx{X|+{VgS|qqB73NEg1)kWUDU_PXl&sR(ywC zGPdX~(@=cnCV(IGX?{+DA#(PyRGtoVsxl^yW+wk!uvMx2wc{mJHzkEg-4;l2(SmAYx@7|k751@^W)Zg5LoH_1*O_6oYUp$M4KERjk`Eo3R) zP!r}x)nEbRLeAta`39oK1O{AIBTU=T35NkxEnjcVKG@KUuYxuU2QBs{to5kFjF{R2 zb20u?2j)?+*St^#3C2{KSZ=+O3YF@sqf()4)@yeVsJc)N z{6;OxN+@EKmNSI@Lp6($`U|o?@xIPNVC>F09+ylHVxQ$#Zh9zy2EW`X1`Vq(2&qfN zW44hqPTpM{8c>OL z9KvJPJnSeT-^Nw*P3$}P0~A9J=>qTcB2v3Z)qi+9C6BXk z>18(Y>=@7G7-k(yR#8sP+n4Q0B{m;qv99*GgshKOm#_xr_{?m<(!+pU;k5G8e#3U6 z55HM>=MvtQd^6Uy)XdoZ*3s*SU}LZ77+Rze#))6*87O7y(ngRi7zw%!dz`ddI1D^k zHD96Mh?QD5s4=HPLe1JWODE6uxmt$1fpz-8HJiF@dDnDeJJANrEzKyfMceEIY}w0I zoes5dwdYIim#elOS3cc(0+Oza&NO^|ULiE=yi*jJ-J2jTNApcq=9|q};!{=>d1Ez2 zUK`}|kV7qPb$C!|8<@nwJ%d)l=L5xD9utgZkjs0er**&pgZrWmIyk-nRu&KsA=nF_*jDMN07n zbrCTG1$}lHa5_%xvuP}M*tQ122lTc_dVv&GnbZ@t$~~f7@Z=tg-DgJ5>>0iK%#m&9 zv_)Y=P|OP6ZReMO{mePuerhcVq$S>wanlRygn_R^3cU;|Gef)MBYdHr5~;o6ru#vR z>qH-v)*WLat>o*2o1PN`!#hYUm?er-L6PW2YJ73&4MuAOdq!TbJ6~xBU!MZ8zuTkl z6E!WgB&c4z0-xr`DPJ-c&o$`jQXUhbgAr8~6E3p@*$rWY9>~t`$o2+(%V+qh0 z1J?xLywCcNRJJ@AI|~II0H7B6-$`ZvvVi=rsO-8E@-pTZuGG@l8L28RflBQFDO**ji=rA>q5&^Qaad>LGfW(_Q`$E ziB-a=o2!~tkV$NoJK28vV#~4j@zbX7F}K;r_B8J&xqsqwVgy|wN+GK=@jzV1l%4(N zfvV823TfO8adIwv3FmB2e;t#p#8G*Q1 zcmntX2^Vb6m)EEHGgh|x3-{>2yL4xrv-ToEZ=p6Pdt^1Qp2;1o1AAno?p ze3;Pa#FNWZqA_>bejCfzKtE|m8F#>FF2d31wJOcw4dmR0cFT(M^%Upf$1)lJFi>nn z+O_w{9>_6gv8duT$rPB^+%Y9MGWvi7?ZBY(Ys2`}&}O9`ty-$2072zSRFsgYw;VUT z4?|V6NVo1_92r8vGG6;IYQiTwC~VR7^qoW0!6OqxcVYDA29mR!5AuczX^p}O$q7Bv zl3mb5CE*_}2NzBN!pa$jV~CEyfJ2`(tK^Cdr>xa_%h79m zx)(|-E7jwshi+;`Gz$R;JQ z@)hl7*6%y3$@J-Hoq#Pr~>xtvOG4cqcMH&BrP(4&etIzUFL@(<&Z8{xXwd*E0q7La6jM~P{zKsfm#lg zn7gYF#hgg!0g-#5N#Z)VCk-XWb;yV?Ak}e`eZ2MvHNv$w@R>eOXU9T&0#02UJJUM# zHET}yHY3OQY<3vuoo_x&=Dq-2zpA=_GS9^AS`A>V=bzLct5&Fg8w({h4KfvJP4yTo zjQ_0HIXnWw1;71)C;b)~Iy|#M_QcI4L0P*A|5|_E_}0Ei$N#&Or!~gF zHZyUg)@bW^k#r5sK_wE~q&_hKZs}nc)8v8W6L@{ow(c*6MloJ;Z zwOLGFLoH2MCS9OQo~p?X>H>_E((?qPJ4v~F8Hv8C`(Z}AP?Bk~(YR=hDhNv|iET`M zI6I8#@m6i6_>Xke@4Ui{xjQkLt;&VxnbQ!*KM(_Lk8Sv?l4+-%xUDLTHX|9ac4db+ zDMKg7#&gX+({63d#P1>^Qz&oG!%}XkJ_Td5Xhp~cv{$< zs_GN^%{13CUdNK2zU5&#_D7w`RD*YyK%Wxuw|NNF^4Cpw#Pwf9 z2%phtrW7!Id@l&C&v2aMzJIS9nD(vVa6NG@p*ZE{zx9!ZeCxgohd)!@zl^xV-0$rLM*9S;656 zDEVJY^q&MJ|5<&4gfbe|VA`=x(YRZi<4T1$T_hamCklSv=UhwK*k;6Dz7fyUlkEqq zjqAyD$%4L}dRQjk)JZl2rLdy~W{76Y?bz|@u52|qg0>yWqA7xAqLQeX|_f@2Rka zx}2fJi;uq70qUFeXk#iq0$<3ma}-O?Cf^b?wh96jnBQ_T1g>RAjGHx4t?<0$-O1c% zyQ@>rgS1$6m2RShED3EV2HiAus|On6c61|I@cAFv*RgAdFiC&tKX>Ui*oJN@Cm%On zq~XglBKAz+W~i`(2lkhq#E$z|mG@!@)M6dEc-J3+>XVsQLAH!8ViJxh3M8{EkVUpI~tReL?^`M28^UaMxuMzi{dfTv|Sd(v!>$0hJK zY8X_p%82{Fd~wEpYRl6XWr7|g0xWnqBBww4N!yuC|NXSrwxFo_$uhP{-?GvveOfUapFI(_%xdHOCWtia4zMZy29wJ%WJb95M_n@!Yl1+=o%L_YL zkU^*$JV3F=JgU1P+3Acml%r>)7odFhes?ZkcssM>yyYPRy^+BEb-~wvGiLu0d{sGi znG-~Mg|3up6U0gM1KSGAS(#z}93tvfBayp-l0a2im?Is3$ZW-2M~rCcwfS`F@<1ay z>5IeK$?X>9lQ6{b4XLrstWf`u?fZ!2+0)C}cxKm2=UW`s~Oif#xCm7z|Jav^YFM{a`czdPwg!Gh8+ z`FDV1Ft?WmPEpQ#+(}F?6$pJL#Ry&<&% zSD$;?YV1SV)O>o67!*3I(H5h&n8jQ;^|u;SpnfGrWpsLNHMbKH@?wTADq4W^auY z8r?k`tFg>1nS0=hn?){Y2pgU{tz$KSDPH2C{>ylRkZjC|<=nT51a~blJC7uDj?i?m zVVsP7q-F8A!pTbDqXfN2W{S`?P1I0zZglCY#=ycP{X5dg!}w_=0hKaV`!Vh1nMx(| z5-;E*ilfn2HBky+rbAb=d-ckr9N;1lg!EtFdwEGAeuMj)T;6$|^EOuPX9v+?1sp5*u2wGm*Zr zGqR}>)j6g`J>**&U~-RaD0mugIB-?^^i=Ohc~tqRO@#H2Ed%)A4^lE+;vP{j;8m!x z_j?T?Zf4~r)X6bbvQhVy1jiv=)gN}M3Ho|yhuRB|(8@dnMBHcL^8@M379LhfGF?#h zRV9jngE8s1s_q_ASH42gdZicXq&uTjMl683TYr8Qg+1PwveEEtM)mU^$M{erThxF( zYm2y7ZW_4`{p06{ZF1(jpD+d=upyV4KL=M$Z{C)x$^@^(pC*-?7M~_rG*uh4@3Z4{ zM3GqMVVXXCY_rZeXNj9fI9``>KM)rlJQF~5uR-*395^n{418wgO-}H4YJ}DUsjGLd z$)wW4T$rVkDi2Nd3XP`91pUI~>RKQzF0q9-TgThGoL$}c5+D%SPP1^?xVBf>E#sO^ z9Qvp=M1_T2e8Ig=y5XAnIcf!vuq^a5x_wvRLS{$PMO@TkjhCVDxl@KJ?U3oUBbq%O`mf(XT0INSOVs9g-2RseSdU;hQ&vrbQP{V`!!e#x^} zpv?t#_o?>>k+Y=u>y%bW@;)`7_}9B!s=4Xnsv1yGB}Y+4+QFa-ArN+SCF7_ zdj;Nz?buD(AcHn!K$nVB1eNuR{IzD^)`u^mi}Yn#$n3qOqeqPd%J_JpNPUTJ=!y@w z-#J>itU=VyG-h_Apl*-Z$xKBlusWbn=>yTk=q?H^CQA;Up%hF6*YU}g)=>t@zd)1= zQb6ycV@)8BYK60MBNwRi=GIgzb$ys+ccor7reiJ zSiXO5;hMcEmX6O1*3nJtNnVEZc;UO{5TjR-I$6RKzK$>3xy z+aCS6DVhOk2PVNWj!N(g4V@Z|Be-7>lt3YNRx(+6-USiaEBQ~PDuP_FOwnzHRl|eb zBxE1sC9WdfN!_Dyu-C|MNKl?ddgPHB%~DpRy>K8ibhTOAvOEQ=hZPxR?}FSC!k>jAv!sD^IQ70XA~}kA6MgEDIJK6QN4lt9D!XGMf~#AIFO> znM%r|8v>NFPGr*r$DE8+-|n;Fdm8uQ5rZ6+W`g8N&NL>h+cWxE9Gj&1 zOZ2jGmj_WYf6S4cSAv;484J2YIFtc93|puw4qbiTG^RlBt%6|5NtKLXX@?_$jdCl3 zu-PWyM_^jpZQR9Wn%=(Bk*9fu=o6IC{IMl+SAyfnXSHAhCLm*GK-JX3-V|prr7>H~ zKQe<2-i(>hf*8m5`ZK?`rAg2Oyzy$V!DYcw(u$!4`Azo?!;cI z#BlK4I>c~zNh%Nb69(~5(HI%h+N`lYcZ4X4Gdhc;6SxW*v{rql?<`K;1htGwqJM!N zIas-ZuZQl~2RL_n_3e!8hlhS-^V$GF4adxkbri!5E2t91Sm2SLkmv$0w4PBqwLc5+iL=PEPf-WSeQB(GmNUDlH1^g+0Jm2N&35}^ z2(4_f^ahT52<7gjzE<0&TxFJiyyuUSroz}mc|%am*>haIs0(A*njBbcz0igAvLC$d zulRXn()zx<`?R9n!Zjh;m^W9VWo3(%tIEWjD7+H1~kq;q!RMfhnnLJTe% zZJwd6GHU7BPug1wSYEfA?6hc9TWZ(0V-uLT5p}wG%&9syTYNJf^^xW>Vr{q@XV9&OyU$1TR;QM|!QfVD~W-$pXS$)Wwo2aU>(CELTGW zR-0(fX_0KiMFR?!l$hHA-b;cx2UUgX=CGL!<# zT@P$v%c2xaKg0rUDz^KRzZ*AUQGvwcl)((F%c`0CW^(|J)Z`?!B(*g4I1Rm{;zGfR znh?Qh0&@$6SE&Gto~f(C5Un+xX4!6k&hEUwZZmv(=^DsLe3C11m|T0|W}`Ibp;b`t z0FsvPn7g1tieinj8XSTWv$L>}l0+zq`~qVUL8x6&k=KXwb0!95g`3nm^0u^KLOMK^ zW38a`$+qF^v(z{~TK?jMD8@AtFq34_7==4!^o2?A)7?B#^Auw=&JPVlU*oE|^1HdL z6^}_1oFNEr_t3{s9CqcUn`e4hZCX}fu~Sk4Mm&rpTlZ5 zkPSCCBo^c*H%O*?7_~(Ai$!1dvo-nn(H=W=a<5p7#;dOEKJ$I0FR*Aj!jw-gZ-=TV zM8l}Vne<^AV|gCsKm{Km4>htF!|yAw;>gMq`|usYc4Y%p*q}<2a+KE(ntMwq=)s{X zWpmk17?QC^+)o=jAH_CkH*`-+B+?EgIkE98giRkAR=>hrcPYnDK?w6>*Up1iOiv91 z{TE!eD4ox5ge}D)iN7~VoFM-26^YuEUkC>TdjA3biTYG%+M(Y4yhbAgZWNvBL& zqg)whl4~;eEK#j*JO{lU=S=WdpYE(ZU6DqD+lPPzsujDL1)e)r#))s=KDE zrg?h9bf!qk>>t_Vz7%xu%Y0~UBz_4%@q}NK;vBMo@;KnxSrk;!jj$&s+twN=%|~$R zE=4YP@kCMw4NY=Zzddy@P+prN4O02M+pwT_Y6$|^YE0PLJOiOg3eb9k{~jaQ4tTGj=0OuuaRLdIpmwxPC8M0HWisXP~O#Ck4gfcklDn; zJkLoaxs}50QfiVl$ww#fzElOi&ytkPuI|8lyE8m}qnZEf=1*B02gm=G5u|eHGN*|2 z+Hoj3OJtSHPeu-*YSb#K2`~Uym?>|XNljm<4tF{J)gX;I9!dRr$>-n;B}jZzSvVzl0fN3F zA@5iV5Eh4JM|U|x&7eVn*3vc+G>okdhjCN>IzJTT%C{P-q|8}YUlWvoq=s;JdT07qvp(cDeHr?+N6c=oHSTV#gV}N2J%F53n?{NF07xcErUu% zuKjWu)EcRGms?XY6K?Cv4`K!xvljznGx3!^rc*R-Q8aC!Sjtr^386O9t;W_|sZZ3} z2Thr#kSH)D=}l!dB-mmHp(t=V_;Enq48lccizz!WEFFi3Rau+*606l?H<33>4V*6Z z*w(`YLDXwmEiHTxY`6U(`Qwa3Bsvm0TYMr*LQ-Y|DjO>+lY3+eS)E;P7jiXe)W5PB zkpuZez6R5Uu0Ao5*v(fnCElZjq;pfb7yqLRgX9_(EJ;P)Aw(N8*HS)yly;DFEzynx zO((JXRP=|BcV$bEk&dInL{tnzIhs4da+295%|zVrjcVS+XXise5IBl8uutfZ3*XK{ zG__Nz93AOw5bUn#+B9=E!ibBDqZptFkFy1#xC9xPbxF6y5UyTIRB_0Tgq4>dVCbG4seFN7M(rO95B z#z9HZOG2Ho&g|-G$%vMv(-!JuFesMx2ja7c+F4Wr@O<}+7bQzkvV(AdN*`<4ET8H> z05qZ_QW-bIRk;A}dv;>|!tQ@JCjyL)ETyUjF&I@~9agH(O?9G{#oWGZV$@7GWTn*! z+FR0=6F_%72!H!OAnU-{%8xQXhUz?S+4^#M1*q%#u^)*i5 z_$n@rneQB216a3>qQ|(zPwoNH`D2+f?oosla=L^!q^%t} zy#)|0&Lr_>Jd;Oab<(p#Y%~*}?K!R4Kc>lOE|A(`qS{`SE8JS>JeSk1?xFjTVTCFP z^~DIzEEZJ^tL|v>V{4sHy=w7;)g%UUart=l9;+6#ccUe*sNc3fcw?`%^Ic%Bwl0$0 ziy+D!t6PX*)wN>`mlm5zv?yW>FMmo%Hzc|XC;NfCJvUxhk(UW?(E4*(M`vV(f<@Z} zJ`SFUAer(^+UB#2)nUlrN;|H53v5vCFge=5>t*>DZyD_13f@BJLr<)FeS?Ypp-Q$uP6e;Ie%b3#+ zi00jYL~88Q3i@FRi@Ol{95tIktZ-}w=5H{RwK%1gZ9tlze5n-o^7|5@b#o2(p>Nyq zJb{OCZr*X!2i(1kFj@}>npMi?`idqr3)_pw|HqG4;6j@Xl6o(hJg^DZ**r&|n6sy; zh-cH{i9@ltlbhmA$}GTkxpGh{5&K-QiRtE8LYdk4*#;@~gd&RV7;Cd7zGzBdshC^S z8)80vFP^;9wm9U-EZZrKZ#qOM7441UX<2i$pHKm?3B$Q8@_ck>J^ceDp8Y1whd1GN zmW@$3u`>4pFez`M;j7An+EXj;pN^~0%JWpOIH3F#(|%L)f|?kV+Y$u za}e(ejgw^w>r0kr!Y*j<61roZQVrIIY&D$2yNoJMF=oRY>3N(H4JX2ffr)r2d9`qg9PCuvp>(PL*&^9dez_i;s<>~!O-Yr@J%6cS9s3z zX*ps(ac$Dr3;am!Uek?UaQ*~#4eue}2dO8YL4q0Va@O{+&Dv-0_G7^de5MVF8&ryB2v;mvKxODHaQtR8xw0;}ITgTs^mB`^(E8JfFI`}^$lK43 zUXiMO;;%}KQlms52kHYmDof<3m>0LaHsUuLf|$zU{ij730QfAzH{^oisMFI@#^>D> z2@}b70a?3fnB=kSf>(g(4gPS{@sxflV#`4(GZQa-(SAkw^q;l+j(vAU{3VaWo_#}- z&BV>QF$h-;D3mxg!7rzfDKQ2xvlh~|4TqO)2fjetY+4~OPLFvwcKFM*W9i)^Wp7JZ zzQX?g=I}aMEt&c47D5yT06_geKIq8X=-HcDoBSasV1x3Z3Ze?qD;L5@Z%GKLA7l-6 zKu*-BEk6mdfH%^9{+bUB+;Mp5qQ+Z>e!6#F>!?pPue_7f)Kdo1$uiGVu+Q=C?(2r4 zeqvnerz0a@+waag&ezt@*Phn0eb%3lyXjVM^FgR#R}mI+lXAVIXl&UAegtqbGZYU} z1}!V3RjHYvY4+TT!L~yMLo?u|gRn4h(dDa|WyLB4Wx@G8$m_KhdnC}9f^)~qi?|NL z!fJBQ@m=Y51VAqh)8a*3Kt&TNv03lcH+vEkr%BMu5K1!jP>oL+ImHf9vJH+m7hNY9 zQ;h_ky0@y_fpct!#LiO96WTx`KxvKUyMH_GAr8W3228^XPNr;_WPF&3$r@4HPAa{s z$NjNHnPYCKikblf$=gWJSb3DOjiN))`Eez{m7%XOr;q|Z@F3|z zKN1ZZ6EiY4qUmz-RwkJ|8?8MujUQcvHuUEq~}4>lGxbks`L3?F2vkX)(lL&}> z&Y{Jyh^@Sk8w3G|QlwF#jBa8=&Tugh9kolwN~$csBq|@ANwE<9FwUlkJO{vt+rw&H z72A1#3xuqpAM^OrnWVif-_&9a&(e!b>BjSbNRT| zT$wFQ^y2h(E*}kr{2fIiSbOT^rX8Iale6T;!8c*$zM#=%r4;}o8<@;k5iyO9kBm}Z z0Jkz3Rm&uF0z@&V)^7gHW-dcMH4-&hD&#Ebi8qWCmLE^dY$ELjOQsS}JQK4xHioE~ zlMF^*h}Pg_mZ4hwFe~IFf)B}=m=B*n^Nn}6v1o3!PB_f46PlYC<__qhxmg+Th&~=^ ze+jXP0@>WYVJ^*Yo+1K^*4~lBtg^sC%3t>)E37aBGs0(eIF-(c7tzx4OBO|N09Pwm zAaviTy5c@TSt8?>8qJ!?!Bh0maxQNdYmzj`(xAHM!H~%PlD|xU>;+lhnmoDLqaWQD z_R4l!aVR+FAhFU*DYtN2V8~8rd#)_`IuGQ&lu4tq47=AN)|v#jC4!n6!GL1)?>qaGE-7+93T87i>ZSPhY6r66D~jehLFsCf{()Pknuy+c3= znMh&1<^#wQ*vij=R~LdR8&BXbZHy}ZZ0^>dz+*nLEQ|7lxM7HW#4d6b>72ylyBI?H)0@BZ}3ry zAgV7P6qYidD7y8a2l=Em-!qyE8``Q$N475FmUG8+h@Sm?lk~LH53%dpT9X`Q+ekhe z2RklT+ksnv?Gf?Us61pYLOZNeRz4k)X$hW9I~Jki{COzrm`sqcZo!2?H~-VNa7J}K zJd<DmL5v7bu>bPR@Ymk2E#e@3yd%(YBm zr@17;yZMT?hyq9^xGk#QRKb7QP^xV;wWtAk^o0Jxv9e06gtw+9EIPvbX=mX8i+ak3I-EA$Eag0bMWc`H)j5RhsZohkK+-;A%6uhCM3)3A`hJx3g9Ya!rE zE|F7uIPDxgk}!PrDq`oG%$;u-*lfLm z0!{MFgd`rJTN5+cht?<;CEkq(`XBmCFeNXcabY{@${qMWic02&8PvkOEY6(t_q0ysQJc{)d07es#?n$rIb1yGR0n&s zaAq`Cs=}~rCEe~i7!5J_#Pz6GTW+y)p@A>yAbjl{yy*}}AsG=Zg4F5hjEU=JdW>?+ zWT=O>y#ulZViBW%+LBH9{xiyORQT(-^M)hsTF)E+!K{>Pa+bxHaD}Dl(@oV=6l<3n zt<~ISL6z&6Vg$&7VpA*TyM(Lp_@O-yLGJKas;@8I2dkc-s4vR)jmc%OK_6!Z<9VqU zUr9m$A?;xA+OB$Re!po8J0YWhga!Z@C;8tX9{rnG+&_10@2Q3r>@430SWeYFbQR{& zU%k9+S8i_k+|YSRArQpzKDS%KNeMtsGgnGcHA;T7vd5&BvhS0U50}nNx0YscIt#sD z%{bW&D#;xdBz9mgc1ZV-xpO`r@4%fH>U4-_7E5n58+dN;NpqOwT6gIjI`_zMc$%|- z#RaPNZAU@Mk>!t#(3d1oJXL-~_K*b!J*nhsQ__i`wTees&`$>sY&6O&B;_jw4v(N8 z4)|;hd?GG*c`ASEO;Gi4>akT;vseWT7=$6KpN7*|4irXXd`A|f14+S0)wsBpKfo_; z_^j^XC^NMd<8F#Hrg^Be#>Hn67H$n!#G-Xz=hUph#qm~%5j?wXPDp7rv9wkU?4>Tr z^j1ljzNMEP)2+w%k(@|zBqhqcHxElOh+Wr41r%dTed^1VGx7(fg&LMT<{H`7n&j%i zM#@F&AdLDUS8PPkCQ>>g|DdMKP&EYOu@rv=u(82UcZWgdxbjL)WIdmHdRI6idFxP& zKozeSiZr(9s7z5EqUgU4aDrbOWrTgCw7Ru$MxI49{f$dPFryy*D*KgW8h2AdVf5Zq8KEwbRZOjY#np z!@WdI!QZ=ha;PrA!z#YP^!gxko>vr_q5Vde6REN3IN7~{S2c71dY%r&Wp=|V$XdOR zD@a3%)zz`<2yS_CMAbLZvN5Go=p=n1!uD&DKn0Y);L@nxe039%j~4ri$0uibM={72 zOHXwA40?WV5pZRA(Hh-18Ifw`LG}ugieU*Nm8(=QHDvk6Ic9;=c@V`e5pX(t=4JZo z{YZ3m)(teSI?J1ckC=8z0q*E9miU$_dBy>$>W<;Y)yY~#Lcz8}F=r?qmPvLn6Gn#i z)rM6D{2J0V7#nfbLc#=TU{3LuxVafDVzsEBiI?nENf^Fr5a2tJ@rf|)A6n6`vPR2R z2;%ce3q#h?^V^V5eq%t;W>6<|HLTBzL>xA4wEQ^HiZ#|~hrXLI?C0!CVhVi!Cnj*NX%lD1+zKF5VqaZZyX9hVKr&>-GH*K=_@v72?eS#*<#(}6| zSX~=LphaS7VJ$tVNlVEM$mgRDWWw5ii$#%eBni?~g&r~)M=XN%n7cdW7pcr4URFVY zuFV$z1OVJ+%tBe67588jw~ zuB;}(ibhW%{0iLQnZeE;`U%5G0GtJGLCO-dK_wK_K#jO|!@XUG_d?i@+;J7?LuqeP5cP!^fc0BDP33q;`Mr3cJ zQWoc)gB&nV#VZ zT_3v#dKqpRl6|6(#A3ORJ?13LwbW*Y5-$sHr$LL@DRlo+g^yM{i zBM$eVd|Br^lDw#~pY~MoB^?ijEk6ots-e1NP(=nDO}C`Cu_;qyC#9|Q zEYET+362k+K~((MFrnc@9z>j_5HD1tmFH^l;K)p}{h0Z&N|KDz5F@J8ZamZNbwa)1 zOo~B+dFEviwK$%YKFVd2J)q|@T6tUNa^HoGt2X{LOJ65#UKrYC^SIdgtKP+9os4EV zu}6i8+ap6|*4G81mrae-pXV)Lh3c#bGT&q7rNE*h`lxd8<33ixH*nYp+C%CZYTZ_4 zU2E7;N>rnvXV*zj0!?Hn_ZH!fCZ0sgv1zo?>5exL#{tvqFSXN7SOu_m z^E>=BN>vpsvXCypL3TNEu@Zmj9KeqlzB9KX}cC~B)V(KOOTV%ivq3sCO~nRP-X;Wna7EMRFFj3 zXdN}q)&&}4M|P4q~01T6xq1&qmGWboszr4B5kv%1g3H(kSZ_8^v~#4msJgZPnFjsaC(-mEIaF zSncKAz$7lz%I?k1rrsfwo?#lOm8hG_oJNG)s>3rieK= z>JHW4XEtCj5zODZ*I>(INiIor@X>~laQvXxb1(6Ll&l#cM9TMpW#`NFdW&Gc0Xgvf zA?aS&tH|ahqqh#_i#$BfH#`j2oaS5F1MixIFop1bo-WDlE_?eAVv4<8f`x`+@Lwe1 zQL+6TZGlf0pqBIWmg(c>p!9`)s?L>$nvm{(plt*M&(L8F zfkasH#ySN*SqlCpVkee7ujzfO*WRGb{keHQ?(>(*WuZ9do1GA7!hNxJvw^ zPq98DVx+n4I^SHDeTwF%&Dn-{@qo^2{y?`H!FtwR*bJ+(g{K)fWXF^>(BS%EaZ^Y$ zI@vb=FtL~xF}VhJwA7F51}&Pr6hf)J(XWWXw1{DdyQ~PkM7PsBO{`9KM6*occ*%k+ zI&9i9&A|P^h&`~P!XItC(2bN)B} z1nKs$I=z=AA~Xa92kjm#@TT^OR%KDr*pPipLQRU$Aw&dY@p|mHg!md;Et*^Uqm*=Mg1?LUGf_e> zc+x8l(nW?KW*H^EQ8BPqFkyb}vcSD3AgoXyfY zPdH@mz$C2@<+yjm7r@ckhvb|ACJ&n}0{7N{G`2CXH*ngL8!o{Y@_;@v8|y=tM?f)W zEZ2S;gzYe_OAzU_(cc@u(rbYCb=f8c%x~}2880gQow!yK92qj>6QVKP&NS_3P|WvI z&6Q8Zu<}t7;)0h;uoqqde}5{=-{uZ2gC$atCgqr?EHT&O*UEA7N zQM+1Mepj)wSz(5IIn(Juf0_b>-)$tDGLC|15|hm@#TJ}s^(HqBOG*_j5SRDvseZo2 z78{9090E^jMMnV!cXdBIb32PlSn_HbW?(cBcOZjVp|LHFOv$0&#OJcqiIDbVIdzdM zf0BT!hkL}#v>y#H2B$|zB@^iIlf6eB3r$5Fmsr=c$Q7YsgKAU#d@O(rX$a`!yw;2QBImH)=Y-gqydw!37w?!MX@w3uC;nl^-!g zC@x@ShfqNCGoFiLMzKDboH7a_O;T8~g8z=@=PNRrPus&~p)%&|%OcuDARQ8F%Jvwe z<$fo*>t4{)603ZZ02}Sg^V_%A>lq%>Nur|u;^S4@1dTZrwB2YWyF!DZIcpvA&7**f zUCMGhOK`TIq@m;DiJ{;PzH5O66b&MQ>ak`|(%EIyD4MB`LCNHH`roqxtWgw0#?n>h z8m>!J<;-jo;eUk7H-vIzpl^zFe75%o;M{{aqMqT}uR1m}bv|yjXOf6n2j)Euu~BM6 z&G%jvKKHQWSrgEC%#ZF1hku;m;5mmDe%?ldo}3Qe+$iBWf(5N+=j-TE*?%nMVOZI@H0(*-}tDHy$z zLuz#e#^dnEgCL8>B%wm_Bq5&APtw@*azrw;8fc_(mD(&7d<(v$eIx-Gs_eSmDAf=p zD{X|^#^0bZ(T?#&C(c0HicykirI2YXneqt2g*zAkNJvUPx1bQ) zH*_j|Pb+}Vvla&sbpjUoqDl6~q{|V`10(Xi+IvGU%xG#7orAy_Ll3Lh2Jel8Y6f~S zORY*zYGI59EuthR``sv~wW`3V20u2C|FNMvWe0`qlmW!8?p;cgCTBrR62 zkDN<>(=(wR%*^9bnURA8CHaV?ge3K)WQ`OV`MBh?R3*Tb zV?1KGJpDC-1HA-!%S`x{1mx$d2cbjv(2t=b=%wzuh3=)uM5S&6sSj=gnntBUpMolD zLbH6ce66dq4S1`AeZWxPqSPM%k$~RVtl#P{L2Kj`-@yO&ya@#T<@?)@zby=Z zH~U4^a_zR=3H!#!1@;!=e*5chPl4}SpzoKN6rZ%1u#kcxwUqFmIex1v`^WP)$EI)X z{^r=h$llq^z{r8-&+OlpvVXBpd9(kUQrw?eyfMuDi$&`{EdHS__h*K0Sz`ZUxcz&> zzl+lSnK|k|tpWb_P7&UE@84+J{aGUfziUL~-#4Q75BW7S8|y#q4YJ=g81g@B(CBYk zZ-3e>)9;#{{Qqh8Uj_XBtSh*G=K}w&u=wBf=f8Hv-;w{Ti{CT?|I8Qv?|gax$G-j+ z&Hgigu)p)S_U7-mQDLR$Z1m?JDaG%MA>T&GJL5mw>o(K!I<=zd95Yk`ObuZd@tWn{$~Am{nhuz@1; z|Ao(=`Y*-z$Ba+!;k_r_{RNMN= z0R9*Iz^{qrkAb}>DEtL%iRb?a*q@G#_xxwS@SzL-rTG45W8*#d%P)BC5`QVY|M}Q> zPj&GNqOHvTF~mRq7kppb{0kti@?QqyL_j-s5_o#rzA`t;v6i>(6qU-{W|npY{t!!Q00BKijwcF^)gVR(p@?eY(&u zRONR6DXRa^qr?f(JR;3M(? literal 0 HcmV?d00001 diff --git a/integration-tests-ex/cases/pom.xml b/integration-tests-ex/cases/pom.xml index 74609a2267f8..15f23dbd7596 100644 --- a/integration-tests-ex/cases/pom.xml +++ b/integration-tests-ex/cases/pom.xml @@ -282,6 +282,11 @@ junit test + + org.junit.jupiter + junit-jupiter-api + test + pl.pragmatists JUnitParams diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java new file mode 100644 index 000000000000..12c622d1e011 --- /dev/null +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.testsEx.DruidExactCardinality; + +import com.google.inject.Inject; +import org.apache.druid.indexing.kafka.KafkaConsumerConfigs; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.testing.IntegrationTestingConfig; +import org.apache.druid.testing.utils.DataLoaderHelper; +import org.apache.druid.testing.utils.ITRetryUtil; +import org.apache.druid.testing.utils.KafkaAdminClient; +import org.apache.druid.testing.utils.KafkaEventWriter; +import org.apache.druid.testing.utils.KafkaUtil; +import org.apache.druid.testing.utils.MsqTestQueryHelper; +import org.apache.druid.testing.utils.StreamEventWriter; +import org.apache.druid.testsEx.indexer.AbstractIndexerTest; +import org.joda.time.Interval; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; +import java.util.function.Function; + +public class DruidExactCardinalityTest extends AbstractIndexerTest +{ + private static final Logger LOG = new Logger(DruidExactCardinalityTest.class); + private static final String SUPERVISOR_TEMPLATE = "/druid-exact-cardinality/kafka_supervisor_template.json"; + private static final String QUERY_FILE = "/druid-exact-cardinality/queries.json"; + private static final String DATA_FILE = "/druid-exact-cardinality/union_data.json"; + private static final String DATASOURCE = "wikipedia_index_test"; + private final String fullDatasourceName = DATASOURCE + config.getExtraDatasourceNameSuffix(); + + @Test + public void testQuery() throws Exception + { + final String baseName = fullDatasourceName + UUID.randomUUID(); + KafkaAdminClient streamAdminClient = new KafkaAdminClient(config); + List supervisors = new ArrayList<>(); + + final int numDatasources = 3; + for (int i = 0; i < numDatasources; i++) { + String datasource = baseName + "-" + i; + streamAdminClient.createStream(datasource, 1, Collections.emptyMap()); + ITRetryUtil.retryUntil( + () -> streamAdminClient.isStreamActive(datasource), + true, + 10000, + 30, + "Wait for stream active" + ); + String supervisorSpec = generateStreamIngestionPropsTransform( + datasource, + datasource, + config + ).apply(getResourceAsString(SUPERVISOR_TEMPLATE)); + LOG.info("supervisorSpec: [%s]\n", supervisorSpec); + // Start supervisor + String specResponse = indexer.submitSupervisor(supervisorSpec); + LOG.info("Submitted supervisor [%s]", specResponse); + supervisors.add(specResponse); + + int ctr = 0; + try ( + StreamEventWriter streamEventWriter = new KafkaEventWriter(config, false); + BufferedReader reader = new BufferedReader( + new InputStreamReader(getResourceAsStream(DATA_FILE), StandardCharsets.UTF_8) + ) + ) { + String line; + while ((line = reader.readLine()) != null) { + streamEventWriter.write(datasource, StringUtils.toUtf8(line)); + ctr++; + } + } + final int numWritten = ctr; + + LOG.info("Waiting for stream indexing tasks to consume events"); + + ITRetryUtil.retryUntilTrue( + () -> + numWritten == this.queryHelper.countRows( + datasource, + Intervals.ETERNITY, + name -> new LongSumAggregatorFactory(name, "count") + ), + StringUtils.format( + "dataSource[%s] consumed [%,d] events, expected [%,d]", + datasource, + this.queryHelper.countRows( + datasource, + Intervals.ETERNITY, + name -> new LongSumAggregatorFactory(name, "count") + ), + numWritten + ) + ); + } + + String queryResponseTemplate = StringUtils.replace( + getResourceAsString(QUERY_FILE), + "%%DATASOURCE%%", + baseName + ); + + queryHelper.testQueriesFromString(queryResponseTemplate); + + + for (int i = 0; i < numDatasources; i++) { + indexer.terminateSupervisor(supervisors.get(i)); + streamAdminClient.deleteStream(baseName + "-" + i); + } + + for (int i = 0; i < numDatasources; i++) { + final int datasourceNumber = i; + ITRetryUtil.retryUntil( + () -> coordinator.areSegmentsLoaded(baseName + "-" + datasourceNumber), + true, + 10000, + 10, + "Kafka segments loaded" + ); + } + + queryHelper.testQueriesFromString(queryResponseTemplate); + + for (int i = 0; i < numDatasources; i++) { + final String datasource = baseName + "-" + i; + List intervals = coordinator.getSegmentIntervals(datasource); + + Collections.sort(intervals); + String first = intervals.get(0).split("/")[0]; + String last = intervals.get(intervals.size() - 1).split("/")[1]; + Interval interval = Intervals.of(first + "/" + last); + coordinator.unloadSegmentsForDataSource(baseName + "-" + i); + ITRetryUtil.retryUntilFalse( + () -> coordinator.areSegmentsLoaded(datasource), + "Segment Unloading" + ); + coordinator.deleteSegmentsDataSource(baseName + "-" + i, interval); + } + } + + /** + * sad version of + * {@link org.apache.druid.tests.indexer.AbstractKafkaIndexingServiceTest#generateStreamIngestionPropsTransform} + */ + private Function generateStreamIngestionPropsTransform( + String streamName, + String fullDatasourceName, + IntegrationTestingConfig config + ) + { + final Map consumerConfigs = KafkaConsumerConfigs.getConsumerProperties(); + final Properties consumerProperties = new Properties(); + consumerProperties.putAll(consumerConfigs); + consumerProperties.setProperty("bootstrap.servers", config.getKafkaInternalHost()); + KafkaUtil.addPropertiesFromTestConfig(config, consumerProperties); + return spec -> { + try { + spec = StringUtils.replace( + spec, + "%%DATASOURCE%%", + fullDatasourceName + ); + spec = StringUtils.replace( + spec, + "%%TOPIC_VALUE%%", + streamName + ); + return StringUtils.replace( + spec, + "%%STREAM_PROPERTIES_VALUE%%", + jsonMapper.writeValueAsString(consumerProperties) + ); + } + catch (Exception e) { + throw new RuntimeException(e); + } + }; + } +} diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/extensionDruidExactCardinality/ITDruidExactCardinalityTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/ITDruidExactCardinalityTest.java similarity index 94% rename from integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/extensionDruidExactCardinality/ITDruidExactCardinalityTest.java rename to integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/ITDruidExactCardinalityTest.java index 67ca81f0ce69..208743f263d0 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/extensionDruidExactCardinality/ITDruidExactCardinalityTest.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/ITDruidExactCardinalityTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.testsEx.extensionDruidExactCardinality; +package org.apache.druid.testsEx.DruidExactCardinality; import org.apache.druid.testsEx.categories.DruidExactCardinality; import org.apache.druid.testsEx.config.DruidTestRunner; diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/extensionDruidExactCardinality/DruidExactCardinalityTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/extensionDruidExactCardinality/DruidExactCardinalityTest.java deleted file mode 100644 index edaebfecb5eb..000000000000 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/extensionDruidExactCardinality/DruidExactCardinalityTest.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.testsEx.extensionDruidExactCardinality; - -import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.testsEx.indexer.AbstractIndexerTest; - -public class DruidExactCardinalityTest extends AbstractIndexerTest -{ - private static final Logger LOG = new Logger(DruidExactCardinalityTest.class); - - -} diff --git a/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/data.json b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/data.json new file mode 100644 index 000000000000..b186657dbcf0 --- /dev/null +++ b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/data.json @@ -0,0 +1,10 @@ +{"timestamp": "2013-08-31T01:02:33Z", "page": "Gypsy Danger", "language" : "en", "user" : "nuclear", "unpatrolled" : "true", "newPage" : "true", "robot": "false", "anonymous": "false", "namespace":"article", "continent":"North America", "country":"United States", "region":"Bay Area", "city":"San Francisco", "added": 57, "deleted": 200, "delta": -143} +{"timestamp": "2013-08-31T03:32:45Z", "page": "Striker Eureka", "language" : "en", "user" : "speed", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Australia", "country":"Australia", "region":"Cantebury", "city":"Syndey", "added": 459, "deleted": 129, "delta": 330} +{"timestamp": "2013-08-31T07:11:21Z", "page": "Cherno Alpha", "language" : "ru", "user" : "masterYi", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"article", "continent":"Asia", "country":"Russia", "region":"Oblast", "city":"Moscow", "added": 123, "deleted": 12, "delta": 111} +{"timestamp": "2013-08-31T11:58:39Z", "page": "Crimson Typhoon", "language" : "zh", "user" : "triplets", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"China", "region":"Shanxi", "city":"Taiyuan", "added": 905, "deleted": 5, "delta": 900} +{"timestamp": "2013-08-31T12:41:27Z", "page": "Coyote Tango", "language" : "ja", "user" : "stringer", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"Japan", "region":"Kanto", "city":"Tokyo", "added": 1, "deleted": 10, "delta": -9} +{"timestamp": "2013-09-01T01:02:33Z", "page": "Gypsy Danger", "language" : "en", "user" : "nuclear", "unpatrolled" : "true", "newPage" : "true", "robot": "false", "anonymous": "false", "namespace":"article", "continent":"North America", "country":"United States", "region":"Bay Area", "city":"San Francisco", "added": 57, "deleted": 200, "delta": -143} +{"timestamp": "2013-09-01T03:32:45Z", "page": "Striker Eureka", "language" : "en", "user" : "speed", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Australia", "country":"Australia", "region":"Cantebury", "city":"Syndey", "added": 459, "deleted": 129, "delta": 330} +{"timestamp": "2013-09-01T07:11:21Z", "page": "Cherno Alpha", "language" : "ru", "user" : "masterYi", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"article", "continent":"Asia", "country":"Russia", "region":"Oblast", "city":"Moscow", "added": 123, "deleted": 12, "delta": 111} +{"timestamp": "2013-09-01T11:58:39Z", "page": "Crimson Typhoon", "language" : "zh", "user" : "triplets", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"China", "region":"Shanxi", "city":"Taiyuan", "added": 905, "deleted": 5, "delta": 900} +{"timestamp": "2013-09-01T12:41:27Z", "page": "Coyote Tango", "language" : "ja", "user" : "stringer", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"Japan", "region":"Kanto", "city":"Tokyo", "added": 1, "deleted": 10, "delta": -9} \ No newline at end of file diff --git a/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/kafka_supervisor_template.json b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/kafka_supervisor_template.json new file mode 100644 index 000000000000..a37340fe858f --- /dev/null +++ b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/kafka_supervisor_template.json @@ -0,0 +1,69 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "%%DATASOURCE%%", + "timestampSpec": { + "column": "timestamp", + "format": "auto" + }, + "dimensionsSpec": { + "dimensions": [ + "page", + "language", + "user", + "unpatrolled", + "newPage", + "robot", + "anonymous", + "namespace", + "continent", + "country", + "region", + "city" + ], + "dimensionExclusions": [], + "spatialDimensions": [] + }, + "metricsSpec": [ + { + "type": "count", + "name": "count" + }, + { + "type": "doubleSum", + "name": "added", + "fieldName": "added" + }, + { + "type": "doubleSum", + "name": "deleted", + "fieldName": "deleted" + }, + { + "type": "doubleSum", + "name": "delta", + "fieldName": "delta" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "second" + } + }, + "tuningConfig": { + "type": "kafka", + "intermediatePersistPeriod": "PT1H", + "maxRowsPerSegment": 5000000, + "maxRowsInMemory": 500000 + }, + "ioConfig": { + "topic": "%%TOPIC_VALUE%%", + "consumerProperties": %%STREAM_PROPERTIES_VALUE%%, + "taskCount": 2, + "replicas": 1, + "taskDuration": "PT120S", + "useEarliestOffset": true, + "inputFormat" : {"type": "json"} + } +} \ No newline at end of file diff --git a/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json new file mode 100644 index 000000000000..2a166210c617 --- /dev/null +++ b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json @@ -0,0 +1,49 @@ +[ + { + "description": "Timeseries query.", + "query": { + "queryType": "timeseries", + "dataSource": { + "type": "union", + "dataSources": [ + "%%DATASOURCE%%-1", "%%DATASOURCE%%-2", "%%DATASOURCE%%-3", + "%%DATASOURCE%%-0" + ] + }, + "intervals": ["2013-08-31/2013-09-01"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "time_count" + }, + { + "type": "Bitmap64ExactCardinalityBuild", + "name": "time_cardinality", + "fieldName": "__time" + }, + { + "type": "count", + "name": "deleted_count", + "fieldName": "deleted" + }, + { + "type": "Bitmap64ExactCardinalityBuild", + "name": "deleted_cardinality", + "fieldName": "deleted" + } + ] + }, + "expectedResults": [ + { + "timestamp": "2016-06-27T00:00:00.000Z", + "result": { + "time_count": 40, + "time_cardinality": 10, + "deleted_count": 40, + "deleted_cardinality": 5 + } + } + ] + } +] From 2a8164ade4a91750361496e518c23c8b3fc6a02b Mon Sep 17 00:00:00 2001 From: GWphua Date: Tue, 27 May 2025 15:24:03 +0800 Subject: [PATCH 27/65] Checkstyle --- .../DruidExactCardinality/DruidExactCardinalityTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java index 12c622d1e011..fd9050261c04 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java @@ -19,19 +19,16 @@ package org.apache.druid.testsEx.DruidExactCardinality; -import com.google.inject.Inject; import org.apache.druid.indexing.kafka.KafkaConsumerConfigs; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.testing.IntegrationTestingConfig; -import org.apache.druid.testing.utils.DataLoaderHelper; import org.apache.druid.testing.utils.ITRetryUtil; import org.apache.druid.testing.utils.KafkaAdminClient; import org.apache.druid.testing.utils.KafkaEventWriter; import org.apache.druid.testing.utils.KafkaUtil; -import org.apache.druid.testing.utils.MsqTestQueryHelper; import org.apache.druid.testing.utils.StreamEventWriter; import org.apache.druid.testsEx.indexer.AbstractIndexerTest; import org.joda.time.Interval; From b897b861a347a4e085f2a4d1528911a2e0bce61c Mon Sep 17 00:00:00 2001 From: GWphua Date: Wed, 28 May 2025 11:31:26 +0800 Subject: [PATCH 28/65] Change groupId to contrib --- .../druid-exact-cardinality-34.0.0-SNAPSHOT.jar | Bin 38784 -> 0 bytes .../druid-exact-cardinality/pom.xml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 apache-druid-34.0.0-SNAPSHOT/extensions/druid-exact-cardinality/druid-exact-cardinality-34.0.0-SNAPSHOT.jar diff --git a/apache-druid-34.0.0-SNAPSHOT/extensions/druid-exact-cardinality/druid-exact-cardinality-34.0.0-SNAPSHOT.jar b/apache-druid-34.0.0-SNAPSHOT/extensions/druid-exact-cardinality/druid-exact-cardinality-34.0.0-SNAPSHOT.jar deleted file mode 100644 index 34bb42a227192dd93d936be08d89d49f7a3ec209..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 38784 zcmce-1C*>$vZz_MZQJ%K+qP}nt~zDA>XdDpr)=9gWm{9XyXW5SzW4Q-H#7Zmtz4=2 zBX`D**ctKd*z!`qAW#4R5C8z+bqdM=e|LlYel8=TEI=zED@rf_Er#+f2JsiMJCFTt zc);(MKmY*W@BWvVjDW0!sECp>os4L7{I*;W1H$mO#54Y-f{`TH0RC?DB!*5F4K-tT z&c!aBAYqa8)AqRO5Ydf+oXsWgh0p5+5-j=x*NtTJ;Tj1781iD>lvQ z#n8PzL-Z5We(`LR48b-+A#AyCyvuI)3=+>m_AL0sQ%-qtVMG-7w0`iH0tA_0cOmsl z+ToY~@d*s(LtChQyoNGqUuUz74ZQd-O+L!*cI4?clvx{Z51yn^pL-}KNIQS?#UrxN zYc7b`-i+pVEAHxWFU#aZH?Ahf>Q}v?a?a$?;n$q24}kwL;QwrAq`x=3qizx6f|~pu`vD@;^_aAxPyy{qsPCvi1>ef zk%5_+qluY;vxS}QzfeH=|4f02yMdAOzfi;azoW*;z|q*k*1+1r`Ct7s-v75M4K18) z4D8uh{>5+w|3B+?a6(3!Y9o7g&i zPvyVzU7`OyMSsyu`R6)=7TQS)8iiG<2Umz?ZFCr@}A}b^zqEr<# z3k$>uBlN{Bg62Qv{~Lk^K@%UnkW2+jHt^(xkP+~C+PC>2>*TsYamv|YiI5PxG+&6i z5dv&gux>2((pBsyB=DB7f`x+>vf(fGt=t7OeP5&G<%8wpW&7|GhpUfWYc3Z$>Obf3 zWhVvf|F~Q>HtvPmd>dSs1^@uz?*}L?AtWNJB(kZg<+MJ8>ibl);#jO?0al#NVu5}9 zv+f|YO2|UE&6X@NjI@@Z1*~?2jv(UW;YVs$K%ueQtEx;ftHk+g!|`qQRn{Q%Ou zwC}3Z8_A(i%lOrLc3=mux{X1}L!zDT5*%=xM=gF8e{^O!QHN&ZfLOOTA)&cyd5)|T zD)HNx5LZjw|3oqh11Gsijw6+T)bz*P(V!pd1=ba)!m)z|!>zP{BCLYK$5L$Xo!@CO zQtPXqYpgoyB>Bo+IWrh`bRTGQH={Y3jcsN}U>q%<-R%=&rZ*84bJ!EqyH#vZYX*4I zxJIJNhGezPI8xF6fTw05uNduq1Mk#5-63>kbomO=`uhqpW6DPrRy94cU!)&&kI&mbl_ zVA~fkhyb( zgWGih^F!_MwPPb8M-k9j*aoqHE)1W1@O#aJYvVn4Ymow`_lET+Xr6s}`#Ib@ZmW?s z1)rsr=5yV~C|n~PB#^=^r?+CLf4&{oR~?#St%tZaEOvNdDC+K(p zGyfWL@Tpm7+1K7b7xHk&wiQ&ggZ6*!jQm(0=OHQ1cctdi>>-G0i~KM984ZDIPu}Y^ zH2~vy#5YG|kZ<`0ihyQOfNPU@lZ=0d_}l7(xpVYmYeC^H8LM)_sB4V~3TV;q0m0?s znXC^?)gt8L_Gh8*B}R6u)!q@4C>cGbjzX!8or2fy1tFOuuye-%Z0vW!zMpiFGz-Xk z6XSB#6ew5P!$O53@BcVxZ3qr?KsE+*d7|E_!Kd|=4vx)SfK!BTBf z9DY~sYDA$lN9Pi)sI-KO1K%WP+5pQ3-7ie6YKM_s1`uveAw~U3R1rU0N-mk_l2n$s zyiJt!hUK@kKg@}s#I{b4HAAjlTBHDv!gd2WXn&Y49%sD5)(GY6C;37o6#=NQ-Mb{? zZ^Ld2z|G?hvL!Gpl9A}ii)v2Xe}k-v*eQK@xoR1Yj4EuhhPUri(3{4Zwr6pBYqF?2 zv6pVDi?;j*FQI!P z2_s!tz|^J@+kV5*S^3anIOTe7ek3;7y^N)7SU>ngLx`teLAz#rHVQ7GCHWCq@R!Uv zg*@HieKYZSn7lB4yuq|6QQ)FA=wk}e^y14oOGxaq3yhZaagJ_lHw^ zJ$Cq;WUJ+nh0l;N{xA`DI?1UY&2Kh#*0y16HAIswT?*?f-=iR0 z9F!3km&uHls(>x&r!>Ja_EV&CEm2GJP+O`o1COYM=z-ijCJ(&`@ssTWFkX{_79Eg& zU9R^Yh*;?}POpND{$RFUQ(_hq=P^PI>iaB`5-6keU~xPKg_60bPjRp)@nIUhM}Ufh zOD~GZRfu{#9YQlyH%uRT74%%KTqOoncX}juvRvpD?<)fnw)}NoP-Qee*$fxHaod=0A z2o_ok=!^Tlz#HYe;Sidf57C>1pV>Idias2dB;6%USdSaVUbj1B3Kr)DwSLEWF@p$! zj{@{naDGEc#k z+22UWN*SF#X5Mh!48rwFOsC#u4g$1J zI1YmFh*)bz_g=x;Y+E8H&#mUE(rmGsE!4y2x&0GGtuUi7%TrS1N2QE7RcNnTb4$$LA`|kI5>= zGbm6rdXqwGfKK4p=D=wfu}({luFJvEAuW#O#)EfEJGHqzx$~RP@W>FcGzA$$_94-f?mXb%C68BYgOa4LY(&6PMf^ajmQ|OlSryTBxv)ZLxe?^ujK-HkbPvGR4`E62*6Z z?}5b?Sh9KJ;scP&U|yd?WCmaK{`gNPdWUMzI6ke)LB}7+ zd^AhrE)18$haa)*_?LsjgPWU%EXp7|etOl_{+H(6fWE&^N~OqW zgQ->6$Yd~uofI+cPXT1;(j%~lyrAmzC*BS}-p+t3yFJ^qyE?S0KRrL@A5Oaw_;j|r zJ`fwV-)p^lI=|8d|7_m+x^x{agQMvVUF^Z(Jnnw%$!_?zc64dzIW--w_k-D?@qgj5 zEjpj^vomj?$_{|?n910&&N*?}!_J1p%D>h~O6By1BO4UCYTEb{N;jRV?g2XU%a-7WI+*w3Of9l=uq-!rQ*b5? zL@psdrnBX`xxQXeJI$Vr8zbi7(CD<;{zu?Xps9Q8EW5Yf&fq&8?GwMKJ?CylA{l$c$kvy3Y? z5@$;fS{`Bb7ghjEAH|83H2G#eiLhK^J`K#fG=>+M4l}P&zf*Uj&8c~&AHI)9) zV)|@bBQ2oJ%b5r~XhgQwa()%%}%{VX_9J;IkN0wGmm>h$u zv@P$~R6Tha2p-(4{jS6QE-RerLc*}dLpZxiPK-H|J$mrxHwxnD}QRPAoB9{+Sp?M}w%+(P7 z6d{PG=`P1tyw*2MpARJEVp6^SnO}Q&pV`bPiO~zP+xGoZLGJ-27|&ngJ-|0)4OPh* zr3isp=CHqEaH`g05bXIOb7`E1?H*q&B=*iO@3rsHzX1QSG#98Wk9Yen$&0`5|A0hf z<&?jRbDs$TxPAfz{@EWTX+?CwDMcNk`=D#k2&Ob7utNHA?q1Qw>(yW5zv}M-c62p` zUH0GU9AJT5F@_^?zY8v{626QV%1zWj@Rnd%BJ!dX1nPtSl}>(XUn_CcQ`vakMh7ww z^3!L~5`u)9q5G|-H^xa5R2W2%7*pCWe*BRR8vFoZwrnV1MzxYk$E`t>bOf)7+gu>B9tS<|Oe4oADFgfB@N?s< z__twS+Y}=8dOXq=%eb*d};O2 zpx!_F6kum{#p$;%{r>hVrSHf80Lp*E#eb#1f5p0gPiX%NSpOcw{U^p0{2R{{vU9O@ zHgTjgvNmvXs*-1v9pFdEf+h<^00KqQtHu5_NHL-t}h@(s`zfUdHF5b6lN@1eFwU(nKSjY2GPc#O< z$kOC%bcdT?WQe~`q)sfhQt)74yjlI)k(>D~web7-JeDbEl@1jUcu10c^M}{bgxl21 zQC9lL=l-1<0J{B;5R5&EMojVCYyqlerWd4|g8_g%AA`B zJJnXLWVn{19_y4My<@54>5OIK-B32l&-%2eV(KR{SPC}}^QgSd8VS2UD4f}AWHliy zTyc^_Ho3@P1BbNF*|hW#5j{Yp5#_omG4_=24kzM%frL4e$q6fcwbynQ6O@;d!~jU;o`p}!@p*zEwoF=f4Hp1@E_J;Qcmx|%pCMZQC_+gz&8QlTVp z{ASpk>*QwiAK0=?f&XDSds2)>c_wqCz7#FQXx6zro^yTj8{3Jd4& z{JOj>pR{9F+4^u6Zj<6C$q;6xO}Pj#a_iWP+Hve}FtgdgsteFFQzS}U=Kec7%2=r3 z){8hg8t`3%xW^2*SMl&!5pejp(LZ(|bASBEV-i0>B<8~tLGBO3wu27&h|<3TtKaZv zKk(992>DYkPi9I(QAiv~9@xnUV+{zBSU7MoNyss-iG=8hS7e?Rbv;!Bj>Gu$m+9fT zM?mZ;SVCY@R6R~CJWu*4qyvp#vm$fx|Hu*8aKA%(VRRelbA7?3C;no<80czx1~j=1 z&iBFEgJ&~kyLlsiXMxEgnlmJsx5ZUhyq12kcxRW{d6a+Q0b#)cL)=kk@yy79F|%g_ z!usjI*9B~4pj#dIQ;gwQF!L^5!iTCR@(?(t8nny?Ns1`2(?4?e4vB8Q0TxuduSs61 z*SPf$jW)pR2gpB0`?+TB5c&63X%Hp=fX)B8X#ZQRt4C_#ETVqtsHd5v2g8C)45%mM zYoLvhWqj|Sy7cMuqbZ^>lCYiBa?>PEOj(oVxaBoM-ujGhcpYzT|2THTd)n}L+E&{Jx<-6w4yIC;f*2_ksOsgN zAN!q@m5`xoWKoe@WMl?i&Wf$Xgef&i(QJxU*SgGB^L#Uj*jg;*NX7#oHl zoE;n&os z2#F2aFEt>sJ{Kj8+rL7Rryz>V25r2Q8^k4)Kdcy$yaXe%e7?F;XljV@yPsjT3Dw~(_`CxXE z7E`%jdtxG(&`KD*RI65TR?Dv=nwQ-;3uBN%rwCnnv0{)6qM*;;12x&^yc7l4KY;SO zT~%D+}8 zZU^xpUW+k+@}17|vb{Pe4~YqMj!Dv$U|G2N1#CeXR%1}14xX6aGh)IUXy=+HfP1Vz zB9ja=N{e&Q?StTHxVE-pN)n9hok{XePwkQ~V4j=$Co;+I?mNTH)fN9?scTa#=f>*? z%Q7}qWSX<4O3!^ul?@{(Ha2FE$(Nz_@N5*8Rqn-v*kzd#pD{7UEXorm;SzvgFLOvS z-4W$FmBHl9LTx3j$jSq;Ekt=Un9M0w-BN|&O$1*)fKG-{=o{=G8Lw49D1E|(0V;Fh z0Fq3^yZVI0WX@elRWx-~WHvt*49Q;BOA&?Oq1sP$k4hsbSu)%AVBg=syTRX80v+J| zFft#+q#Kl>JFw?s%9?b z7=N-tQzO)gAA?ry&BGA?9h;4Xfyd30!QuVI?|pt5bfIi$Vp$?&o@_c- z@|BwjEfV7(NssH6CQd9B*y(sU(?<87#FkLXOsFl^hrgza&UU?Il%^W*WF>U$ZHQ{S z0+-{*8HpQBmmU3nNXEyjK-#D8-vaJTBWMv(hn6+>SGdeBqeecHpi}A=e2N*p>U8?n zHRcxN#7LczCZz*z(N`3+*6Yr3x|Z%>*bB<<-epazvbzQ~x2YQipKvkT6h}HAX;e4i zVxoF;uO!ZOQ9Dprpj9TV)ztC1RdM=uU?ij9sNT5>9-a##g?P!#PpzjUI2BETGsluP!Ti&Qb^bEQ{_UyOsKRJi# zAPF!}rJE_!%e5AX&!Xl$jGxLltNh5vu+W)oflWV37qopCGhP*>FfNJx5ZvsbW6_QX&XBwB+QLQpfeoGK7(+f zZOx|V$o5qD0kC$ma4z zIjFP+r1b&aXf0}I<8L7o_9X)S#)egt6mH34E6}r4@|(@W@MrIa?-PTU5uYJ_PfunK zF}bd>;F8L(qvS$*nNW3$>!6?0&EkV(SK)$dc4vg0;ht{o@O+Bgr{Avo@p5{n883Hv zb{yLGv0gEcM|eUxn2s3wZN+0bNu&hyZUZhU0zZLz?)f93Y@v1xN7tZHQBb@?o|bg4 zwX4Y2ZkQ~A(7CsU$P&(W;a(?qji&R8Ux_ejvh*prB~CagrU)XXJ|s@{Xs{g6MWC*! z9f#Bi;=ODKNyng4M`v}(DYtv^Cf>B`OUzgBrfaKOlUdx?oJKuC3T5{6R%j4Y1NG5S zyQdpOr6w~u*c2x7Ny&j|6dVC9DM9UCDaDaaeMa=`;;+2o|6|7XB@IvV`Q5lS3igdu z{%skXpo@jI@qZO*e=CWVt^T7n`yqtCo+5%MFbEFSy%td&3uT!sOy$i0ic}$UzwYc- zA7ksbd{N!kvsW_%A~+-5^Hvb$7TToD7@xapjOTXs<9LJH?P&c;P7h$hC@+}M7{*xU zAkGZKXK>qbe81bO%0(BF+jL|*wL1zOgXvJRBOE9KOp+|rfGM27krvPKt*ADOB0lDU z3~`n_vcr}{8*jX~(-IJbCAEayX*Gsv@Xo4Ff z#AJrRsPb_{WFrnq&~-?C^>yk8{zHkMrvqudjKIS(S-m7e7X%>$9`!95?m4{E=uo#w zKj%3H^jnEk6<(+J4YCF~f#=Cg?Q;wcdx`~7HXJbO4~LVR10rEk-LS9Hx25_S!VPyT zLx|>9q|TZl7W4hbVZzjcp0mzFbL3B;N`j$FOW~@Sq?0l`3g-jWNF5G9%tF`H07wr)GB-|1(oBbgXY9nouJHnK4jxfm|;$NQD zUrEN~MxdCV@o1uC+!@U|m+MY534MU_7}kB?$+m^VKUzd_ce2gRqNKL3OVyeJMdvdV za%_~^EdTk6y_OBws2DM{U{z-q9KuJm@Fr4maz}hc^>#=g*}fA?5mp2k?2W(M{pfuS z{EuNczJSdZ{*6gQzA=g8|EVzi8!P$0Ny)k+vKYb-V#skME4n~^0tZ82N|JSRLJWu% z!scMm;^6)MnDWvwp`nfflWX0iSvr&B%v*$n@;E6KUIP4N$!SaC)dYseAom&DY3V0U zAJ1a3x9J|gPuLy2oTGR_uE?kWWDFyFpu@Dz!K&2$Mimm9yD;(6ABo&CNO15bJDG;5 zT6q=@CQ`pTm^PSOO#a*~ZGtu=U{tIXt=*|h^aHCDgz#T5M$t;>g~DT2qo370TiLd6 zxJ^vTOKfZ4WKpu#W0rJ5=|3`5?3HeEMLwZ8t4;6DuK!F!vwtvhhwfKa<09~!Fq>Ugib6)MyrIrxKY>cO(YUoUo`zTgj2I;jkE&eT}EgSgD35NWj~8; zXq~#=G&kmhJ|98dr(l%E)y&{2OJin;#fE9i)qw3xp?1YgiFM-u8y6y~2$0c~TP~ts zBDT{irR~0$lg`6a->|VKec(u+2&Om5?8866Fn~U#`DlHD3UVwXnACOB^~8J0yB8K} zCME6MdlSGg6Ou?hQ{73I*@6pMd(gf^ZE;;oBERBFm$T4T5p+VnP7cz?)@ z4QDE;f)^BkWb7-`>yOgrl&vVg!~O zU|%XF^F48Hhy~*|Q=pJWA+a6N7XJz&^`UJLM#q%$l_%G!+>WKb-gy2AaE@Z$JbKIG z2RAqS7uq>&I%V4Z#IS*gBh4i9OtHEB zFJJhzgLT#`f*=u^S^m52+!ip2UoW2}bWiF$Dls0>Q;z-P*Qj`8d8|NloM@kdmxCMU zb{sF@Q^nV4iF{9lw7SJK@G1De7X)2QO-&sC`A>RK-9iax8TCi323BYafjCi)xH)t}=wub>IFh~w zgFhydzYRzP3c2dEG;`yh6TGKD;jESVYz_sq;Z-9uCsk5>MN*Z)QU;Yez7rv;F9N?o zjZ3dh7GjLjIzg3)^2_C#s~=Zc9i7Vq+g~4j+WgHb2Hn$0tgdM(~{+<<4{C%DzGJsbJMFY>7vV~-h(w|`2`CBqvJ@=6Y8*K zV4zcUlo#qVek~^jUYvj)lFButqBN;DHszn5rj(hpFLqWOsC49JmJ7H`%DPEezF5~V zp}>f?N_cJ>j?72*$FZ}QtOfxlm@t#vrWhdDjk{tznzb0OO>6i{ib6j!EO2LB4cN1F z{H!iOl{%<^DT_&InlM~r>P+CQz{+96eSRQ^u=y3G>Yb5CYG+kmA!ErtXQho1LHR4P zOlxc=q2guYjTtv{*{krnkapUcRhH!EwD=1BKEN8rqq0WQeG0AE@d!~F=A~0OH;$=0 zjYv{C;~`Y+*fWhZY*I^ig3cUsEVaOn2KG$rZrj8hgvHdV&7mflp!zrK&r~su^jGpqE#K%qgcxE@Q0FGj6@_33>34Ivi{wY=`Xh7l+2kNz7%+ zsw{IXotU)u>lu=RK8CdUXpXNyYL7m!(vp^V==8r`{r1m>2&IB!M+GLL1BJW0;u3U zt7P7%A~C{Q*l`Qz0#^kVLp_wKQ#59L(q==g#Fc{?COSpIosDPGRi?wzH$akKSh0Vy z30b~zxyWO^eK)MtX?)*ZiQN`C-eFE^{h{^{a)$OI^kjszLt#L%1^Sx8M$cow5u$u+ z38owdPlPmiRQW2*GLc6ry4cm|2N~>=X0wH~UB83x$C)tPFba2s>|~Ybtc6c&aO#+q z$@}eBox#b$2q+gZDV@bdhy0&)yV4?U0h{gjyd~cVgIZ5tE?x=w6(CSD^}{@;#t#yQ z^kyFJn&?v=$r3(=Ox~&VS+mzTleQSEibKfB>=r0lW;&L^8)RSKFXf?i_XgC?AVuyVg94{zC z$_@JFvR+H|Xq}EJonTn9FLmYwBqQgPG@ycFfdG=-?Hva1dq6g|M6Qah87S3cA$4a%slX$3 z08DieThz2>SNj`b*z&*1_+^4ieeqiU0T{K0ECUUH$!n6fPrH&8oV4 zLF*aidZwaHoYNM~y;v2oR6hJ<{v+;do%Jmh@AlZcZSD%X0)Hp$db&VwW+V36=$!_m zOQ>M`r;h*b(=+jB@PXeyb&a4e9_@2}BLpB&005VNZ;m2lXJc<|;x1$2>}X-6Wa4OS z@}C>M$})EI{0Kgo!DNLNl(c$9@=%n7wD{D8Kw$(VSWWqg5Q>40+bh&&)L!g5HH05( z_Ja01;MWBRM$48`19{q>*AwT@UFoZsv3?()H^{xzJmvZRhL}ZIbOxaY272B|JSbc{ z_@QrePB6?h$*PyILUnD7X(FJhI{(B!Ri`@e6xlpiO| zpb*8Ek&Rh0vJ*jk%#z_Wv9lXEwM-_M)5=!Be>R3`g9Ic8*pHdhk@q`oATOzD29M$1 z<~hb=^la>{x78&Iu91gd2#5Ht3-8OzZb?z~ok^XIKOCFTFm$F#VvQ3dti_X|d_}f0 zJER0+kIP$P>oIAcP|$Wrup~VeV1yoMoMXrZ0~@G#PGys_D>4oU90%5{7^bC)iAeR} zMx;fkTjt>LACaijhQSJ_$@gU+IhTEiW-wP<3eKE&C!B_NV%<8XVhLk>9OSe$oDRFb=?aAj3}S)VnqFh%={dFA~q zL-Ay1^2I!ZnGJzU!&* z@F6Ym7fE1FcS4mSu@VE&9&nk1VYg={=bIcI9PCeJUU;Rnyh-zf9uJl*>o|!Gb3VLD zyrMw2Pw7d&S&m`gwaA6O+Lb#{6~AK=R~8SzUsi4%zm4t7NHq6e8pu^_dO2~l>L@H#Q)ESt)F z$(dsbU=S4vM=VDCPs)++7Q6;Tje-FmCPYZ*X6NFIW_~Y^!`v`BPdXlapXbzKlad9A zdve;YzO$RHJN+DD~KwI`=ihF4N=l@rTMu)@E+`1?lN<_-q790Ia&NT%;vjT>#`Tx{X|+{VgS|qqB73NEg1)kWUDU_PXl&sR(ywC zGPdX~(@=cnCV(IGX?{+DA#(PyRGtoVsxl^yW+wk!uvMx2wc{mJHzkEg-4;l2(SmAYx@7|k751@^W)Zg5LoH_1*O_6oYUp$M4KERjk`Eo3R) zP!r}x)nEbRLeAta`39oK1O{AIBTU=T35NkxEnjcVKG@KUuYxuU2QBs{to5kFjF{R2 zb20u?2j)?+*St^#3C2{KSZ=+O3YF@sqf()4)@yeVsJc)N z{6;OxN+@EKmNSI@Lp6($`U|o?@xIPNVC>F09+ylHVxQ$#Zh9zy2EW`X1`Vq(2&qfN zW44hqPTpM{8c>OL z9KvJPJnSeT-^Nw*P3$}P0~A9J=>qTcB2v3Z)qi+9C6BXk z>18(Y>=@7G7-k(yR#8sP+n4Q0B{m;qv99*GgshKOm#_xr_{?m<(!+pU;k5G8e#3U6 z55HM>=MvtQd^6Uy)XdoZ*3s*SU}LZ77+Rze#))6*87O7y(ngRi7zw%!dz`ddI1D^k zHD96Mh?QD5s4=HPLe1JWODE6uxmt$1fpz-8HJiF@dDnDeJJANrEzKyfMceEIY}w0I zoes5dwdYIim#elOS3cc(0+Oza&NO^|ULiE=yi*jJ-J2jTNApcq=9|q};!{=>d1Ez2 zUK`}|kV7qPb$C!|8<@nwJ%d)l=L5xD9utgZkjs0er**&pgZrWmIyk-nRu&KsA=nF_*jDMN07n zbrCTG1$}lHa5_%xvuP}M*tQ122lTc_dVv&GnbZ@t$~~f7@Z=tg-DgJ5>>0iK%#m&9 zv_)Y=P|OP6ZReMO{mePuerhcVq$S>wanlRygn_R^3cU;|Gef)MBYdHr5~;o6ru#vR z>qH-v)*WLat>o*2o1PN`!#hYUm?er-L6PW2YJ73&4MuAOdq!TbJ6~xBU!MZ8zuTkl z6E!WgB&c4z0-xr`DPJ-c&o$`jQXUhbgAr8~6E3p@*$rWY9>~t`$o2+(%V+qh0 z1J?xLywCcNRJJ@AI|~II0H7B6-$`ZvvVi=rsO-8E@-pTZuGG@l8L28RflBQFDO**ji=rA>q5&^Qaad>LGfW(_Q`$E ziB-a=o2!~tkV$NoJK28vV#~4j@zbX7F}K;r_B8J&xqsqwVgy|wN+GK=@jzV1l%4(N zfvV823TfO8adIwv3FmB2e;t#p#8G*Q1 zcmntX2^Vb6m)EEHGgh|x3-{>2yL4xrv-ToEZ=p6Pdt^1Qp2;1o1AAno?p ze3;Pa#FNWZqA_>bejCfzKtE|m8F#>FF2d31wJOcw4dmR0cFT(M^%Upf$1)lJFi>nn z+O_w{9>_6gv8duT$rPB^+%Y9MGWvi7?ZBY(Ys2`}&}O9`ty-$2072zSRFsgYw;VUT z4?|V6NVo1_92r8vGG6;IYQiTwC~VR7^qoW0!6OqxcVYDA29mR!5AuczX^p}O$q7Bv zl3mb5CE*_}2NzBN!pa$jV~CEyfJ2`(tK^Cdr>xa_%h79m zx)(|-E7jwshi+;`Gz$R;JQ z@)hl7*6%y3$@J-Hoq#Pr~>xtvOG4cqcMH&BrP(4&etIzUFL@(<&Z8{xXwd*E0q7La6jM~P{zKsfm#lg zn7gYF#hgg!0g-#5N#Z)VCk-XWb;yV?Ak}e`eZ2MvHNv$w@R>eOXU9T&0#02UJJUM# zHET}yHY3OQY<3vuoo_x&=Dq-2zpA=_GS9^AS`A>V=bzLct5&Fg8w({h4KfvJP4yTo zjQ_0HIXnWw1;71)C;b)~Iy|#M_QcI4L0P*A|5|_E_}0Ei$N#&Or!~gF zHZyUg)@bW^k#r5sK_wE~q&_hKZs}nc)8v8W6L@{ow(c*6MloJ;Z zwOLGFLoH2MCS9OQo~p?X>H>_E((?qPJ4v~F8Hv8C`(Z}AP?Bk~(YR=hDhNv|iET`M zI6I8#@m6i6_>Xke@4Ui{xjQkLt;&VxnbQ!*KM(_Lk8Sv?l4+-%xUDLTHX|9ac4db+ zDMKg7#&gX+({63d#P1>^Qz&oG!%}XkJ_Td5Xhp~cv{$< zs_GN^%{13CUdNK2zU5&#_D7w`RD*YyK%Wxuw|NNF^4Cpw#Pwf9 z2%phtrW7!Id@l&C&v2aMzJIS9nD(vVa6NG@p*ZE{zx9!ZeCxgohd)!@zl^xV-0$rLM*9S;656 zDEVJY^q&MJ|5<&4gfbe|VA`=x(YRZi<4T1$T_hamCklSv=UhwK*k;6Dz7fyUlkEqq zjqAyD$%4L}dRQjk)JZl2rLdy~W{76Y?bz|@u52|qg0>yWqA7xAqLQeX|_f@2Rka zx}2fJi;uq70qUFeXk#iq0$<3ma}-O?Cf^b?wh96jnBQ_T1g>RAjGHx4t?<0$-O1c% zyQ@>rgS1$6m2RShED3EV2HiAus|On6c61|I@cAFv*RgAdFiC&tKX>Ui*oJN@Cm%On zq~XglBKAz+W~i`(2lkhq#E$z|mG@!@)M6dEc-J3+>XVsQLAH!8ViJxh3M8{EkVUpI~tReL?^`M28^UaMxuMzi{dfTv|Sd(v!>$0hJK zY8X_p%82{Fd~wEpYRl6XWr7|g0xWnqBBww4N!yuC|NXSrwxFo_$uhP{-?GvveOfUapFI(_%xdHOCWtia4zMZy29wJ%WJb95M_n@!Yl1+=o%L_YL zkU^*$JV3F=JgU1P+3Acml%r>)7odFhes?ZkcssM>yyYPRy^+BEb-~wvGiLu0d{sGi znG-~Mg|3up6U0gM1KSGAS(#z}93tvfBayp-l0a2im?Is3$ZW-2M~rCcwfS`F@<1ay z>5IeK$?X>9lQ6{b4XLrstWf`u?fZ!2+0)C}cxKm2=UW`s~Oif#xCm7z|Jav^YFM{a`czdPwg!Gh8+ z`FDV1Ft?WmPEpQ#+(}F?6$pJL#Ry&<&% zSD$;?YV1SV)O>o67!*3I(H5h&n8jQ;^|u;SpnfGrWpsLNHMbKH@?wTADq4W^auY z8r?k`tFg>1nS0=hn?){Y2pgU{tz$KSDPH2C{>ylRkZjC|<=nT51a~blJC7uDj?i?m zVVsP7q-F8A!pTbDqXfN2W{S`?P1I0zZglCY#=ycP{X5dg!}w_=0hKaV`!Vh1nMx(| z5-;E*ilfn2HBky+rbAb=d-ckr9N;1lg!EtFdwEGAeuMj)T;6$|^EOuPX9v+?1sp5*u2wGm*Zr zGqR}>)j6g`J>**&U~-RaD0mugIB-?^^i=Ohc~tqRO@#H2Ed%)A4^lE+;vP{j;8m!x z_j?T?Zf4~r)X6bbvQhVy1jiv=)gN}M3Ho|yhuRB|(8@dnMBHcL^8@M379LhfGF?#h zRV9jngE8s1s_q_ASH42gdZicXq&uTjMl683TYr8Qg+1PwveEEtM)mU^$M{erThxF( zYm2y7ZW_4`{p06{ZF1(jpD+d=upyV4KL=M$Z{C)x$^@^(pC*-?7M~_rG*uh4@3Z4{ zM3GqMVVXXCY_rZeXNj9fI9``>KM)rlJQF~5uR-*395^n{418wgO-}H4YJ}DUsjGLd z$)wW4T$rVkDi2Nd3XP`91pUI~>RKQzF0q9-TgThGoL$}c5+D%SPP1^?xVBf>E#sO^ z9Qvp=M1_T2e8Ig=y5XAnIcf!vuq^a5x_wvRLS{$PMO@TkjhCVDxl@KJ?U3oUBbq%O`mf(XT0INSOVs9g-2RseSdU;hQ&vrbQP{V`!!e#x^} zpv?t#_o?>>k+Y=u>y%bW@;)`7_}9B!s=4Xnsv1yGB}Y+4+QFa-ArN+SCF7_ zdj;Nz?buD(AcHn!K$nVB1eNuR{IzD^)`u^mi}Yn#$n3qOqeqPd%J_JpNPUTJ=!y@w z-#J>itU=VyG-h_Apl*-Z$xKBlusWbn=>yTk=q?H^CQA;Up%hF6*YU}g)=>t@zd)1= zQb6ycV@)8BYK60MBNwRi=GIgzb$ys+ccor7reiJ zSiXO5;hMcEmX6O1*3nJtNnVEZc;UO{5TjR-I$6RKzK$>3xy z+aCS6DVhOk2PVNWj!N(g4V@Z|Be-7>lt3YNRx(+6-USiaEBQ~PDuP_FOwnzHRl|eb zBxE1sC9WdfN!_Dyu-C|MNKl?ddgPHB%~DpRy>K8ibhTOAvOEQ=hZPxR?}FSC!k>jAv!sD^IQ70XA~}kA6MgEDIJK6QN4lt9D!XGMf~#AIFO> znM%r|8v>NFPGr*r$DE8+-|n;Fdm8uQ5rZ6+W`g8N&NL>h+cWxE9Gj&1 zOZ2jGmj_WYf6S4cSAv;484J2YIFtc93|puw4qbiTG^RlBt%6|5NtKLXX@?_$jdCl3 zu-PWyM_^jpZQR9Wn%=(Bk*9fu=o6IC{IMl+SAyfnXSHAhCLm*GK-JX3-V|prr7>H~ zKQe<2-i(>hf*8m5`ZK?`rAg2Oyzy$V!DYcw(u$!4`Azo?!;cI z#BlK4I>c~zNh%Nb69(~5(HI%h+N`lYcZ4X4Gdhc;6SxW*v{rql?<`K;1htGwqJM!N zIas-ZuZQl~2RL_n_3e!8hlhS-^V$GF4adxkbri!5E2t91Sm2SLkmv$0w4PBqwLc5+iL=PEPf-WSeQB(GmNUDlH1^g+0Jm2N&35}^ z2(4_f^ahT52<7gjzE<0&TxFJiyyuUSroz}mc|%am*>haIs0(A*njBbcz0igAvLC$d zulRXn()zx<`?R9n!Zjh;m^W9VWo3(%tIEWjD7+H1~kq;q!RMfhnnLJTe% zZJwd6GHU7BPug1wSYEfA?6hc9TWZ(0V-uLT5p}wG%&9syTYNJf^^xW>Vr{q@XV9&OyU$1TR;QM|!QfVD~W-$pXS$)Wwo2aU>(CELTGW zR-0(fX_0KiMFR?!l$hHA-b;cx2UUgX=CGL!<# zT@P$v%c2xaKg0rUDz^KRzZ*AUQGvwcl)((F%c`0CW^(|J)Z`?!B(*g4I1Rm{;zGfR znh?Qh0&@$6SE&Gto~f(C5Un+xX4!6k&hEUwZZmv(=^DsLe3C11m|T0|W}`Ibp;b`t z0FsvPn7g1tieinj8XSTWv$L>}l0+zq`~qVUL8x6&k=KXwb0!95g`3nm^0u^KLOMK^ zW38a`$+qF^v(z{~TK?jMD8@AtFq34_7==4!^o2?A)7?B#^Auw=&JPVlU*oE|^1HdL z6^}_1oFNEr_t3{s9CqcUn`e4hZCX}fu~Sk4Mm&rpTlZ5 zkPSCCBo^c*H%O*?7_~(Ai$!1dvo-nn(H=W=a<5p7#;dOEKJ$I0FR*Aj!jw-gZ-=TV zM8l}Vne<^AV|gCsKm{Km4>htF!|yAw;>gMq`|usYc4Y%p*q}<2a+KE(ntMwq=)s{X zWpmk17?QC^+)o=jAH_CkH*`-+B+?EgIkE98giRkAR=>hrcPYnDK?w6>*Up1iOiv91 z{TE!eD4ox5ge}D)iN7~VoFM-26^YuEUkC>TdjA3biTYG%+M(Y4yhbAgZWNvBL& zqg)whl4~;eEK#j*JO{lU=S=WdpYE(ZU6DqD+lPPzsujDL1)e)r#))s=KDE zrg?h9bf!qk>>t_Vz7%xu%Y0~UBz_4%@q}NK;vBMo@;KnxSrk;!jj$&s+twN=%|~$R zE=4YP@kCMw4NY=Zzddy@P+prN4O02M+pwT_Y6$|^YE0PLJOiOg3eb9k{~jaQ4tTGj=0OuuaRLdIpmwxPC8M0HWisXP~O#Ck4gfcklDn; zJkLoaxs}50QfiVl$ww#fzElOi&ytkPuI|8lyE8m}qnZEf=1*B02gm=G5u|eHGN*|2 z+Hoj3OJtSHPeu-*YSb#K2`~Uym?>|XNljm<4tF{J)gX;I9!dRr$>-n;B}jZzSvVzl0fN3F zA@5iV5Eh4JM|U|x&7eVn*3vc+G>okdhjCN>IzJTT%C{P-q|8}YUlWvoq=s;JdT07qvp(cDeHr?+N6c=oHSTV#gV}N2J%F53n?{NF07xcErUu% zuKjWu)EcRGms?XY6K?Cv4`K!xvljznGx3!^rc*R-Q8aC!Sjtr^386O9t;W_|sZZ3} z2Thr#kSH)D=}l!dB-mmHp(t=V_;Enq48lccizz!WEFFi3Rau+*606l?H<33>4V*6Z z*w(`YLDXwmEiHTxY`6U(`Qwa3Bsvm0TYMr*LQ-Y|DjO>+lY3+eS)E;P7jiXe)W5PB zkpuZez6R5Uu0Ao5*v(fnCElZjq;pfb7yqLRgX9_(EJ;P)Aw(N8*HS)yly;DFEzynx zO((JXRP=|BcV$bEk&dInL{tnzIhs4da+295%|zVrjcVS+XXise5IBl8uutfZ3*XK{ zG__Nz93AOw5bUn#+B9=E!ibBDqZptFkFy1#xC9xPbxF6y5UyTIRB_0Tgq4>dVCbG4seFN7M(rO95B z#z9HZOG2Ho&g|-G$%vMv(-!JuFesMx2ja7c+F4Wr@O<}+7bQzkvV(AdN*`<4ET8H> z05qZ_QW-bIRk;A}dv;>|!tQ@JCjyL)ETyUjF&I@~9agH(O?9G{#oWGZV$@7GWTn*! z+FR0=6F_%72!H!OAnU-{%8xQXhUz?S+4^#M1*q%#u^)*i5 z_$n@rneQB216a3>qQ|(zPwoNH`D2+f?oosla=L^!q^%t} zy#)|0&Lr_>Jd;Oab<(p#Y%~*}?K!R4Kc>lOE|A(`qS{`SE8JS>JeSk1?xFjTVTCFP z^~DIzEEZJ^tL|v>V{4sHy=w7;)g%UUart=l9;+6#ccUe*sNc3fcw?`%^Ic%Bwl0$0 ziy+D!t6PX*)wN>`mlm5zv?yW>FMmo%Hzc|XC;NfCJvUxhk(UW?(E4*(M`vV(f<@Z} zJ`SFUAer(^+UB#2)nUlrN;|H53v5vCFge=5>t*>DZyD_13f@BJLr<)FeS?Ypp-Q$uP6e;Ie%b3#+ zi00jYL~88Q3i@FRi@Ol{95tIktZ-}w=5H{RwK%1gZ9tlze5n-o^7|5@b#o2(p>Nyq zJb{OCZr*X!2i(1kFj@}>npMi?`idqr3)_pw|HqG4;6j@Xl6o(hJg^DZ**r&|n6sy; zh-cH{i9@ltlbhmA$}GTkxpGh{5&K-QiRtE8LYdk4*#;@~gd&RV7;Cd7zGzBdshC^S z8)80vFP^;9wm9U-EZZrKZ#qOM7441UX<2i$pHKm?3B$Q8@_ck>J^ceDp8Y1whd1GN zmW@$3u`>4pFez`M;j7An+EXj;pN^~0%JWpOIH3F#(|%L)f|?kV+Y$u za}e(ejgw^w>r0kr!Y*j<61roZQVrIIY&D$2yNoJMF=oRY>3N(H4JX2ffr)r2d9`qg9PCuvp>(PL*&^9dez_i;s<>~!O-Yr@J%6cS9s3z zX*ps(ac$Dr3;am!Uek?UaQ*~#4eue}2dO8YL4q0Va@O{+&Dv-0_G7^de5MVF8&ryB2v;mvKxODHaQtR8xw0;}ITgTs^mB`^(E8JfFI`}^$lK43 zUXiMO;;%}KQlms52kHYmDof<3m>0LaHsUuLf|$zU{ij730QfAzH{^oisMFI@#^>D> z2@}b70a?3fnB=kSf>(g(4gPS{@sxflV#`4(GZQa-(SAkw^q;l+j(vAU{3VaWo_#}- z&BV>QF$h-;D3mxg!7rzfDKQ2xvlh~|4TqO)2fjetY+4~OPLFvwcKFM*W9i)^Wp7JZ zzQX?g=I}aMEt&c47D5yT06_geKIq8X=-HcDoBSasV1x3Z3Ze?qD;L5@Z%GKLA7l-6 zKu*-BEk6mdfH%^9{+bUB+;Mp5qQ+Z>e!6#F>!?pPue_7f)Kdo1$uiGVu+Q=C?(2r4 zeqvnerz0a@+waag&ezt@*Phn0eb%3lyXjVM^FgR#R}mI+lXAVIXl&UAegtqbGZYU} z1}!V3RjHYvY4+TT!L~yMLo?u|gRn4h(dDa|WyLB4Wx@G8$m_KhdnC}9f^)~qi?|NL z!fJBQ@m=Y51VAqh)8a*3Kt&TNv03lcH+vEkr%BMu5K1!jP>oL+ImHf9vJH+m7hNY9 zQ;h_ky0@y_fpct!#LiO96WTx`KxvKUyMH_GAr8W3228^XPNr;_WPF&3$r@4HPAa{s z$NjNHnPYCKikblf$=gWJSb3DOjiN))`Eez{m7%XOr;q|Z@F3|z zKN1ZZ6EiY4qUmz-RwkJ|8?8MujUQcvHuUEq~}4>lGxbks`L3?F2vkX)(lL&}> z&Y{Jyh^@Sk8w3G|QlwF#jBa8=&Tugh9kolwN~$csBq|@ANwE<9FwUlkJO{vt+rw&H z72A1#3xuqpAM^OrnWVif-_&9a&(e!b>BjSbNRT| zT$wFQ^y2h(E*}kr{2fIiSbOT^rX8Iale6T;!8c*$zM#=%r4;}o8<@;k5iyO9kBm}Z z0Jkz3Rm&uF0z@&V)^7gHW-dcMH4-&hD&#Ebi8qWCmLE^dY$ELjOQsS}JQK4xHioE~ zlMF^*h}Pg_mZ4hwFe~IFf)B}=m=B*n^Nn}6v1o3!PB_f46PlYC<__qhxmg+Th&~=^ ze+jXP0@>WYVJ^*Yo+1K^*4~lBtg^sC%3t>)E37aBGs0(eIF-(c7tzx4OBO|N09Pwm zAaviTy5c@TSt8?>8qJ!?!Bh0maxQNdYmzj`(xAHM!H~%PlD|xU>;+lhnmoDLqaWQD z_R4l!aVR+FAhFU*DYtN2V8~8rd#)_`IuGQ&lu4tq47=AN)|v#jC4!n6!GL1)?>qaGE-7+93T87i>ZSPhY6r66D~jehLFsCf{()Pknuy+c3= znMh&1<^#wQ*vij=R~LdR8&BXbZHy}ZZ0^>dz+*nLEQ|7lxM7HW#4d6b>72ylyBI?H)0@BZ}3ry zAgV7P6qYidD7y8a2l=Em-!qyE8``Q$N475FmUG8+h@Sm?lk~LH53%dpT9X`Q+ekhe z2RklT+ksnv?Gf?Us61pYLOZNeRz4k)X$hW9I~Jki{COzrm`sqcZo!2?H~-VNa7J}K zJd<DmL5v7bu>bPR@Ymk2E#e@3yd%(YBm zr@17;yZMT?hyq9^xGk#QRKb7QP^xV;wWtAk^o0Jxv9e06gtw+9EIPvbX=mX8i+ak3I-EA$Eag0bMWc`H)j5RhsZohkK+-;A%6uhCM3)3A`hJx3g9Ya!rE zE|F7uIPDxgk}!PrDq`oG%$;u-*lfLm z0!{MFgd`rJTN5+cht?<;CEkq(`XBmCFeNXcabY{@${qMWic02&8PvkOEY6(t_q0ysQJc{)d07es#?n$rIb1yGR0n&s zaAq`Cs=}~rCEe~i7!5J_#Pz6GTW+y)p@A>yAbjl{yy*}}AsG=Zg4F5hjEU=JdW>?+ zWT=O>y#ulZViBW%+LBH9{xiyORQT(-^M)hsTF)E+!K{>Pa+bxHaD}Dl(@oV=6l<3n zt<~ISL6z&6Vg$&7VpA*TyM(Lp_@O-yLGJKas;@8I2dkc-s4vR)jmc%OK_6!Z<9VqU zUr9m$A?;xA+OB$Re!po8J0YWhga!Z@C;8tX9{rnG+&_10@2Q3r>@430SWeYFbQR{& zU%k9+S8i_k+|YSRArQpzKDS%KNeMtsGgnGcHA;T7vd5&BvhS0U50}nNx0YscIt#sD z%{bW&D#;xdBz9mgc1ZV-xpO`r@4%fH>U4-_7E5n58+dN;NpqOwT6gIjI`_zMc$%|- z#RaPNZAU@Mk>!t#(3d1oJXL-~_K*b!J*nhsQ__i`wTees&`$>sY&6O&B;_jw4v(N8 z4)|;hd?GG*c`ASEO;Gi4>akT;vseWT7=$6KpN7*|4irXXd`A|f14+S0)wsBpKfo_; z_^j^XC^NMd<8F#Hrg^Be#>Hn67H$n!#G-Xz=hUph#qm~%5j?wXPDp7rv9wkU?4>Tr z^j1ljzNMEP)2+w%k(@|zBqhqcHxElOh+Wr41r%dTed^1VGx7(fg&LMT<{H`7n&j%i zM#@F&AdLDUS8PPkCQ>>g|DdMKP&EYOu@rv=u(82UcZWgdxbjL)WIdmHdRI6idFxP& zKozeSiZr(9s7z5EqUgU4aDrbOWrTgCw7Ru$MxI49{f$dPFryy*D*KgW8h2AdVf5Zq8KEwbRZOjY#np z!@WdI!QZ=ha;PrA!z#YP^!gxko>vr_q5Vde6REN3IN7~{S2c71dY%r&Wp=|V$XdOR zD@a3%)zz`<2yS_CMAbLZvN5Go=p=n1!uD&DKn0Y);L@nxe039%j~4ri$0uibM={72 zOHXwA40?WV5pZRA(Hh-18Ifw`LG}ugieU*Nm8(=QHDvk6Ic9;=c@V`e5pX(t=4JZo z{YZ3m)(teSI?J1ckC=8z0q*E9miU$_dBy>$>W<;Y)yY~#Lcz8}F=r?qmPvLn6Gn#i z)rM6D{2J0V7#nfbLc#=TU{3LuxVafDVzsEBiI?nENf^Fr5a2tJ@rf|)A6n6`vPR2R z2;%ce3q#h?^V^V5eq%t;W>6<|HLTBzL>xA4wEQ^HiZ#|~hrXLI?C0!CVhVi!Cnj*NX%lD1+zKF5VqaZZyX9hVKr&>-GH*K=_@v72?eS#*<#(}6| zSX~=LphaS7VJ$tVNlVEM$mgRDWWw5ii$#%eBni?~g&r~)M=XN%n7cdW7pcr4URFVY zuFV$z1OVJ+%tBe67588jw~ zuB;}(ibhW%{0iLQnZeE;`U%5G0GtJGLCO-dK_wK_K#jO|!@XUG_d?i@+;J7?LuqeP5cP!^fc0BDP33q;`Mr3cJ zQWoc)gB&nV#VZ zT_3v#dKqpRl6|6(#A3ORJ?13LwbW*Y5-$sHr$LL@DRlo+g^yM{i zBM$eVd|Br^lDw#~pY~MoB^?ijEk6ots-e1NP(=nDO}C`Cu_;qyC#9|Q zEYET+362k+K~((MFrnc@9z>j_5HD1tmFH^l;K)p}{h0Z&N|KDz5F@J8ZamZNbwa)1 zOo~B+dFEviwK$%YKFVd2J)q|@T6tUNa^HoGt2X{LOJ65#UKrYC^SIdgtKP+9os4EV zu}6i8+ap6|*4G81mrae-pXV)Lh3c#bGT&q7rNE*h`lxd8<33ixH*nYp+C%CZYTZ_4 zU2E7;N>rnvXV*zj0!?Hn_ZH!fCZ0sgv1zo?>5exL#{tvqFSXN7SOu_m z^E>=BN>vpsvXCypL3TNEu@Zmj9KeqlzB9KX}cC~B)V(KOOTV%ivq3sCO~nRP-X;Wna7EMRFFj3 zXdN}q)&&}4M|P4q~01T6xq1&qmGWboszr4B5kv%1g3H(kSZ_8^v~#4msJgZPnFjsaC(-mEIaF zSncKAz$7lz%I?k1rrsfwo?#lOm8hG_oJNG)s>3rieK= z>JHW4XEtCj5zODZ*I>(INiIor@X>~laQvXxb1(6Ll&l#cM9TMpW#`NFdW&Gc0Xgvf zA?aS&tH|ahqqh#_i#$BfH#`j2oaS5F1MixIFop1bo-WDlE_?eAVv4<8f`x`+@Lwe1 zQL+6TZGlf0pqBIWmg(c>p!9`)s?L>$nvm{(plt*M&(L8F zfkasH#ySN*SqlCpVkee7ujzfO*WRGb{keHQ?(>(*WuZ9do1GA7!hNxJvw^ zPq98DVx+n4I^SHDeTwF%&Dn-{@qo^2{y?`H!FtwR*bJ+(g{K)fWXF^>(BS%EaZ^Y$ zI@vb=FtL~xF}VhJwA7F51}&Pr6hf)J(XWWXw1{DdyQ~PkM7PsBO{`9KM6*occ*%k+ zI&9i9&A|P^h&`~P!XItC(2bN)B} z1nKs$I=z=AA~Xa92kjm#@TT^OR%KDr*pPipLQRU$Aw&dY@p|mHg!md;Et*^Uqm*=Mg1?LUGf_e> zc+x8l(nW?KW*H^EQ8BPqFkyb}vcSD3AgoXyfY zPdH@mz$C2@<+yjm7r@ckhvb|ACJ&n}0{7N{G`2CXH*ngL8!o{Y@_;@v8|y=tM?f)W zEZ2S;gzYe_OAzU_(cc@u(rbYCb=f8c%x~}2880gQow!yK92qj>6QVKP&NS_3P|WvI z&6Q8Zu<}t7;)0h;uoqqde}5{=-{uZ2gC$atCgqr?EHT&O*UEA7N zQM+1Mepj)wSz(5IIn(Juf0_b>-)$tDGLC|15|hm@#TJ}s^(HqBOG*_j5SRDvseZo2 z78{9090E^jMMnV!cXdBIb32PlSn_HbW?(cBcOZjVp|LHFOv$0&#OJcqiIDbVIdzdM zf0BT!hkL}#v>y#H2B$|zB@^iIlf6eB3r$5Fmsr=c$Q7YsgKAU#d@O(rX$a`!yw;2QBImH)=Y-gqydw!37w?!MX@w3uC;nl^-!g zC@x@ShfqNCGoFiLMzKDboH7a_O;T8~g8z=@=PNRrPus&~p)%&|%OcuDARQ8F%Jvwe z<$fo*>t4{)603ZZ02}Sg^V_%A>lq%>Nur|u;^S4@1dTZrwB2YWyF!DZIcpvA&7**f zUCMGhOK`TIq@m;DiJ{;PzH5O66b&MQ>ak`|(%EIyD4MB`LCNHH`roqxtWgw0#?n>h z8m>!J<;-jo;eUk7H-vIzpl^zFe75%o;M{{aqMqT}uR1m}bv|yjXOf6n2j)Euu~BM6 z&G%jvKKHQWSrgEC%#ZF1hku;m;5mmDe%?ldo}3Qe+$iBWf(5N+=j-TE*?%nMVOZI@H0(*-}tDHy$z zLuz#e#^dnEgCL8>B%wm_Bq5&APtw@*azrw;8fc_(mD(&7d<(v$eIx-Gs_eSmDAf=p zD{X|^#^0bZ(T?#&C(c0HicykirI2YXneqt2g*zAkNJvUPx1bQ) zH*_j|Pb+}Vvla&sbpjUoqDl6~q{|V`10(Xi+IvGU%xG#7orAy_Ll3Lh2Jel8Y6f~S zORY*zYGI59EuthR``sv~wW`3V20u2C|FNMvWe0`qlmW!8?p;cgCTBrR62 zkDN<>(=(wR%*^9bnURA8CHaV?ge3K)WQ`OV`MBh?R3*Tb zV?1KGJpDC-1HA-!%S`x{1mx$d2cbjv(2t=b=%wzuh3=)uM5S&6sSj=gnntBUpMolD zLbH6ce66dq4S1`AeZWxPqSPM%k$~RVtl#P{L2Kj`-@yO&ya@#T<@?)@zby=Z zH~U4^a_zR=3H!#!1@;!=e*5chPl4}SpzoKN6rZ%1u#kcxwUqFmIex1v`^WP)$EI)X z{^r=h$llq^z{r8-&+OlpvVXBpd9(kUQrw?eyfMuDi$&`{EdHS__h*K0Sz`ZUxcz&> zzl+lSnK|k|tpWb_P7&UE@84+J{aGUfziUL~-#4Q75BW7S8|y#q4YJ=g81g@B(CBYk zZ-3e>)9;#{{Qqh8Uj_XBtSh*G=K}w&u=wBf=f8Hv-;w{Ti{CT?|I8Qv?|gax$G-j+ z&Hgigu)p)S_U7-mQDLR$Z1m?JDaG%MA>T&GJL5mw>o(K!I<=zd95Yk`ObuZd@tWn{$~Am{nhuz@1; z|Ao(=`Y*-z$Ba+!;k_r_{RNMN= z0R9*Iz^{qrkAb}>DEtL%iRb?a*q@G#_xxwS@SzL-rTG45W8*#d%P)BC5`QVY|M}Q> zPj&GNqOHvTF~mRq7kppb{0kti@?QqyL_j-s5_o#rzA`t;v6i>(6qU-{W|npY{t!!Q00BKijwcF^)gVR(p@?eY(&u zRONR6DXRa^qr?f(JR;3M(? diff --git a/extensions-contrib/druid-exact-cardinality/pom.xml b/extensions-contrib/druid-exact-cardinality/pom.xml index ab2e0040e7f7..64c9537ebe5c 100644 --- a/extensions-contrib/druid-exact-cardinality/pom.xml +++ b/extensions-contrib/druid-exact-cardinality/pom.xml @@ -22,7 +22,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - org.apache.druid.extensions + org.apache.druid.extensions.contrib druid-exact-cardinality druid-exact-cardinality Druid Aggregators for exact cardinality From 76d59217605f3523bfa624b1d5315dde3464d001 Mon Sep 17 00:00:00 2001 From: GWphua Date: Wed, 28 May 2025 11:35:05 +0800 Subject: [PATCH 29/65] Change docker file to keep things running --- .../DruidExactCardinality/docker-compose.py | 9 +++++---- integration-tests-ex/cases/pom.xml | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/integration-tests-ex/cases/cluster/DruidExactCardinality/docker-compose.py b/integration-tests-ex/cases/cluster/DruidExactCardinality/docker-compose.py index 74e22e8cdd4b..7316486b62c6 100644 --- a/integration-tests-ex/cases/cluster/DruidExactCardinality/docker-compose.py +++ b/integration-tests-ex/cases/cluster/DruidExactCardinality/docker-compose.py @@ -14,6 +14,7 @@ # limitations under the License. from template import BaseTemplate, generate +from template import COORDINATOR, MIDDLE_MANAGER class Template(BaseTemplate): def extend_druid_service(self, service): @@ -27,9 +28,9 @@ def define_coordinator(self): self.add_env(service, 'druid_coordinator_period', 'PT10S') def define_indexer(self): - ''' - Override the indexer to MIDDLE_MANAGER - ''' - return self.define_std_indexer(MIDDLE_MANAGER) + ''' + Override the indexer to MIDDLE_MANAGER + ''' + return self.define_std_indexer(MIDDLE_MANAGER) generate(__file__, Template()) diff --git a/integration-tests-ex/cases/pom.xml b/integration-tests-ex/cases/pom.xml index 15f23dbd7596..90034298ff4a 100644 --- a/integration-tests-ex/cases/pom.xml +++ b/integration-tests-ex/cases/pom.xml @@ -259,6 +259,12 @@ ${project.parent.version} provided + + org.apache.druid.extensions.contrib + druid-exact-cardinality + ${project.parent.version} + provided + org.apache.commons commons-lang3 @@ -557,5 +563,14 @@ Security + + IT-DruidExactCardinality + + false + + + DruidExactCardinality + + From 2bde0d1285c51bdc47d9259aa7be15045c545f2b Mon Sep 17 00:00:00 2001 From: GWphua Date: Wed, 28 May 2025 11:36:11 +0800 Subject: [PATCH 30/65] Set fullDatasourceName after initialization to prevent NPE with config --- .../DruidExactCardinality/DruidExactCardinalityTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java index fd9050261c04..abcdf49fabb3 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java @@ -52,11 +52,12 @@ public class DruidExactCardinalityTest extends AbstractIndexerTest private static final String QUERY_FILE = "/druid-exact-cardinality/queries.json"; private static final String DATA_FILE = "/druid-exact-cardinality/union_data.json"; private static final String DATASOURCE = "wikipedia_index_test"; - private final String fullDatasourceName = DATASOURCE + config.getExtraDatasourceNameSuffix(); + private String fullDatasourceName; @Test public void testQuery() throws Exception { + fullDatasourceName = DATASOURCE + config.getExtraDatasourceNameSuffix(); final String baseName = fullDatasourceName + UUID.randomUUID(); KafkaAdminClient streamAdminClient = new KafkaAdminClient(config); List supervisors = new ArrayList<>(); From 6ea8f1394c76742f0e47f4bcc61f87b632d153a3 Mon Sep 17 00:00:00 2001 From: GWphua Date: Wed, 28 May 2025 16:29:47 +0800 Subject: [PATCH 31/65] Fix adding contrib pom changes to distribution/pom.xml --- distribution/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index 07eba0f6bb45..91ede01adff3 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -463,7 +463,7 @@ -c org.apache.druid.extensions.contrib:druid-ranger-security -c - org.apache.druid.extensions:druid-exact-cardinality + org.apache.druid.extensions.contrib:druid-exact-cardinality From ba4716f1228be31a5857ea76bf02f143e15e90cc Mon Sep 17 00:00:00 2001 From: GWphua Date: Wed, 28 May 2025 17:09:12 +0800 Subject: [PATCH 32/65] Setup IT for contrib packages --- .../DruidExactCardinality/DruidExactCardinalityTest.java | 2 +- .../test/resources/druid-exact-cardinality/queries.json | 8 ++++---- it.sh | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java index abcdf49fabb3..a6321a2e1574 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java @@ -50,7 +50,7 @@ public class DruidExactCardinalityTest extends AbstractIndexerTest private static final Logger LOG = new Logger(DruidExactCardinalityTest.class); private static final String SUPERVISOR_TEMPLATE = "/druid-exact-cardinality/kafka_supervisor_template.json"; private static final String QUERY_FILE = "/druid-exact-cardinality/queries.json"; - private static final String DATA_FILE = "/druid-exact-cardinality/union_data.json"; + private static final String DATA_FILE = "/druid-exact-cardinality/data.json"; private static final String DATASOURCE = "wikipedia_index_test"; private String fullDatasourceName; diff --git a/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json index 2a166210c617..9b1b2e34cea7 100644 --- a/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json +++ b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json @@ -36,11 +36,11 @@ }, "expectedResults": [ { - "timestamp": "2016-06-27T00:00:00.000Z", + "timestamp": "2013-08-31T01:02:33.000Z", "result": { - "time_count": 40, - "time_cardinality": 10, - "deleted_count": 40, + "time_count": 15, + "time_cardinality": 5, + "deleted_count": 15, "deleted_cardinality": 5 } } diff --git a/it.sh b/it.sh index 53eddd087b07..7157fa99f161 100755 --- a/it.sh +++ b/it.sh @@ -229,13 +229,13 @@ case $CMD in usage ;; "ci" ) - mvn -q clean install -P dist $MAVEN_IGNORE -T1C + mvn -q clean install -P dist,bundle-contrib-exts $MAVEN_IGNORE -T1C ;; "build" ) - mvn -B clean install -P dist $MAVEN_IGNORE -T1.0C $* + mvn -B clean install -P dist,bundle-contrib-exts $MAVEN_IGNORE -T1.0C $* ;; "dist" ) - mvn -B install -P dist $MAVEN_IGNORE -pl :distribution + mvn -B install -P dist,bundle-contrib-exts $MAVEN_IGNORE -pl :distribution ;; "tools" ) mvn -B install -pl :druid-it-tools From c45827b9391e8a529165fde3c976fbcad67d6965 Mon Sep 17 00:00:00 2001 From: GWphua Date: Wed, 28 May 2025 19:40:52 +0800 Subject: [PATCH 33/65] Add DruidExactCardinality into IT --- .github/workflows/revised-its.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/revised-its.yml b/.github/workflows/revised-its.yml index bf3f45567bfb..037d6948a70a 100644 --- a/.github/workflows/revised-its.yml +++ b/.github/workflows/revised-its.yml @@ -68,7 +68,7 @@ jobs: matrix: # jdk: [11, 17] jdk: [17] - it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat, Security, Query] + it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat, Security, Query, DruidExactCardinality] #indexer: [indexer, middleManager] indexer: [middleManager] uses: ./.github/workflows/reusable-revised-its.yml From 74b0d197c9ed4680b177209a37ba52d726af90cb Mon Sep 17 00:00:00 2001 From: GWphua Date: Thu, 29 May 2025 23:53:22 +0800 Subject: [PATCH 34/65] Add case to check cardinality ignores duplicates --- .../druid-exact-cardinality/data.json | 4 +- .../kafka_supervisor_template.json | 5 ++ .../druid-exact-cardinality/queries.json | 48 ++++++++++++++----- 3 files changed, 43 insertions(+), 14 deletions(-) diff --git a/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/data.json b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/data.json index b186657dbcf0..a0ac3524c9f1 100644 --- a/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/data.json +++ b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/data.json @@ -1,9 +1,9 @@ -{"timestamp": "2013-08-31T01:02:33Z", "page": "Gypsy Danger", "language" : "en", "user" : "nuclear", "unpatrolled" : "true", "newPage" : "true", "robot": "false", "anonymous": "false", "namespace":"article", "continent":"North America", "country":"United States", "region":"Bay Area", "city":"San Francisco", "added": 57, "deleted": 200, "delta": -143} +{"timestamp": "2013-08-31T01:02:33Z", "page": "Gypsy Danger", "language" : "en", "user" : "nuclear", "unpatrolled" : "true", "newPage" : "true", "robot": "false", "anonymous": "false", "namespace":"article", "continent":"North America", "country":"United States", "region":"Bay Area", "city":"San Francisco", "added": 459, "deleted": 200, "delta": -143} {"timestamp": "2013-08-31T03:32:45Z", "page": "Striker Eureka", "language" : "en", "user" : "speed", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Australia", "country":"Australia", "region":"Cantebury", "city":"Syndey", "added": 459, "deleted": 129, "delta": 330} {"timestamp": "2013-08-31T07:11:21Z", "page": "Cherno Alpha", "language" : "ru", "user" : "masterYi", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"article", "continent":"Asia", "country":"Russia", "region":"Oblast", "city":"Moscow", "added": 123, "deleted": 12, "delta": 111} {"timestamp": "2013-08-31T11:58:39Z", "page": "Crimson Typhoon", "language" : "zh", "user" : "triplets", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"China", "region":"Shanxi", "city":"Taiyuan", "added": 905, "deleted": 5, "delta": 900} {"timestamp": "2013-08-31T12:41:27Z", "page": "Coyote Tango", "language" : "ja", "user" : "stringer", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"Japan", "region":"Kanto", "city":"Tokyo", "added": 1, "deleted": 10, "delta": -9} -{"timestamp": "2013-09-01T01:02:33Z", "page": "Gypsy Danger", "language" : "en", "user" : "nuclear", "unpatrolled" : "true", "newPage" : "true", "robot": "false", "anonymous": "false", "namespace":"article", "continent":"North America", "country":"United States", "region":"Bay Area", "city":"San Francisco", "added": 57, "deleted": 200, "delta": -143} +{"timestamp": "2013-09-01T01:02:33Z", "page": "Gypsy Danger", "language" : "en", "user" : "nuclear", "unpatrolled" : "true", "newPage" : "true", "robot": "false", "anonymous": "false", "namespace":"article", "continent":"North America", "country":"United States", "region":"Bay Area", "city":"San Francisco", "added": 459, "deleted": 200, "delta": -143} {"timestamp": "2013-09-01T03:32:45Z", "page": "Striker Eureka", "language" : "en", "user" : "speed", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Australia", "country":"Australia", "region":"Cantebury", "city":"Syndey", "added": 459, "deleted": 129, "delta": 330} {"timestamp": "2013-09-01T07:11:21Z", "page": "Cherno Alpha", "language" : "ru", "user" : "masterYi", "unpatrolled" : "false", "newPage" : "true", "robot": "true", "anonymous": "false", "namespace":"article", "continent":"Asia", "country":"Russia", "region":"Oblast", "city":"Moscow", "added": 123, "deleted": 12, "delta": 111} {"timestamp": "2013-09-01T11:58:39Z", "page": "Crimson Typhoon", "language" : "zh", "user" : "triplets", "unpatrolled" : "true", "newPage" : "false", "robot": "true", "anonymous": "false", "namespace":"wikipedia", "continent":"Asia", "country":"China", "region":"Shanxi", "city":"Taiyuan", "added": 905, "deleted": 5, "delta": 900} diff --git a/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/kafka_supervisor_template.json b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/kafka_supervisor_template.json index a37340fe858f..65d6b14f1a2e 100644 --- a/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/kafka_supervisor_template.json +++ b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/kafka_supervisor_template.json @@ -43,6 +43,11 @@ "type": "doubleSum", "name": "delta", "fieldName": "delta" + }, + { + "type": "Bitmap64ExactCardinalityBuild", + "name": "unique_deleted", + "fieldName": "deleted" } ], "granularitySpec": { diff --git a/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json index 9b1b2e34cea7..da82dc0c080c 100644 --- a/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json +++ b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json @@ -1,6 +1,6 @@ [ { - "description": "Timeseries query.", + "description": "Timeseries query on non-rolled-up columns", "query": { "queryType": "timeseries", "dataSource": { @@ -15,22 +15,48 @@ "aggregations": [ { "type": "count", - "name": "time_count" + "name": "added_count", + "fieldName": "added" }, { "type": "Bitmap64ExactCardinalityBuild", - "name": "time_cardinality", - "fieldName": "__time" - }, + "name": "added_cardinality", + "fieldName": "added" + } + ] + }, + "expectedResults": [ + { + "timestamp": "2013-08-31T01:02:33.000Z", + "result": { + "added_count": 15, + "added_cardinality": 4 + } + } + ] + }, + { + "description": "Bitmap64 works on time column", + "query": { + "queryType": "timeseries", + "dataSource": { + "type": "union", + "dataSources": [ + "%%DATASOURCE%%-1", "%%DATASOURCE%%-2", "%%DATASOURCE%%-3", + "%%DATASOURCE%%-0" + ] + }, + "intervals": ["2013-08-31/2013-09-01"], + "granularity": "all", + "aggregations": [ { "type": "count", - "name": "deleted_count", - "fieldName": "deleted" + "name": "time_count" }, { "type": "Bitmap64ExactCardinalityBuild", - "name": "deleted_cardinality", - "fieldName": "deleted" + "name": "time_cardinality", + "fieldName": "__time" } ] }, @@ -39,9 +65,7 @@ "timestamp": "2013-08-31T01:02:33.000Z", "result": { "time_count": 15, - "time_cardinality": 5, - "deleted_count": 15, - "deleted_cardinality": 5 + "time_cardinality": 5 } } ] From 58c5c0005115578f23937b29092f6597b1ba05d7 Mon Sep 17 00:00:00 2001 From: GWphua Date: Thu, 29 May 2025 23:53:37 +0800 Subject: [PATCH 35/65] Add case to check cardinality works on rolled-up columns: --- .../druid-exact-cardinality/queries.json | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json index da82dc0c080c..48251e8e18d0 100644 --- a/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json +++ b/integration-tests-ex/cases/src/test/resources/druid-exact-cardinality/queries.json @@ -35,6 +35,48 @@ } ] }, + { + "description": "Timeseries query on rolled-up columns", + "query": { + "queryType": "timeseries", + "dataSource": { + "type": "union", + "dataSources": [ + "%%DATASOURCE%%-1", "%%DATASOURCE%%-2", "%%DATASOURCE%%-3", + "%%DATASOURCE%%-0" + ] + }, + "intervals": ["2013-08-31/2013-09-01"], + "granularity": "all", + "aggregations": [ + { + "type": "count", + "name": "deleted_count", + "fieldName": "deleted" + }, + { + "type": "Bitmap64ExactCardinalityBuild", + "name": "deleted_cardinality", + "fieldName": "deleted" + }, + { + "type": "Bitmap64ExactCardinalityMerge", + "name": "deleted_cardinality_from_rollup_columns", + "fieldName": "unique_deleted" + } + ] + }, + "expectedResults": [ + { + "timestamp": "2013-08-31T01:02:33.000Z", + "result": { + "deleted_count": 15, + "deleted_cardinality": 5, + "deleted_cardinality_from_rollup_columns": 5 + } + } + ] + }, { "description": "Bitmap64 works on time column", "query": { From 1dea8f0f1aedc269e927c5a88416d8fd51f6bc95 Mon Sep 17 00:00:00 2001 From: GWphua Date: Fri, 30 May 2025 10:26:00 +0800 Subject: [PATCH 36/65] Fix fullDatasourceName can be a local variable --- .../DruidExactCardinality/DruidExactCardinalityTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java index a6321a2e1574..56e0e8e7ac58 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCardinality/DruidExactCardinalityTest.java @@ -52,13 +52,12 @@ public class DruidExactCardinalityTest extends AbstractIndexerTest private static final String QUERY_FILE = "/druid-exact-cardinality/queries.json"; private static final String DATA_FILE = "/druid-exact-cardinality/data.json"; private static final String DATASOURCE = "wikipedia_index_test"; - private String fullDatasourceName; @Test public void testQuery() throws Exception { - fullDatasourceName = DATASOURCE + config.getExtraDatasourceNameSuffix(); - final String baseName = fullDatasourceName + UUID.randomUUID(); + final String fullDatasourceName = DATASOURCE + config.getExtraDatasourceNameSuffix(); + String baseName = fullDatasourceName + UUID.randomUUID(); KafkaAdminClient streamAdminClient = new KafkaAdminClient(config); List supervisors = new ArrayList<>(); From 029de04a9ea6c3bd1b8b5ccb5f1f00395ad66acd Mon Sep 17 00:00:00 2001 From: GWphua Date: Wed, 4 Jun 2025 11:44:48 +0800 Subject: [PATCH 37/65] Bitmap64 exactcount update --- distribution/pom.xml | 2 +- docs/configuration/extensions.md | 2 +- ...ct-cardinality.md => druid-exact-count.md} | 56 +++++++-------- .../pom.xml | 6 +- .../exact/count/bitmap64/Bitmap64.java} | 6 +- .../Bitmap64ExactCountAggregatorFactory.java} | 42 +++++------ .../Bitmap64ExactCountBuildAggregator.java} | 8 +-- ...ap64ExactCountBuildAggregatorFactory.java} | 14 ++-- ...map64ExactCountBuildBufferAggregator.java} | 30 ++++---- ...p64ExactCountBuildComplexMetricSerde.java} | 4 +- .../Bitmap64ExactCountMergeAggregator.java} | 10 +-- ...ap64ExactCountMergeAggregatorFactory.java} | 18 ++--- ...map64ExactCountMergeBufferAggregator.java} | 20 +++--- ...p64ExactCountMergeComplexMetricSerde.java} | 10 +-- .../bitmap64/Bitmap64ExactCountModule.java} | 28 ++++---- .../Bitmap64ExactCountObjectStrategy.java} | 16 ++--- .../Bitmap64ExactCountPostAggregator.java} | 16 ++--- .../bitmap64/RoaringBitmap64Counter.java | 6 +- .../RoaringBitmap64CounterJsonSerializer.java | 2 +- .../sql/Bitmap64ExactCountSqlAggregator.java} | 14 ++-- ...rg.apache.druid.initialization.DruidModule | 2 +- ...map64ExactCountAggregatorFactoryTest.java} | 72 +++++++++---------- ...ExactCountBuildAggregatorFactoryTest.java} | 32 ++++----- ...itmap64ExactCountBuildAggregatorTest.java} | 16 ++--- ...4ExactCountBuildBufferAggregatorTest.java} | 34 ++++----- ...ExactCountMergeAggregatorFactoryTest.java} | 34 ++++----- ...itmap64ExactCountMergeAggregatorTest.java} | 22 +++--- ...4ExactCountMergeBufferAggregatorTest.java} | 36 +++++----- .../Bitmap64ExactCountModuleTest.java} | 18 ++--- ...Bitmap64ExactCountPostAggregatorTest.java} | 30 ++++---- .../Bitmap64ExactCountSqlAggregatorTest.java} | 46 ++++++------ .../docker-compose.py | 2 +- integration-tests-ex/cases/pom.xml | 6 +- .../DruidExactCountTest.java} | 12 ++-- .../ITDruidExactCountTest.java} | 8 +-- ...tCardinality.java => DruidExactCount.java} | 2 +- .../data.json | 0 .../kafka_supervisor_template.json | 2 +- .../queries.json | 8 +-- pom.xml | 2 +- .../query/aggregation/AggregatorUtil.java | 4 +- .../aggregation/post/PostAggregatorIds.java | 2 +- website/.spelling | 6 +- 43 files changed, 353 insertions(+), 353 deletions(-) rename docs/development/extensions-contrib/{druid-exact-cardinality.md => druid-exact-count.md} (85%) rename extensions-contrib/{druid-exact-cardinality => druid-exact-count}/pom.xml (96%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64Counter.java => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64.java} (93%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactory.java => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountAggregatorFactory.java} (72%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregator.java => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountBuildAggregator.java} (85%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactory.java => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountBuildAggregatorFactory.java} (77%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregator.java => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountBuildBufferAggregator.java} (71%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildComplexMetricSerde.java => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountBuildComplexMetricSerde.java} (88%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregator.java => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountMergeAggregator.java} (81%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactory.java => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountMergeAggregatorFactory.java} (72%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeBufferAggregator.java => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountMergeBufferAggregator.java} (77%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeComplexMetricSerde.java => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountMergeComplexMetricSerde.java} (89%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModule.java => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountModule.java} (61%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityObjectStrategy.java => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountObjectStrategy.java} (71%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregator.java => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountPostAggregator.java} (87%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count}/bitmap64/RoaringBitmap64Counter.java (93%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count}/bitmap64/RoaringBitmap64CounterJsonSerializer.java (95%) rename extensions-contrib/{druid-exact-cardinality/src/main/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregator.java => druid-exact-count/src/main/java/org/apache/druid/query/aggregation/exact/count/bitmap64/sql/Bitmap64ExactCountSqlAggregator.java} (90%) rename extensions-contrib/{druid-exact-cardinality => druid-exact-count}/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule (89%) rename extensions-contrib/{druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityAggregatorFactoryTest.java => druid-exact-count/src/test/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountAggregatorFactoryTest.java} (70%) rename extensions-contrib/{druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorFactoryTest.java => druid-exact-count/src/test/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountBuildAggregatorFactoryTest.java} (67%) rename extensions-contrib/{druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildAggregatorTest.java => druid-exact-count/src/test/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountBuildAggregatorTest.java} (85%) rename extensions-contrib/{druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityBuildBufferAggregatorTest.java => druid-exact-count/src/test/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountBuildBufferAggregatorTest.java} (82%) rename extensions-contrib/{druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorFactoryTest.java => druid-exact-count/src/test/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountMergeAggregatorFactoryTest.java} (66%) rename extensions-contrib/{druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeAggregatorTest.java => druid-exact-count/src/test/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountMergeAggregatorTest.java} (85%) rename extensions-contrib/{druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityMergeBufferAggregatorTest.java => druid-exact-count/src/test/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountMergeBufferAggregatorTest.java} (83%) rename extensions-contrib/{druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityModuleTest.java => druid-exact-count/src/test/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountModuleTest.java} (67%) rename extensions-contrib/{druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/Bitmap64ExactCardinalityPostAggregatorTest.java => druid-exact-count/src/test/java/org/apache/druid/query/aggregation/exact/count/bitmap64/Bitmap64ExactCountPostAggregatorTest.java} (76%) rename extensions-contrib/{druid-exact-cardinality/src/test/java/org/apache/druid/query/aggregation/exact/cardinality/bitmap64/sql/Bitmap64ExactCardinalitySqlAggregatorTest.java => druid-exact-count/src/test/java/org/apache/druid/query/aggregation/exact/count/bitmap64/sql/Bitmap64ExactCountSqlAggregatorTest.java} (81%) rename integration-tests-ex/cases/cluster/{DruidExactCardinality => DruidExactCount}/docker-compose.py (98%) rename integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/{DruidExactCardinality/DruidExactCardinalityTest.java => DruidExactCount/DruidExactCountTest.java} (94%) rename integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/{DruidExactCardinality/ITDruidExactCardinalityTest.java => DruidExactCount/ITDruidExactCountTest.java} (80%) rename integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/{DruidExactCardinality.java => DruidExactCount.java} (96%) rename integration-tests-ex/cases/src/test/resources/{druid-exact-cardinality => druid-exact-count}/data.json (100%) rename integration-tests-ex/cases/src/test/resources/{druid-exact-cardinality => druid-exact-count}/kafka_supervisor_template.json (96%) rename integration-tests-ex/cases/src/test/resources/{druid-exact-cardinality => druid-exact-count}/queries.json (92%) diff --git a/distribution/pom.xml b/distribution/pom.xml index 91ede01adff3..71a530764a5b 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -463,7 +463,7 @@ -c org.apache.druid.extensions.contrib:druid-ranger-security -c - org.apache.druid.extensions.contrib:druid-exact-cardinality + org.apache.druid.extensions.contrib:druid-exact-count diff --git a/docs/configuration/extensions.md b/docs/configuration/extensions.md index 97e5c368efe9..9323c6590c90 100644 --- a/docs/configuration/extensions.md +++ b/docs/configuration/extensions.md @@ -86,7 +86,7 @@ All of these community extensions can be downloaded using [pull-deps](../operati |druid-ddsketch|Support for DDSketch approximate quantiles based on [DDSketch](https://github.com/datadog/sketches-java) | [link](../development/extensions-contrib/ddsketch-quantiles.md)| |druid-deltalake-extensions|Support for ingesting Delta Lake tables.|[link](../development/extensions-contrib/delta-lake.md)| |druid-distinctcount|DistinctCount aggregator|[link](../development/extensions-contrib/distinctcount.md)| -|druid-exact-cardinality|Support for exact cardinality counting using Roaring Bitmaps.|[link](../development/extensions-contrib/druid-exact-cardinality.md)| +|druid-exact-count|Support for exact cardinality counting using Roaring Bitmaps.|[link](../development/extensions-contrib/druid-exact-count.md)| |druid-iceberg-extensions|Support for ingesting Iceberg tables.|[link](../development/extensions-contrib/iceberg.md)| |druid-redis-cache|A cache implementation for Druid based on Redis.|[link](../development/extensions-contrib/redis-cache.md)| |druid-time-min-max|Min/Max aggregator for timestamp.|[link](../development/extensions-contrib/time-min-max.md)| diff --git a/docs/development/extensions-contrib/druid-exact-cardinality.md b/docs/development/extensions-contrib/druid-exact-count.md similarity index 85% rename from docs/development/extensions-contrib/druid-exact-cardinality.md rename to docs/development/extensions-contrib/druid-exact-count.md index 01171b8ee693..088e09c30727 100644 --- a/docs/development/extensions-contrib/druid-exact-cardinality.md +++ b/docs/development/extensions-contrib/druid-exact-count.md @@ -1,6 +1,6 @@ --- -id: druid-exact-cardinality -title: "Exact Cardinality" +id: druid-exact-count +title: "Exact Count" ---