From 457deed949b02461a96340d0e9dea38eab057bc9 Mon Sep 17 00:00:00 2001 From: liuboyu Date: Fri, 21 Dec 2018 11:09:28 +0800 Subject: [PATCH 1/6] add extension accurate-cardinality --- .idea/inspectionProfiles/Druid.xml | 30 +- .../inspectionProfiles/profiles_settings.xml | 1 + distribution/pom.xml | 4 +- .../accurate-cardinality.md | 87 + .../accurate-cardinality/pom.xml | 92 + .../AccurateCardinalityAggregator.java | 86 + .../AccurateCardinalityAggregatorFactory.java | 264 ++ .../AccurateCardinalityBufferAggregator.java | 110 + .../accurate/AccurateCardinalityModule.java | 78 + .../accurate/BitmapAggregator.java | 73 + .../accurate/BitmapAggregatorCombiner.java | 76 + .../accurate/BitmapAggregatorFactory.java | 200 + .../accurate/BitmapBufferAggregator.java | 108 + .../accurate/bitmap64/BitmapFactory.java | 33 + .../accurate/bitmap64/ImmutableBitmap.java | 55 + .../bitmap64/ImmutableRoaringBitmap.java | 105 + .../accurate/bitmap64/MutableBitmap.java | 97 + .../accurate/bitmap64/RoaringBitmap.java | 115 + .../bitmap64/RoaringBitmapFactory.java | 63 + .../accurate/collector/Collector.java | 33 + .../accurate/collector/CollectorFactory.java | 36 + .../collector/RoaringBitmapCollector.java | 99 + ...ringBitmapCollectorComplexMetricSerde.java | 140 + .../RoaringBitmapCollectorFactory.java | 63 + .../RoaringBitmapCollectorJsonSerializer.java | 36 + .../sql/AccurateCardinalitySqlAggregator.java | 140 + ...alityAggregatorColumnSelectorStrategy.java | 28 + ...gregatorColumnSelectorStrategyFactory.java | 50 + ...alityAggregatorColumnSelectorStrategy.java | 33 + ...alityAggregatorColumnSelectorStrategy.java | 35 + ...rg.apache.druid.initialization.DruidModule | 16 + .../AccurateCardinalityAggGroupbyTest.java | 192 + .../AccurateCardinalityAggregatorTest.java | 734 ++++ .../accurate/BitmapAggGroupbyTest.java | 193 + .../AccurateCardinalityAggregatorSqlTest.java | 309 ++ .../bitmap_test_data_aggregators.json | 11 + .../bitmap_test_data_group_by_query.json | 18 + .../bitmap_test_data_record_parser.json | 22 + .../src/test/resources/simple_test_data.tsv | 3360 +++++++++++++++++ .../simple_test_data_aggregators.json | 6 + .../simple_test_data_group_by_query.json | 24 + .../simple_test_data_group_by_query1.json | 16 + .../simple_test_data_record_parser.json | 26 + pom.xml | 1 + .../query/aggregation/AggregatorUtil.java | 2 + .../aggregation/AggregationTestHelper.java | 10 +- 46 files changed, 7287 insertions(+), 23 deletions(-) create mode 100644 docs/content/development/extensions-contrib/accurate-cardinality.md create mode 100644 extensions-contrib/accurate-cardinality/pom.xml create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/AccurateCardinalityAggregator.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/AccurateCardinalityAggregatorFactory.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/AccurateCardinalityBufferAggregator.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/AccurateCardinalityModule.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/BitmapAggregator.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/BitmapAggregatorCombiner.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/BitmapAggregatorFactory.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/BitmapBufferAggregator.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/bitmap64/BitmapFactory.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/bitmap64/ImmutableBitmap.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/bitmap64/ImmutableRoaringBitmap.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/bitmap64/MutableBitmap.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/bitmap64/RoaringBitmap.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/bitmap64/RoaringBitmapFactory.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/collector/Collector.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/collector/CollectorFactory.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/collector/RoaringBitmapCollector.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/collector/RoaringBitmapCollectorComplexMetricSerde.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/collector/RoaringBitmapCollectorFactory.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/collector/RoaringBitmapCollectorJsonSerializer.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/sql/AccurateCardinalitySqlAggregator.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/types/AccurateCardinalityAggregatorColumnSelectorStrategy.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/types/AccurateCardinalityAggregatorColumnSelectorStrategyFactory.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/types/LongAccurateCardinalityAggregatorColumnSelectorStrategy.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/java/org/apache/druid/query/aggregation/cardinality/accurate/types/StringAccurateCardinalityAggregatorColumnSelectorStrategy.java create mode 100644 extensions-contrib/accurate-cardinality/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule create mode 100644 extensions-contrib/accurate-cardinality/src/test/java/org/apache/druid/query/aggregation/cardinality/accurate/AccurateCardinalityAggGroupbyTest.java create mode 100644 extensions-contrib/accurate-cardinality/src/test/java/org/apache/druid/query/aggregation/cardinality/accurate/AccurateCardinalityAggregatorTest.java create mode 100644 extensions-contrib/accurate-cardinality/src/test/java/org/apache/druid/query/aggregation/cardinality/accurate/BitmapAggGroupbyTest.java create mode 100644 extensions-contrib/accurate-cardinality/src/test/java/org/apache/druid/query/aggregation/cardinality/accurate/sql/AccurateCardinalityAggregatorSqlTest.java create mode 100644 extensions-contrib/accurate-cardinality/src/test/resources/bitmap_test_data_aggregators.json create mode 100644 extensions-contrib/accurate-cardinality/src/test/resources/bitmap_test_data_group_by_query.json create mode 100644 extensions-contrib/accurate-cardinality/src/test/resources/bitmap_test_data_record_parser.json create mode 100644 extensions-contrib/accurate-cardinality/src/test/resources/simple_test_data.tsv create mode 100644 extensions-contrib/accurate-cardinality/src/test/resources/simple_test_data_aggregators.json create mode 100644 extensions-contrib/accurate-cardinality/src/test/resources/simple_test_data_group_by_query.json create mode 100644 extensions-contrib/accurate-cardinality/src/test/resources/simple_test_data_group_by_query1.json create mode 100644 extensions-contrib/accurate-cardinality/src/test/resources/simple_test_data_record_parser.json diff --git a/.idea/inspectionProfiles/Druid.xml b/.idea/inspectionProfiles/Druid.xml index bac71e735086..a1ac002660a6 100644 --- a/.idea/inspectionProfiles/Druid.xml +++ b/.idea/inspectionProfiles/Druid.xml @@ -33,7 +33,7 @@ - @@ -96,6 +96,10 @@ + + - @@ -157,29 +158,29 @@ - + - + - + - + - + @@ -366,27 +367,20 @@ - - - - + @@ -432,4 +426,4 @@ - \ No newline at end of file + diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml index 3660904c1029..700480abde93 100644 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -1,6 +1,7 @@