From baef0422754ca21b0f623890c898f21fcc31a9fd Mon Sep 17 00:00:00 2001 From: Dave Li Date: Tue, 10 May 2016 14:53:31 -0700 Subject: [PATCH 1/9] add get dimension rangeset to filters --- .../src/main/java/io/druid/query/Query.java | 3 +++ .../DataSourceMetadataQuery.java | 7 +++++++ .../io/druid/query/filter/AndDimFilter.java | 19 +++++++++++++++++++ .../io/druid/query/filter/BoundDimFilter.java | 17 +++++++++++++++++ .../java/io/druid/query/filter/DimFilter.java | 3 +++ .../query/filter/ExtractionDimFilter.java | 7 +++++++ .../io/druid/query/filter/InDimFilter.java | 16 ++++++++++++++++ .../query/filter/JavaScriptDimFilter.java | 7 +++++++ .../io/druid/query/filter/NoopDimFilter.java | 8 ++++++++ .../io/druid/query/filter/NotDimFilter.java | 8 ++++++++ .../io/druid/query/filter/OrDimFilter.java | 17 +++++++++++++++++ .../io/druid/query/filter/RegexDimFilter.java | 7 +++++++ .../query/filter/SearchQueryDimFilter.java | 7 +++++++ .../druid/query/filter/SelectorDimFilter.java | 14 ++++++++++++++ .../druid/query/filter/SpatialDimFilter.java | 7 +++++++ .../io/druid/query/groupby/GroupByQuery.java | 6 ++++++ .../metadata/SegmentMetadataQuery.java | 7 +++++++ .../query/search/search/SearchQuery.java | 6 ++++++ .../io/druid/query/select/SelectQuery.java | 6 ++++++ .../query/timeboundary/TimeBoundaryQuery.java | 7 +++++++ .../query/timeseries/TimeseriesQuery.java | 6 ++++++ .../java/io/druid/query/topn/TopNQuery.java | 6 ++++++ .../druid/client/CachingClusteredClient.java | 10 ++++++++++ 23 files changed, 201 insertions(+) diff --git a/processing/src/main/java/io/druid/query/Query.java b/processing/src/main/java/io/druid/query/Query.java index e22a3e3ebd91..cff46cb94ad2 100644 --- a/processing/src/main/java/io/druid/query/Query.java +++ b/processing/src/main/java/io/druid/query/Query.java @@ -24,6 +24,7 @@ import com.google.common.collect.Ordering; import com.metamx.common.guava.Sequence; import io.druid.query.datasourcemetadata.DataSourceMetadataQuery; +import io.druid.query.filter.DimFilter; import io.druid.query.groupby.GroupByQuery; import io.druid.query.metadata.metadata.SegmentMetadataQuery; import io.druid.query.search.search.SearchQuery; @@ -65,6 +66,8 @@ public interface Query boolean hasFilters(); + DimFilter getFilter(); + String getType(); Sequence run(QuerySegmentWalker walker, Map context); diff --git a/processing/src/main/java/io/druid/query/datasourcemetadata/DataSourceMetadataQuery.java b/processing/src/main/java/io/druid/query/datasourcemetadata/DataSourceMetadataQuery.java index df2c52673fe8..186fcaf2f6ef 100644 --- a/processing/src/main/java/io/druid/query/datasourcemetadata/DataSourceMetadataQuery.java +++ b/processing/src/main/java/io/druid/query/datasourcemetadata/DataSourceMetadataQuery.java @@ -27,6 +27,7 @@ import io.druid.query.DataSource; import io.druid.query.Query; import io.druid.query.Result; +import io.druid.query.filter.DimFilter; import io.druid.query.spec.MultipleIntervalSegmentSpec; import io.druid.query.spec.QuerySegmentSpec; import org.joda.time.DateTime; @@ -66,6 +67,12 @@ public boolean hasFilters() return false; } + @Override + public DimFilter getFilter() + { + return null; + } + @Override public String getType() { diff --git a/processing/src/main/java/io/druid/query/filter/AndDimFilter.java b/processing/src/main/java/io/druid/query/filter/AndDimFilter.java index de1b6025dde2..1cc1eb8f74de 100644 --- a/processing/src/main/java/io/druid/query/filter/AndDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/AndDimFilter.java @@ -23,6 +23,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.collect.RangeSet; +import com.google.common.collect.TreeRangeSet; import io.druid.query.Druids; import io.druid.segment.filter.AndFilter; import io.druid.segment.filter.Filters; @@ -72,6 +74,23 @@ public Filter toFilter() return new AndFilter(Filters.toFilters(fields)); } + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + RangeSet retSet = TreeRangeSet.create(); + for (DimFilter field : fields) { + RangeSet rangeSet = field.getDimensionRangeSet(dimension); + if (rangeSet != null) { + if (retSet.isEmpty()) { + retSet.addAll(rangeSet); + } else { + retSet.removeAll(rangeSet.complement()); + } + } + } + return retSet; + } + @Override public boolean equals(Object o) { diff --git a/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java b/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java index d1dcd453704f..2f6b9cc48bc3 100644 --- a/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java @@ -22,11 +22,16 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.BoundType; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import com.google.common.collect.TreeRangeSet; import com.metamx.common.StringUtils; import io.druid.query.extraction.ExtractionFn; import io.druid.segment.filter.BoundFilter; import java.nio.ByteBuffer; +import java.util.Objects; public class BoundDimFilter implements DimFilter { @@ -165,6 +170,18 @@ public Filter toFilter() return new BoundFilter(this); } + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + if (!Objects.equals(getDimension(), dimension) || getExtractionFn() != null) { + return null; + } + RangeSet retSet = TreeRangeSet.create(); + retSet.add(Range.range(getLower(), isLowerStrict() ? BoundType.OPEN : BoundType.CLOSED, + getUpper(), isUpperStrict() ? BoundType.OPEN : BoundType.CLOSED)); + return retSet; + } + @Override public boolean equals(Object o) { diff --git a/processing/src/main/java/io/druid/query/filter/DimFilter.java b/processing/src/main/java/io/druid/query/filter/DimFilter.java index ff9450fb852b..cd4f9a4d1409 100644 --- a/processing/src/main/java/io/druid/query/filter/DimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/DimFilter.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.collect.RangeSet; /** */ @@ -56,4 +57,6 @@ public interface DimFilter * @return a Filter that implements this DimFilter, or null if this DimFilter is a no-op. */ public Filter toFilter(); + + public RangeSet getDimensionRangeSet (String dimension); } diff --git a/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java b/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java index ad0f165966a4..0c7063508f63 100644 --- a/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.RangeSet; import com.metamx.common.StringUtils; import io.druid.query.extraction.ExtractionFn; @@ -103,6 +104,12 @@ public Filter toFilter() return new SelectorDimFilter(dimension, value, extractionFn).toFilter(); } + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + return null; + } + @Override public String toString() { diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index 6c896cd8e09e..fc96be11da25 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -26,6 +26,9 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Iterables; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import com.google.common.collect.TreeRangeSet; import com.metamx.common.StringUtils; import io.druid.query.extraction.ExtractionFn; import io.druid.query.lookup.LookupExtractionFn; @@ -170,6 +173,19 @@ public Filter toFilter() return new InFilter(dimension, values, extractionFn); } + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + if (!Objects.equals(getDimension(), dimension) || getExtractionFn() != null) { + return null; + } + RangeSet retSet = TreeRangeSet.create(); + for (String value : values) { + retSet.add(Range.singleton(value)); + } + return retSet; + } + @Override public boolean equals(Object o) { diff --git a/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java b/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java index 97b01eff0ecc..dcab4147ef14 100644 --- a/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.collect.RangeSet; import com.metamx.common.ISE; import com.metamx.common.StringUtils; import io.druid.js.JavaScriptConfig; @@ -117,6 +118,12 @@ public Filter toFilter() return new JavaScriptFilter(dimension, predicate); } + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + return null; + } + @Override public String toString() { diff --git a/processing/src/main/java/io/druid/query/filter/NoopDimFilter.java b/processing/src/main/java/io/druid/query/filter/NoopDimFilter.java index 74c72500aa43..bda0c849d3bd 100644 --- a/processing/src/main/java/io/druid/query/filter/NoopDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/NoopDimFilter.java @@ -19,6 +19,8 @@ package io.druid.query.filter; +import com.google.common.collect.RangeSet; + import java.nio.ByteBuffer; /** @@ -42,4 +44,10 @@ public Filter toFilter() { return null; } + + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + return null; + } } diff --git a/processing/src/main/java/io/druid/query/filter/NotDimFilter.java b/processing/src/main/java/io/druid/query/filter/NotDimFilter.java index 32cbb658baec..b9a6f1d6c04e 100644 --- a/processing/src/main/java/io/druid/query/filter/NotDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/NotDimFilter.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.RangeSet; import io.druid.query.Druids; import io.druid.segment.filter.NotFilter; @@ -68,6 +69,13 @@ public Filter toFilter() return new NotFilter(field.toFilter()); } + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + RangeSet rangeSet = field.getDimensionRangeSet(dimension); + return rangeSet == null ? null : rangeSet.complement(); + } + @Override public boolean equals(Object o) { diff --git a/processing/src/main/java/io/druid/query/filter/OrDimFilter.java b/processing/src/main/java/io/druid/query/filter/OrDimFilter.java index 49b7b77f7f18..9928a8c92d34 100644 --- a/processing/src/main/java/io/druid/query/filter/OrDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/OrDimFilter.java @@ -23,6 +23,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.collect.RangeSet; +import com.google.common.collect.TreeRangeSet; import io.druid.query.Druids; import io.druid.segment.filter.Filters; import io.druid.segment.filter.OrFilter; @@ -72,6 +74,21 @@ public Filter toFilter() return new OrFilter(Filters.toFilters(fields)); } + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + RangeSet retSet = TreeRangeSet.create(); + for (DimFilter field : fields) { + RangeSet rangeSet = field.getDimensionRangeSet(dimension); + if (rangeSet == null) { + return null; + } else { + retSet.addAll(rangeSet); + } + } + return retSet; + } + @Override public boolean equals(Object o) { diff --git a/processing/src/main/java/io/druid/query/filter/RegexDimFilter.java b/processing/src/main/java/io/druid/query/filter/RegexDimFilter.java index da6e49f33096..047aab6b244b 100644 --- a/processing/src/main/java/io/druid/query/filter/RegexDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/RegexDimFilter.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.RangeSet; import com.metamx.common.StringUtils; import io.druid.query.extraction.ExtractionFn; import io.druid.segment.filter.RegexFilter; @@ -101,6 +102,12 @@ public Filter toFilter() return new RegexFilter(dimension, compiledPattern, extractionFn); } + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + return null; + } + @Override public String toString() { diff --git a/processing/src/main/java/io/druid/query/filter/SearchQueryDimFilter.java b/processing/src/main/java/io/druid/query/filter/SearchQueryDimFilter.java index b5bbdf4024a5..761ee492e22a 100644 --- a/processing/src/main/java/io/druid/query/filter/SearchQueryDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SearchQueryDimFilter.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.RangeSet; import com.metamx.common.StringUtils; import io.druid.query.extraction.ExtractionFn; import io.druid.query.search.search.SearchQuerySpec; @@ -97,6 +98,12 @@ public Filter toFilter() return new SearchQueryFilter(dimension, query, extractionFn); } + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + return null; + } + @Override public String toString() { diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index 08f7e289420e..eba093704f32 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -25,6 +25,9 @@ import com.google.common.base.Predicate; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import com.google.common.collect.TreeRangeSet; import com.metamx.common.StringUtils; import io.druid.query.extraction.ExtractionFn; import io.druid.segment.filter.DimensionPredicateFilter; @@ -152,6 +155,17 @@ public boolean equals(Object o) return extractionFn != null ? extractionFn.equals(that.extractionFn) : that.extractionFn == null; } + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + if (!Objects.equals(getDimension(), dimension) || getExtractionFn() != null) { + return null; + } + RangeSet retSet = TreeRangeSet.create(); + retSet.add(Range.singleton(value)); + return retSet; + } + @Override public int hashCode() { diff --git a/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java b/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java index 0bf514abd8ba..352ffa86fb41 100644 --- a/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.RangeSet; import com.metamx.collections.spatial.search.Bound; import com.metamx.common.StringUtils; import io.druid.segment.filter.SpatialFilter; @@ -85,6 +86,12 @@ public Filter toFilter() return new SpatialFilter(dimension, bound); } + @Override + public RangeSet getDimensionRangeSet(String dimension) + { + return null; + } + @Override public boolean equals(Object o) { diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java index 9abc6d6f317c..7639d16dece6 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java @@ -212,6 +212,12 @@ public boolean hasFilters() return dimFilter != null; } + @Override + public DimFilter getFilter() + { + return dimFilter; + } + @Override public String getType() { diff --git a/processing/src/main/java/io/druid/query/metadata/metadata/SegmentMetadataQuery.java b/processing/src/main/java/io/druid/query/metadata/metadata/SegmentMetadataQuery.java index d02f9c924e4b..b6086eccf18a 100644 --- a/processing/src/main/java/io/druid/query/metadata/metadata/SegmentMetadataQuery.java +++ b/processing/src/main/java/io/druid/query/metadata/metadata/SegmentMetadataQuery.java @@ -30,6 +30,7 @@ import io.druid.query.Query; import io.druid.query.TableDataSource; import io.druid.query.UnionDataSource; +import io.druid.query.filter.DimFilter; import io.druid.query.spec.MultipleIntervalSegmentSpec; import io.druid.query.spec.QuerySegmentSpec; import org.joda.time.Interval; @@ -153,6 +154,12 @@ public boolean hasFilters() return false; } + @Override + public DimFilter getFilter() + { + return null; + } + @Override public String getType() { diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQuery.java b/processing/src/main/java/io/druid/query/search/search/SearchQuery.java index 1d67cdb9a583..f3d1976c86f1 100644 --- a/processing/src/main/java/io/druid/query/search/search/SearchQuery.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchQuery.java @@ -78,6 +78,12 @@ public boolean hasFilters() return dimFilter != null; } + @Override + public DimFilter getFilter() + { + return dimFilter; + } + @Override public String getType() { diff --git a/processing/src/main/java/io/druid/query/select/SelectQuery.java b/processing/src/main/java/io/druid/query/select/SelectQuery.java index 2e3e36304c7f..fb552447c432 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQuery.java +++ b/processing/src/main/java/io/druid/query/select/SelectQuery.java @@ -86,6 +86,12 @@ public boolean hasFilters() return dimFilter != null; } + @Override + public DimFilter getFilter() + { + return dimFilter; + } + @Override public String getType() { diff --git a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java index 92770e69fb90..a492eda59166 100644 --- a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java +++ b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java @@ -29,6 +29,7 @@ import io.druid.query.DataSource; import io.druid.query.Query; import io.druid.query.Result; +import io.druid.query.filter.DimFilter; import io.druid.query.spec.MultipleIntervalSegmentSpec; import io.druid.query.spec.QuerySegmentSpec; import org.joda.time.DateTime; @@ -79,6 +80,12 @@ public boolean hasFilters() return false; } + @Override + public DimFilter getFilter() + { + return null; + } + @Override public String getType() { diff --git a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java index c2c71e5d7a02..60c32e482a60 100644 --- a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java @@ -74,6 +74,12 @@ public boolean hasFilters() return dimFilter != null; } + @Override + public DimFilter getFilter() + { + return dimFilter; + } + @Override public String getType() { diff --git a/processing/src/main/java/io/druid/query/topn/TopNQuery.java b/processing/src/main/java/io/druid/query/topn/TopNQuery.java index 86046748fc67..c06a7134adda 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNQuery.java +++ b/processing/src/main/java/io/druid/query/topn/TopNQuery.java @@ -91,6 +91,12 @@ public boolean hasFilters() return dimFilter != null; } + @Override + public DimFilter getFilter() + { + return dimFilter; + } + @Override public String getType() { diff --git a/server/src/main/java/io/druid/client/CachingClusteredClient.java b/server/src/main/java/io/druid/client/CachingClusteredClient.java index d4e1e7e6ee6e..de67a2dc2794 100644 --- a/server/src/main/java/io/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/io/druid/client/CachingClusteredClient.java @@ -29,6 +29,7 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.RangeSet; import com.google.common.collect.Sets; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; @@ -60,6 +61,7 @@ import io.druid.query.Result; import io.druid.query.SegmentDescriptor; import io.druid.query.aggregation.MetricManipulatorFns; +import io.druid.query.filter.DimFilter; import io.druid.query.spec.MultipleSpecificSegmentSpec; import io.druid.server.coordination.DruidServerMetadata; import io.druid.timeline.DataSegment; @@ -541,6 +543,14 @@ protected Sequence mergeCachedAndUncachedSequences( ); } + protected > List> filterSecondary ( + List> serverLookup, + RangeSet filterRangeSet + ) + { + return null; + } + private static class CachePopulator { private final Cache cache; From d6bbc5c03301e9efb19128698c09ac6f4a6a231e Mon Sep 17 00:00:00 2001 From: Dave Li Date: Wed, 11 May 2016 12:15:05 -0700 Subject: [PATCH 2/9] add get domain to ShardSpec and added chunk filter in caching clustered client --- .../timeline/partition/NoneShardSpec.java | 9 ++++ .../druid/timeline/partition/ShardSpec.java | 4 ++ .../partition/SingleDimensionShardData.java | 34 +++++++++++++++ .../io/druid/timeline/DataSegmentTest.java | 7 ++++ .../druid/client/CachingClusteredClient.java | 42 +++++++++++++------ .../partition/HashBasedNumberedShardSpec.java | 9 ++++ .../timeline/partition/LinearShardSpec.java | 9 ++++ .../timeline/partition/NumberedShardSpec.java | 9 ++++ .../partition/SingleDimensionShardSpec.java | 14 +++++++ 9 files changed, 125 insertions(+), 12 deletions(-) create mode 100644 api/src/main/java/io/druid/timeline/partition/SingleDimensionShardData.java diff --git a/api/src/main/java/io/druid/timeline/partition/NoneShardSpec.java b/api/src/main/java/io/druid/timeline/partition/NoneShardSpec.java index e33dd2afc854..c980e32ea6d4 100644 --- a/api/src/main/java/io/druid/timeline/partition/NoneShardSpec.java +++ b/api/src/main/java/io/druid/timeline/partition/NoneShardSpec.java @@ -20,9 +20,12 @@ package io.druid.timeline.partition; import com.fasterxml.jackson.annotation.JsonCreator; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.RangeSet; import io.druid.data.input.InputRow; import java.util.List; +import java.util.Map; /** */ @@ -71,6 +74,12 @@ public ShardSpec getShardSpec(long timestamp, InputRow row) }; } + @Override + public Map> getDomain() + { + return ImmutableMap.of(); + } + @Override public boolean equals(Object obj) { diff --git a/api/src/main/java/io/druid/timeline/partition/ShardSpec.java b/api/src/main/java/io/druid/timeline/partition/ShardSpec.java index 9be668e57053..72a88fa28b78 100644 --- a/api/src/main/java/io/druid/timeline/partition/ShardSpec.java +++ b/api/src/main/java/io/druid/timeline/partition/ShardSpec.java @@ -21,9 +21,11 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.collect.RangeSet; import io.druid.data.input.InputRow; import java.util.List; +import java.util.Map; /** * A Marker interface that exists to combine ShardSpec objects together for Jackson @@ -41,4 +43,6 @@ public interface ShardSpec public int getPartitionNum(); public ShardSpecLookup getLookup(List shardSpecs); + + public Map> getDomain(); } diff --git a/api/src/main/java/io/druid/timeline/partition/SingleDimensionShardData.java b/api/src/main/java/io/druid/timeline/partition/SingleDimensionShardData.java new file mode 100644 index 000000000000..1819650f2a84 --- /dev/null +++ b/api/src/main/java/io/druid/timeline/partition/SingleDimensionShardData.java @@ -0,0 +1,34 @@ +package io.druid.timeline.partition; + +import com.google.common.collect.Range; + +/** + */ +public class SingleDimensionShardData +{ + private final String dimension; + private final String start; + private final String finish; + + public SingleDimensionShardData(String dimension, String start, String finish) { + this.dimension = dimension; + this.start = start; + this.finish = finish; + } + + public String getDimension() { + return dimension; + } + + public String getStart() { + return start; + } + + public String getFinish() { + return finish; + } + + public Range getRange() { + return Range.closed(start, finish); + } +} diff --git a/api/src/test/java/io/druid/timeline/DataSegmentTest.java b/api/src/test/java/io/druid/timeline/DataSegmentTest.java index 7e1b04f9623c..a958b8a03c4b 100644 --- a/api/src/test/java/io/druid/timeline/DataSegmentTest.java +++ b/api/src/test/java/io/druid/timeline/DataSegmentTest.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.google.common.collect.RangeSet; import com.google.common.collect.Sets; import io.druid.TestObjectMapper; import io.druid.data.input.InputRow; @@ -76,6 +77,12 @@ public ShardSpecLookup getLookup(List shardSpecs) { return null; } + + @Override + public Map> getDomain() + { + return ImmutableMap.of(); + } }; } diff --git a/server/src/main/java/io/druid/client/CachingClusteredClient.java b/server/src/main/java/io/druid/client/CachingClusteredClient.java index de67a2dc2794..913a0f890513 100644 --- a/server/src/main/java/io/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/io/druid/client/CachingClusteredClient.java @@ -31,6 +31,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.RangeSet; import com.google.common.collect.Sets; +import com.google.common.collect.TreeRangeSet; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -68,6 +69,7 @@ import io.druid.timeline.TimelineLookup; import io.druid.timeline.TimelineObjectHolder; import io.druid.timeline.partition.PartitionChunk; +import io.druid.timeline.partition.SingleDimensionShardData; import org.joda.time.Interval; import java.io.IOException; @@ -221,14 +223,38 @@ public Sequence run(final Query query, final Map responseC final List> filteredServersLookup = toolChest.filterSegments(query, serversLookup); + List dimensions = Lists.newArrayList(); + Map> dimensionRangeMap = Maps.newHashMap(); + DimFilter filter = query.getFilter(); + if (filter != null) { + for (String dimension : dimensions) { + dimensionRangeMap.put(dimension, filter.getDimensionRangeSet(dimension)); + } + } + + // Filter unneeded chunks based on partition dimension for (TimelineObjectHolder holder : filteredServersLookup) { for (PartitionChunk chunk : holder.getObject()) { ServerSelector selector = chunk.getObject(); - final SegmentDescriptor descriptor = new SegmentDescriptor( - holder.getInterval(), holder.getVersion(), chunk.getChunkNumber() - ); + boolean include = true; + + if (filter != null) { + Map> domain = selector.getSegment().getShardSpec().getDomain(); + for (Map.Entry> entry : domain.entrySet()) { + RangeSet intersectRange = TreeRangeSet.create(dimensionRangeMap.get(entry.getKey())); + intersectRange.removeAll(entry.getValue().complement()); + if (intersectRange.isEmpty()) { + include = false; + } + } + } - segments.add(Pair.of(selector, descriptor)); + if (include) { + final SegmentDescriptor descriptor = new SegmentDescriptor( + holder.getInterval(), holder.getVersion(), chunk.getChunkNumber() + ); + segments.add(Pair.of(selector, descriptor)); + } } } @@ -543,14 +569,6 @@ protected Sequence mergeCachedAndUncachedSequences( ); } - protected > List> filterSecondary ( - List> serverLookup, - RangeSet filterRangeSet - ) - { - return null; - } - private static class CachePopulator { private final Cache cache; diff --git a/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java b/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java index 49b55de0a2c4..729d1f973769 100644 --- a/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java @@ -28,7 +28,9 @@ import com.google.common.base.Function; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.google.common.collect.RangeSet; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; import io.druid.data.input.InputRow; @@ -36,6 +38,7 @@ import javax.annotation.Nullable; import java.util.List; +import java.util.Map; public class HashBasedNumberedShardSpec extends NumberedShardSpec { @@ -121,4 +124,10 @@ public ShardSpec getShardSpec(long timestamp, InputRow row) } }; } + + @Override + public Map> getDomain() + { + return ImmutableMap.of(); + } } diff --git a/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java b/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java index e51e6b87bdb7..d12d43a3347f 100644 --- a/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java @@ -22,9 +22,12 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.RangeSet; import io.druid.data.input.InputRow; import java.util.List; +import java.util.Map; public class LinearShardSpec implements ShardSpec { @@ -57,6 +60,12 @@ public ShardSpec getShardSpec(long timestamp, InputRow row) }; } + @Override + public Map> getDomain() + { + return ImmutableMap.of(); + } + @Override public PartitionChunk createChunk(T obj) { return new LinearPartitionChunk(partitionNum, obj); diff --git a/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java b/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java index 13ac73a21089..1c078118d0c7 100644 --- a/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java @@ -23,9 +23,12 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.RangeSet; import io.druid.data.input.InputRow; import java.util.List; +import java.util.Map; public class NumberedShardSpec implements ShardSpec { @@ -67,6 +70,12 @@ public ShardSpec getShardSpec(long timestamp, InputRow row) }; } + @Override + public Map> getDomain() + { + return ImmutableMap.of(); + } + @JsonProperty("partitions") public int getPartitions() { diff --git a/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java b/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java index d0aadd5303d6..f5b616dfa3af 100644 --- a/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java @@ -20,10 +20,16 @@ package io.druid.timeline.partition; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableRangeSet; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; import com.metamx.common.ISE; import io.druid.data.input.InputRow; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.TreeMap; /** * Class uses getters/setters to work around http://jira.codehaus.org/browse/MSHADE-92 @@ -112,6 +118,14 @@ public ShardSpec getShardSpec(long timestamp, InputRow row) }; } + @Override + public Map> getDomain() + { + Map> retMap = new HashMap<>(); + retMap.put(dimension, ImmutableRangeSet.of(Range.closed(start, end))); + return retMap; + } + public void setPartitionNum(int partitionNum) { this.partitionNum = partitionNum; From 15d2517d7b11dca80caac95550efb239982649bf Mon Sep 17 00:00:00 2001 From: Dave Li Date: Thu, 12 May 2016 11:37:40 -0700 Subject: [PATCH 3/9] add null check and modified not filter, started with unit test --- .../partition/SingleDimensionShardData.java | 34 --- .../io/druid/query/filter/AndDimFilter.java | 6 +- .../io/druid/query/filter/BoundDimFilter.java | 14 +- .../java/io/druid/query/filter/DimFilter.java | 9 + .../io/druid/query/filter/InDimFilter.java | 2 +- .../io/druid/query/filter/NotDimFilter.java | 20 ++ .../druid/query/filter/SelectorDimFilter.java | 2 +- .../filter/GetDimensionRangeSetTest.java | 139 +++++++++ .../druid/client/CachingClusteredClient.java | 20 +- .../partition/SingleDimensionShardSpec.java | 12 +- .../client/CachingClusteredClientTest.java | 278 ++++++++++++++++-- 11 files changed, 451 insertions(+), 85 deletions(-) delete mode 100644 api/src/main/java/io/druid/timeline/partition/SingleDimensionShardData.java create mode 100644 processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java diff --git a/api/src/main/java/io/druid/timeline/partition/SingleDimensionShardData.java b/api/src/main/java/io/druid/timeline/partition/SingleDimensionShardData.java deleted file mode 100644 index 1819650f2a84..000000000000 --- a/api/src/main/java/io/druid/timeline/partition/SingleDimensionShardData.java +++ /dev/null @@ -1,34 +0,0 @@ -package io.druid.timeline.partition; - -import com.google.common.collect.Range; - -/** - */ -public class SingleDimensionShardData -{ - private final String dimension; - private final String start; - private final String finish; - - public SingleDimensionShardData(String dimension, String start, String finish) { - this.dimension = dimension; - this.start = start; - this.finish = finish; - } - - public String getDimension() { - return dimension; - } - - public String getStart() { - return start; - } - - public String getFinish() { - return finish; - } - - public Range getRange() { - return Range.closed(start, finish); - } -} diff --git a/processing/src/main/java/io/druid/query/filter/AndDimFilter.java b/processing/src/main/java/io/druid/query/filter/AndDimFilter.java index 1cc1eb8f74de..716e12831408 100644 --- a/processing/src/main/java/io/druid/query/filter/AndDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/AndDimFilter.java @@ -77,12 +77,12 @@ public Filter toFilter() @Override public RangeSet getDimensionRangeSet(String dimension) { - RangeSet retSet = TreeRangeSet.create(); + RangeSet retSet = null; for (DimFilter field : fields) { RangeSet rangeSet = field.getDimensionRangeSet(dimension); if (rangeSet != null) { - if (retSet.isEmpty()) { - retSet.addAll(rangeSet); + if (retSet == null) { + retSet = TreeRangeSet.create(rangeSet); } else { retSet.removeAll(rangeSet.complement()); } diff --git a/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java b/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java index 2f6b9cc48bc3..3afdac5243cd 100644 --- a/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java @@ -173,12 +173,20 @@ public Filter toFilter() @Override public RangeSet getDimensionRangeSet(String dimension) { - if (!Objects.equals(getDimension(), dimension) || getExtractionFn() != null) { + if (!Objects.equals(getDimension(), dimension) || getExtractionFn() != null || alphaNumeric) { return null; } RangeSet retSet = TreeRangeSet.create(); - retSet.add(Range.range(getLower(), isLowerStrict() ? BoundType.OPEN : BoundType.CLOSED, - getUpper(), isUpperStrict() ? BoundType.OPEN : BoundType.CLOSED)); + Range range; + if (getLower() == null) { + range = isUpperStrict() ? Range.lessThan(getUpper()) : Range.atMost(getUpper()); + } else if (getUpper() == null) { + range = isLowerStrict() ? Range.greaterThan(getLower()) : Range.atLeast(getLower()); + } else { + range = Range.range(getLower(), isLowerStrict() ? BoundType.OPEN : BoundType.CLOSED, + getUpper(), isUpperStrict() ? BoundType.OPEN : BoundType.CLOSED); + } + retSet.add(range); return retSet; } diff --git a/processing/src/main/java/io/druid/query/filter/DimFilter.java b/processing/src/main/java/io/druid/query/filter/DimFilter.java index cd4f9a4d1409..01068928fbe4 100644 --- a/processing/src/main/java/io/druid/query/filter/DimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/DimFilter.java @@ -58,5 +58,14 @@ public interface DimFilter */ public Filter toFilter(); + /** + * Returns a RangeSet that represents the possible range of the input dimension for this DimFilter.This is + * applicable to filters that use dimensions such as select, in, bound, and logical filters such as and, or, not. + * Note that + * + * @param dimension name of the dimension to get range for + * @return a RangeSet that represent the possible range of the input dimension, or null if it is not possible to + * determine for this DimFilter. + */ public RangeSet getDimensionRangeSet (String dimension); } diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index fc96be11da25..021503aebc9a 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -181,7 +181,7 @@ public RangeSet getDimensionRangeSet(String dimension) } RangeSet retSet = TreeRangeSet.create(); for (String value : values) { - retSet.add(Range.singleton(value)); + retSet.add(Range.singleton(Strings.nullToEmpty(value))); } return retSet; } diff --git a/processing/src/main/java/io/druid/query/filter/NotDimFilter.java b/processing/src/main/java/io/druid/query/filter/NotDimFilter.java index b9a6f1d6c04e..ac72d9b9e84f 100644 --- a/processing/src/main/java/io/druid/query/filter/NotDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/NotDimFilter.java @@ -21,12 +21,16 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Function; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import com.google.common.collect.RangeSet; +import com.sun.org.apache.xpath.internal.operations.And; import io.druid.query.Druids; import io.druid.segment.filter.NotFilter; import java.nio.ByteBuffer; +import java.util.List; /** */ @@ -34,6 +38,11 @@ public class NotDimFilter implements DimFilter { final private DimFilter field; + final private Function negate = + new Function() { + public DimFilter apply(DimFilter filter) { return Druids.newNotDimFilterBuilder().field(filter).build(); } + }; + @JsonCreator public NotDimFilter( @JsonProperty("field") DimFilter field @@ -72,6 +81,17 @@ public Filter toFilter() @Override public RangeSet getDimensionRangeSet(String dimension) { + if (field instanceof AndDimFilter) { + List fields = ((AndDimFilter) field).getFields(); + return Druids.newOrDimFilterBuilder().fields(Lists.transform(fields, negate)).build().getDimensionRangeSet(dimension); + } + if (field instanceof OrDimFilter) { + List fields = ((OrDimFilter) field).getFields(); + return Druids.newAndDimFilterBuilder().fields(Lists.transform(fields, negate)).build().getDimensionRangeSet(dimension); + } + if (field instanceof NotDimFilter) { + return ((NotDimFilter) field).getField().getDimensionRangeSet(dimension); + } RangeSet rangeSet = field.getDimensionRangeSet(dimension); return rangeSet == null ? null : rangeSet.complement(); } diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index eba093704f32..1ca9ef035944 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -162,7 +162,7 @@ public RangeSet getDimensionRangeSet(String dimension) return null; } RangeSet retSet = TreeRangeSet.create(); - retSet.add(Range.singleton(value)); + retSet.add(Range.singleton(Strings.nullToEmpty(value))); return retSet; } diff --git a/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java b/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java new file mode 100644 index 000000000000..48c17dcd48fa --- /dev/null +++ b/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java @@ -0,0 +1,139 @@ +package io.druid.query.filter; + + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableRangeSet; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import io.druid.js.JavaScriptConfig; +import io.druid.query.extraction.IdentityExtractionFn; +import io.druid.query.search.search.ContainsSearchQuerySpec; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +public class GetDimensionRangeSetTest +{ + + private final DimFilter selector1 = new SelectorDimFilter("dim1", "a", null); + private final DimFilter selector2 = new SelectorDimFilter("dim1", "z", null); + private final DimFilter selector3 = new SelectorDimFilter("dim2", "c", null); + private final DimFilter selector4 = new SelectorDimFilter("dimWorld", "find", IdentityExtractionFn.getInstance()); + private final DimFilter selector5 = new SelectorDimFilter("dim1", null, null); + private final DimFilter in1 = new InDimFilter("dim1", ImmutableList.of("testing", "this", "filter", "tillend"), null); + private final DimFilter in2 = new InDimFilter("dim2", ImmutableList.of("again"), null); + private final DimFilter in3 = new InDimFilter("dim1", Arrays.asList("null", null), null); + private final DimFilter bound1 = new BoundDimFilter("dim1", "from", "to", false, false, false, null); + private final DimFilter bound2 = new BoundDimFilter("dim1", null, "tillend", false, false, false, null); + private final DimFilter bound3 = new BoundDimFilter("dim1", "notincluded", null, true, false, false, null); + private final DimFilter bound4 = new BoundDimFilter("dim2", "again", "exclusive", true, true, false, null); + private final DimFilter other1 = new RegexDimFilter("someDim", "pattern", null); + private final DimFilter other2 = new JavaScriptDimFilter("someOtherDim", "function(x) { return x }", null, + JavaScriptConfig.getDefault()); + private final DimFilter other3 = new SearchQueryDimFilter("dim", new ContainsSearchQuerySpec("a", true), null); + + private static final RangeSet all = rangeSet(ImmutableList.of(Range.all())); + private static final RangeSet empty = rangeSet(ImmutableList.>of()); + + @Test + public void testSimpleFilter () { + RangeSet expected1 = rangeSet(point("a")); + Assert.assertEquals(expected1, selector1.getDimensionRangeSet("dim1")); + Assert.assertNull(selector1.getDimensionRangeSet("dim2")); + + RangeSet expected2 = rangeSet(point("")); + Assert.assertEquals(expected2, selector5.getDimensionRangeSet("dim1")); + + RangeSet expected3 = rangeSet(ImmutableList.of(point("testing"), point("this"), point("filter"), point("tillend"))); + Assert.assertEquals(expected3, in1.getDimensionRangeSet("dim1")); + + RangeSet expected4 = rangeSet(ImmutableList.of(point("null"), point(""))); + Assert.assertEquals(expected4, in3.getDimensionRangeSet("dim1")); + + RangeSet expected5 = ImmutableRangeSet.of(Range.closed("from", "to")); + Assert.assertEquals(expected5, bound1.getDimensionRangeSet("dim1")); + + RangeSet expected6 = ImmutableRangeSet.of(Range.atMost("tillend")); + Assert.assertEquals(expected6, bound2.getDimensionRangeSet("dim1")); + + RangeSet expected7 = ImmutableRangeSet.of(Range.greaterThan("notincluded")); + Assert.assertEquals(expected7, bound3.getDimensionRangeSet("dim1")); + + Assert.assertNull(other1.getDimensionRangeSet("someDim")); + Assert.assertNull(other2.getDimensionRangeSet("someOtherDim")); + Assert.assertNull(other3.getDimensionRangeSet("dim")); + } + + @Test + public void testAndFilter () { + DimFilter and1 = new AndDimFilter(ImmutableList.of(selector1, selector2, in1)); + Assert.assertEquals(empty, and1.getDimensionRangeSet("dim1")); + Assert.assertNull(and1.getDimensionRangeSet("dim2")); + + DimFilter and2 = new AndDimFilter(ImmutableList.of(selector3, bound1, other1)); + RangeSet expected2 = rangeSet(ImmutableList.of(Range.closed("from", "to"))); + Assert.assertEquals(expected2, and2.getDimensionRangeSet("dim1")); + + DimFilter and3 = new AndDimFilter(ImmutableList.of(in2, bound1, bound2, bound3, bound4)); + RangeSet expected3 = rangeSet(Range.openClosed("notincluded", "tillend")); + Assert.assertEquals(expected3, and3.getDimensionRangeSet("dim1")); + Assert.assertEquals(empty, and3.getDimensionRangeSet("dim2")); + + DimFilter and4 = new AndDimFilter(ImmutableList.of(in3, bound3)); + RangeSet expected4 = rangeSet(point("null")); + Assert.assertEquals(expected4, and4.getDimensionRangeSet("dim1")); + + DimFilter and5 = new AndDimFilter(ImmutableList.of(and3, in1)); + RangeSet expected5 = rangeSet(ImmutableList.of(point("testing"), point("this"), point("tillend"))); + Assert.assertEquals(expected5, and5.getDimensionRangeSet("dim1")); + } + + @Test + public void testOrFilter () { + DimFilter or1 = new OrDimFilter(ImmutableList.of(selector1, selector2, selector5)); + RangeSet expected1 = rangeSet(ImmutableList.of(point(""), point("a"), point("z"))); + Assert.assertEquals(expected1, or1.getDimensionRangeSet("dim1")); + + DimFilter or2 = new OrDimFilter(ImmutableList.of(selector5, in1, in3)); + RangeSet expected2 = rangeSet(ImmutableList.of(point("testing"), point("this"), point("filter"), point("tillend"), + point("null"), point(""))); + Assert.assertEquals(expected2, or2.getDimensionRangeSet("dim1")); + + DimFilter or3 = new OrDimFilter(ImmutableList.of(bound1, bound2, bound3)); + Assert.assertEquals(all, or3.getDimensionRangeSet("dim1")); + + DimFilter or4 = new OrDimFilter(ImmutableList.of(selector1, selector2, selector3, selector4, selector5)); + Assert.assertNull(or4.getDimensionRangeSet("dim1")); + Assert.assertNull(or4.getDimensionRangeSet("dim2")); + } + + public void testNotFilter () { + DimFilter not1 = new NotDimFilter(selector1); + RangeSet expected1 = rangeSet(ImmutableList.of(Range.lessThan("a"), Range.greaterThan("a"))); + Assert.assertEquals(expected1, not1.getDimensionRangeSet("dim1")); + + } + + public void testCombinedFilter () { + + } + + private static Range point(String s) { + return Range.singleton(s); + } + + private static RangeSet rangeSet (Range ranges) { + return ImmutableRangeSet.of(ranges); + } + + private static RangeSet rangeSet (List> ranges) { + ImmutableRangeSet.Builder builder = ImmutableRangeSet.builder(); + for (Range range : ranges) { + builder.add(range); + } + return builder.build(); + } + +} diff --git a/server/src/main/java/io/druid/client/CachingClusteredClient.java b/server/src/main/java/io/druid/client/CachingClusteredClient.java index 913a0f890513..9c4b856e0b8f 100644 --- a/server/src/main/java/io/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/io/druid/client/CachingClusteredClient.java @@ -69,7 +69,6 @@ import io.druid.timeline.TimelineLookup; import io.druid.timeline.TimelineObjectHolder; import io.druid.timeline.partition.PartitionChunk; -import io.druid.timeline.partition.SingleDimensionShardData; import org.joda.time.Interval; import java.io.IOException; @@ -223,14 +222,8 @@ public Sequence run(final Query query, final Map responseC final List> filteredServersLookup = toolChest.filterSegments(query, serversLookup); - List dimensions = Lists.newArrayList(); Map> dimensionRangeMap = Maps.newHashMap(); DimFilter filter = query.getFilter(); - if (filter != null) { - for (String dimension : dimensions) { - dimensionRangeMap.put(dimension, filter.getDimensionRangeSet(dimension)); - } - } // Filter unneeded chunks based on partition dimension for (TimelineObjectHolder holder : filteredServersLookup) { @@ -241,10 +234,15 @@ public Sequence run(final Query query, final Map responseC if (filter != null) { Map> domain = selector.getSegment().getShardSpec().getDomain(); for (Map.Entry> entry : domain.entrySet()) { - RangeSet intersectRange = TreeRangeSet.create(dimensionRangeMap.get(entry.getKey())); - intersectRange.removeAll(entry.getValue().complement()); - if (intersectRange.isEmpty()) { - include = false; + if (dimensionRangeMap.get(entry.getKey()) == null) { + dimensionRangeMap.put(entry.getKey(), filter.getDimensionRangeSet(entry.getKey())); + } + if (dimensionRangeMap.get(entry.getKey()) != null) { + RangeSet intersectRange = TreeRangeSet.create(dimensionRangeMap.get(entry.getKey())); + intersectRange.removeAll(entry.getValue().complement()); + if (intersectRange.isEmpty()) { + include = false; + } } } } diff --git a/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java b/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java index f5b616dfa3af..208a29db45fd 100644 --- a/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java @@ -122,7 +122,17 @@ public ShardSpec getShardSpec(long timestamp, InputRow row) public Map> getDomain() { Map> retMap = new HashMap<>(); - retMap.put(dimension, ImmutableRangeSet.of(Range.closed(start, end))); + Range range; + if (start == null && end == null) { + range = Range.all(); + } else if (start == null) { + range = Range.atMost(end); + } else if (end == null) { + range = Range.atLeast(start); + } else { + range = Range.closed(start, end); + } + retMap.put(dimension, ImmutableRangeSet.of(range)); return retMap; } diff --git a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java index 1ca49b2a1c40..27d035420e87 100644 --- a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java +++ b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java @@ -87,7 +87,10 @@ import io.druid.query.aggregation.post.FieldAccessPostAggregator; import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.dimension.DimensionSpec; +import io.druid.query.filter.BoundDimFilter; import io.druid.query.filter.DimFilter; +import io.druid.query.filter.InDimFilter; +import io.druid.query.filter.SelectorDimFilter; import io.druid.query.groupby.GroupByQuery; import io.druid.query.groupby.GroupByQueryConfig; import io.druid.query.groupby.GroupByQueryEngine; @@ -103,6 +106,7 @@ import io.druid.query.select.SelectQueryQueryToolChest; import io.druid.query.select.SelectResultValue; import io.druid.query.spec.MultipleIntervalSegmentSpec; +import io.druid.query.spec.MultipleSpecificSegmentSpec; import io.druid.query.timeboundary.TimeBoundaryQuery; import io.druid.query.timeboundary.TimeBoundaryQueryQueryToolChest; import io.druid.query.timeboundary.TimeBoundaryResultValue; @@ -120,6 +124,7 @@ import io.druid.timeline.partition.NoneShardSpec; import io.druid.timeline.partition.PartitionChunk; import io.druid.timeline.partition.ShardSpec; +import io.druid.timeline.partition.SingleDimensionShardSpec; import io.druid.timeline.partition.SingleElementPartitionChunk; import io.druid.timeline.partition.StringPartitionChunk; import org.easymock.Capture; @@ -137,6 +142,7 @@ import javax.annotation.Nullable; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -719,6 +725,7 @@ client, new TimeseriesQueryQueryToolChest( "populateCache", "true" ) ).build(), + null, new Interval("2011-01-01/2011-01-02"), makeTimeResults(new DateTime("2011-01-01"), 50, 5000) ); @@ -738,6 +745,7 @@ client, new TimeseriesQueryQueryToolChest( "populateCache", "false" ) ).build(), + null, new Interval("2011-01-01/2011-01-02"), makeTimeResults(new DateTime("2011-01-01"), 50, 5000) ); @@ -755,6 +763,7 @@ client, new TimeseriesQueryQueryToolChest( "populateCache", "false" ) ).build(), + null, new Interval("2011-01-01/2011-01-02"), makeTimeResults(new DateTime("2011-01-01"), 50, 5000) ); @@ -1430,6 +1439,198 @@ public void testTimeBoundaryCaching() throws Exception ); } + @Test + public void testTimeSeriesWithFilter () { + + DimFilter filter = Druids.newAndDimFilterBuilder() + .fields( + Arrays.asList( +// Druids.newOrDimFilterBuilder().fields( +// Arrays.asList( + new SelectorDimFilter("dim0", "1", null) +// new BoundDimFilter("dim0", "222", "333", true, true, false, null) +// ) +// ).build(), +// Druids.newAndDimFilterBuilder().fields( +// Arrays.asList( +// new InDimFilter("dim1", Arrays.asList("1", "3", "5", "6", "7"), null), +// new BoundDimFilter("dim1", "1", "7", true, true, false, null), +// new BoundDimFilter("dim1", "1", "9999", false, false, false, null) +// ) +// ).build() + ) + ) + .build(); + + final Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder() + .dataSource(DATA_SOURCE) + .intervals(SEG_SPEC) + .filters(new SelectorDimFilter("dim0", "1", null)) + .granularity(GRANULARITY) + .aggregators(AGGS) + .postAggregators(POST_AGGS) + .context(CONTEXT); + + QueryRunner runner = new FinalizeResultsQueryRunner( + client, new TimeseriesQueryQueryToolChest( + QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() + ) + ); + + List>> expectedResult = Arrays.asList( + makeTimeResults(new DateTime("2011-01-02"), 10, 1252, + new DateTime("2011-01-03"), 20, 6213), + Collections.>emptyList() + ); + + testQueryCaching( + runner, + 3, + true, + builder.build(), + expectedResult, + new Interval("2011-01-01/2011-01-05"), makeTimeResults(new DateTime("2011-01-01"), 50, 5000), + new Interval("2011-01-01/2011-01-05"), makeTimeResults(new DateTime("2011-01-02"), 10, 1252), + new Interval("2011-01-01/2011-01-05"), makeTimeResults(new DateTime("2011-01-03"), 20, 6213), + new Interval("2011-01-01/2011-01-05"), makeTimeResults(new DateTime("2011-01-04"), 30, 743), + new Interval("2011-01-01/2011-01-05"), makeTimeResults(new DateTime("2011-01-05"), 40, 6000), + new Interval("2011-01-06/2011-01-10"), makeTimeResults(new DateTime("2011-01-06"), 50, 425), + new Interval("2011-01-06/2011-01-10"), makeTimeResults(new DateTime("2011-01-07"), 60, 6020), + new Interval("2011-01-06/2011-01-10"), makeTimeResults(new DateTime("2011-01-08"), 70, 250), + new Interval("2011-01-06/2011-01-10"), makeTimeResults(new DateTime("2011-01-09"), 23, 85312) + ); + + +// HashMap context = new HashMap(); +// TestHelper.assertExpectedResults( +// makeRenamedTimeResults( +// //new DateTime("2011-01-01"), 0, 5000, +// new DateTime("2011-01-01"), 100, 1252, +// //new DateTime("2011-01-02"), 20, 6213, +// new DateTime("2011-01-02"), 30, 743, +// //new DateTime("2011-01-02"), 40, 6000, +// //new DateTime("2011-01-02"), 50, 425, +// new DateTime("2011-01-02"), 60, 6020, +// //new DateTime("2011-01-02"), 70, 250, +// new DateTime("2011-01-04"), 23, 85312 +// ), +// runner.run( +// builder.intervals("2011-01-01/2011-01-10") +// .aggregators(RENAMED_AGGS) +// .postAggregators(RENAMED_POST_AGGS) +// .build(), +// context +// ) +// ); + + } + + @Test + public void testSingleDimensionPruning () { + + DimFilter filter = Druids.newAndDimFilterBuilder() + .fields( + Arrays.asList( + Druids.newOrDimFilterBuilder().fields( + Arrays.asList( + new SelectorDimFilter("dim1", "a", null), + new BoundDimFilter("dim1", "from", "to", false, false, false, null) + ) + ).build(), + Druids.newAndDimFilterBuilder().fields( + Arrays.asList( + new InDimFilter("dim2", Arrays.asList("a", "c", "e", "g"), null), + new BoundDimFilter("dim2", "aaa", "hi", false, false, false, null), + new BoundDimFilter("dim2", "e", "zzz", true, true, false, null) + ) + ).build() + ) + ) + .build(); + + final Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder() + .dataSource(DATA_SOURCE) + .filters(filter) + .granularity(GRANULARITY) + .intervals(SEG_SPEC) + .context(CONTEXT) + .intervals("2011-01-05/2011-01-10") + .aggregators(RENAMED_AGGS) + .postAggregators(RENAMED_POST_AGGS); + + TimeseriesQuery query = builder.build(); + Map context = new HashMap<>(); + + final Interval interval1 = new Interval("2011-01-06/2011-01-07"); + final Interval interval2 = new Interval("2011-01-07/2011-01-08"); + final Interval interval3 = new Interval("2011-01-08/2011-01-09"); + + QueryRunner runner = new FinalizeResultsQueryRunner( + client, new TimeseriesQueryQueryToolChest( + QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() + ) + ); + + final DruidServer lastServer = servers[random.nextInt(servers.length)]; + ServerSelector selector1 = makeMockSingleDimensionSelector(lastServer, "dim1", null, "b", 1); + ServerSelector selector2 = makeMockSingleDimensionSelector(lastServer, "dim1", "e", "f", 2); + ServerSelector selector3 = makeMockSingleDimensionSelector(lastServer, "dim1", "hi", "zzz", 3); + ServerSelector selector4 = makeMockSingleDimensionSelector(lastServer, "dim2", "a", "e", 4); + ServerSelector selector5 = makeMockSingleDimensionSelector(lastServer, "dim2", null, null, 5); + ServerSelector selector6 = makeMockSingleDimensionSelector(lastServer, "other", "b", null, 6); + + timeline.add(interval1, "v", new StringPartitionChunk<>(null, "a", 1, selector1)); + timeline.add(interval1, "v", new StringPartitionChunk<>("a", "b", 2, selector2)); + timeline.add(interval1, "v", new StringPartitionChunk<>("b", null, 3, selector3)); + timeline.add(interval2, "v", new StringPartitionChunk<>(null, "d", 4, selector4)); + timeline.add(interval2, "v", new StringPartitionChunk<>("d", null, 5, selector5)); + timeline.add(interval3, "v", new StringPartitionChunk<>(null, null, 6, selector6)); + + final Capture capture = Capture.newInstance(); + final Capture> contextCap = Capture.newInstance(); + + QueryRunner mockRunner = EasyMock.createNiceMock(QueryRunner.class); + EasyMock.expect(mockRunner.run(EasyMock.capture(capture), EasyMock.capture(contextCap))) + .andReturn(Sequences.empty()) + .anyTimes(); + EasyMock.expect(serverView.getQueryRunner(lastServer)) + .andReturn(mockRunner) + .anyTimes(); + EasyMock.replay(serverView); + EasyMock.replay(mockRunner); + + List descriptors = new ArrayList<>(); + descriptors.add(new SegmentDescriptor(interval1, "v", 1)); + descriptors.add(new SegmentDescriptor(interval1, "v", 3)); + descriptors.add(new SegmentDescriptor(interval2, "v", 5)); + descriptors.add(new SegmentDescriptor(interval3, "v", 6)); + MultipleSpecificSegmentSpec expected = new MultipleSpecificSegmentSpec(descriptors); + + Sequences.toList(runner.run( + query, + context + ), Lists.newArrayList()); + + Assert.assertEquals(expected, capture.getValue().getQuerySegmentSpec()); + } + + private ServerSelector makeMockSingleDimensionSelector( + DruidServer server, String dimension, String start, String end, int partitionNum) + { + DataSegment segment = EasyMock.createNiceMock(DataSegment.class); + EasyMock.expect(segment.getIdentifier()).andReturn(DATA_SOURCE).anyTimes(); + EasyMock.expect(segment.getShardSpec()).andReturn(new SingleDimensionShardSpec(dimension, start, end, partitionNum)) + .anyTimes(); + EasyMock.replay(segment); + + ServerSelector selector = new ServerSelector( + segment, + new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()) + ); + selector.addServerAndUpdateSegment(new QueryableDruidServer(server, null), segment); + return selector; + } + private Iterable> makeTimeBoundaryResult( DateTime timestamp, DateTime minTime, @@ -1466,7 +1667,7 @@ private Iterable> makeTimeBoundaryResult( public void testQueryCaching(QueryRunner runner, final Query query, Object... args) { - testQueryCaching(runner, 3, true, query, args); + testQueryCaching(runner, 3, true, query, null, args); } @SuppressWarnings("unchecked") @@ -1475,6 +1676,7 @@ public void testQueryCaching( final int numTimesToQuery, boolean expectBySegment, final Query query, + final List>> filteredExpected, Object... args // does this assume query intervals must be ordered? ) { @@ -1618,6 +1820,39 @@ public void testQueryCaching( expectedResultsRangeEnd = i + 1; } + final Iterable> expected; + if (filteredExpected != null) { + expected = new ArrayList<>(); + for (int intervalNo = expectedResultsRangeStart; intervalNo < expectedResultsRangeEnd; intervalNo++) { + Iterables.addAll((List)expected, filteredExpected.get(intervalNo)); + } + } else { + expected = new MergeIterable<>( + Ordering.>natural().nullsFirst(), + FunctionalIterable + .create(new RangeIterable(expectedResultsRangeStart, expectedResultsRangeEnd)) + .transformCat( + new Function>>>() + { + @Override + public Iterable>> apply(@Nullable Integer input) + { + List>> retVal = Lists.newArrayList(); + + final Map exps = serverExpectationList.get(input); + for (ServerExpectations expectations : exps.values()) { + for (ServerExpectation expectation : expectations) { + retVal.add(expectation.getResults()); + } + } + + return retVal; + } + } + ) + ); + } + runWithMocks( new Runnable() { @@ -1627,30 +1862,7 @@ public void run() HashMap context = new HashMap(); for (int i = 0; i < numTimesToQuery; ++i) { TestHelper.assertExpectedResults( - new MergeIterable<>( - Ordering.>natural().nullsFirst(), - FunctionalIterable - .create(new RangeIterable(expectedResultsRangeStart, expectedResultsRangeEnd)) - .transformCat( - new Function>>>() - { - @Override - public Iterable>> apply(@Nullable Integer input) - { - List>> retVal = Lists.newArrayList(); - - final Map exps = serverExpectationList.get(input); - for (ServerExpectations expectations : exps.values()) { - for (ServerExpectation expectation : expectations) { - retVal.add(expectation.getResults()); - } - } - - return retVal; - } - } - ) - ), + expected, runner.run( query.withQuerySegmentSpec( new MultipleIntervalSegmentSpec( @@ -1707,10 +1919,11 @@ private List> populateTimeline( serverExpectations.put(lastServer, new ServerExpectations(lastServer, makeMock(mocks, QueryRunner.class))); } + DataSegment mockSegment = makeMock(mocks, DataSegment.class); ServerExpectation expectation = new ServerExpectation( String.format("%s_%s", k, j), // interval/chunk queryIntervals.get(numQueryIntervals), - makeMock(mocks, DataSegment.class), + mockSegment, expectedResults.get(k).get(j) ); serverExpectations.get(lastServer).addExpectation(expectation); @@ -1721,9 +1934,9 @@ private List> populateTimeline( ); selector.addServerAndUpdateSegment(new QueryableDruidServer(lastServer, null), selector.getSegment()); - final PartitionChunk chunk; + final ShardSpec shardSpec; if (numChunks == 1) { - chunk = new SingleElementPartitionChunk<>(selector); + shardSpec = new SingleDimensionShardSpec("dimAll", null, null, 0); } else { String start = null; String end = null; @@ -1733,9 +1946,12 @@ private List> populateTimeline( if (j + 1 < numChunks) { end = String.valueOf(j); } - chunk = new StringPartitionChunk<>(start, end, j, selector); + shardSpec = new SingleDimensionShardSpec("dim"+k, start, end, j); } - timeline.add(queryIntervals.get(k), String.valueOf(k), chunk); + EasyMock.expect(mockSegment.getShardSpec()) + .andReturn(shardSpec) + .anyTimes(); + timeline.add(queryIntervals.get(k), String.valueOf(k), shardSpec.createChunk(selector)); } } return serverExpectationList; @@ -2167,7 +2383,7 @@ private void runWithMocks(Runnable toRun, Object... mocks) toRun.run(); - EasyMock.verify(mocks); +// EasyMock.verify(mocks); EasyMock.reset(mocks); } From e0e1d9f53149d976f80daaea13b617098f334bec Mon Sep 17 00:00:00 2001 From: Dave Li Date: Tue, 17 May 2016 16:43:32 -0700 Subject: [PATCH 4/9] add filter test with caching --- .../filter/GetDimensionRangeSetTest.java | 49 ++- .../client/CachingClusteredClientTest.java | 314 ++++++++++++------ 2 files changed, 259 insertions(+), 104 deletions(-) diff --git a/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java b/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java index 48c17dcd48fa..a972b0e1a58d 100644 --- a/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java +++ b/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java @@ -10,6 +10,7 @@ import io.druid.query.search.search.ContainsSearchQuerySpec; import org.junit.Assert; import org.junit.Test; +import org.mozilla.javascript.tools.debugger.Dim; import java.util.Arrays; import java.util.List; @@ -107,16 +108,56 @@ public void testOrFilter () { DimFilter or4 = new OrDimFilter(ImmutableList.of(selector1, selector2, selector3, selector4, selector5)); Assert.assertNull(or4.getDimensionRangeSet("dim1")); Assert.assertNull(or4.getDimensionRangeSet("dim2")); + + DimFilter or5 = new OrDimFilter(ImmutableList.of(or1, or2, bound1)); + RangeSet expected5 = rangeSet(ImmutableList.of(point(""), point("a"), point("filter"), Range.closed("from", "to"), + point("z"))); + Assert.assertEquals(expected5, or5.getDimensionRangeSet("dim1")); } + @Test public void testNotFilter () { DimFilter not1 = new NotDimFilter(selector1); RangeSet expected1 = rangeSet(ImmutableList.of(Range.lessThan("a"), Range.greaterThan("a"))); Assert.assertEquals(expected1, not1.getDimensionRangeSet("dim1")); - - } - - public void testCombinedFilter () { + Assert.assertNull(not1.getDimensionRangeSet("dim2")); + + DimFilter not2 = new NotDimFilter(in3); + RangeSet expected2 = rangeSet(ImmutableList.of(Range.lessThan(""), Range.open("", "null"), Range.greaterThan("null"))); + Assert.assertEquals(expected2, not2.getDimensionRangeSet("dim1")); + + DimFilter not3 = new NotDimFilter(bound1); + RangeSet expected3 = rangeSet(ImmutableList.of(Range.lessThan("from"), Range.greaterThan("to"))); + Assert.assertEquals(expected3, not3.getDimensionRangeSet("dim1")); + + DimFilter not4 = new NotDimFilter(not2); + RangeSet expected4 = rangeSet(ImmutableList.of(point(""), point("null"))); + Assert.assertEquals(expected4, not4.getDimensionRangeSet("dim1")); + + DimFilter or1 = new OrDimFilter(ImmutableList.of(selector1, selector2, bound1, bound3)); + DimFilter not5 = new NotDimFilter(or1); + RangeSet expected5 = rangeSet(ImmutableList.of(Range.lessThan("a"), Range.open("a", "from"))); + Assert.assertEquals(expected5, not5.getDimensionRangeSet("dim1")); + Assert.assertNull(not5.getDimensionRangeSet("dim2")); + + DimFilter or2 = new OrDimFilter(ImmutableList.of(selector3, in3, bound2, bound4, other3)); + DimFilter not6 = new NotDimFilter(or2); + RangeSet expected6a = rangeSet(ImmutableList.of(Range.greaterThan("tillend"))); + RangeSet expected6b = rangeSet(ImmutableList.of(Range.atMost("again"), Range.atLeast("exclusive"))); + Assert.assertEquals(expected6a, not6.getDimensionRangeSet("dim1")); + Assert.assertEquals(expected6b, not6.getDimensionRangeSet("dim2")); + + DimFilter and1 = new AndDimFilter(ImmutableList.of(in1, bound1, bound2)); + DimFilter not7 = new NotDimFilter(and1); + RangeSet expected7 = rangeSet(ImmutableList.of(Range.lessThan("testing"), Range.open("testing", "this"), + Range.open("this", "tillend"), Range.greaterThan("tillend"))); + Assert.assertEquals(expected7, not7.getDimensionRangeSet("dim1")); + Assert.assertNull(not7.getDimensionRangeSet("dim2")); + + DimFilter and2 = new AndDimFilter(ImmutableList.of(bound1, bound2, bound3, bound4)); + DimFilter not8 = new NotDimFilter(and2); + Assert.assertNull(not8.getDimensionRangeSet("dim1")); + Assert.assertNull(not8.getDimensionRangeSet("dim2")); } diff --git a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java index 27d035420e87..e98a1907b41c 100644 --- a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java +++ b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java @@ -63,6 +63,7 @@ import io.druid.granularity.QueryGranularity; import io.druid.granularity.QueryGranularities; import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.BaseQuery; import io.druid.query.BySegmentResultValueClass; import io.druid.query.DataSource; import io.druid.query.Druids; @@ -129,6 +130,7 @@ import io.druid.timeline.partition.StringPartitionChunk; import org.easymock.Capture; import org.easymock.EasyMock; +import org.easymock.IAnswer; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Interval; @@ -725,7 +727,6 @@ client, new TimeseriesQueryQueryToolChest( "populateCache", "true" ) ).build(), - null, new Interval("2011-01-01/2011-01-02"), makeTimeResults(new DateTime("2011-01-01"), 50, 5000) ); @@ -745,7 +746,6 @@ client, new TimeseriesQueryQueryToolChest( "populateCache", "false" ) ).build(), - null, new Interval("2011-01-01/2011-01-02"), makeTimeResults(new DateTime("2011-01-01"), 50, 5000) ); @@ -763,7 +763,6 @@ client, new TimeseriesQueryQueryToolChest( "populateCache", "false" ) ).build(), - null, new Interval("2011-01-01/2011-01-02"), makeTimeResults(new DateTime("2011-01-01"), 50, 5000) ); @@ -1444,20 +1443,20 @@ public void testTimeSeriesWithFilter () { DimFilter filter = Druids.newAndDimFilterBuilder() .fields( - Arrays.asList( -// Druids.newOrDimFilterBuilder().fields( -// Arrays.asList( - new SelectorDimFilter("dim0", "1", null) -// new BoundDimFilter("dim0", "222", "333", true, true, false, null) -// ) -// ).build(), -// Druids.newAndDimFilterBuilder().fields( -// Arrays.asList( -// new InDimFilter("dim1", Arrays.asList("1", "3", "5", "6", "7"), null), -// new BoundDimFilter("dim1", "1", "7", true, true, false, null), -// new BoundDimFilter("dim1", "1", "9999", false, false, false, null) -// ) -// ).build() + Arrays.asList( + Druids.newOrDimFilterBuilder().fields( + Arrays.asList( + new SelectorDimFilter("dim0", "1", null), + new BoundDimFilter("dim0", "222", "333", false, false, false, null) + ) + ).build(), + Druids.newAndDimFilterBuilder().fields( + Arrays.asList( + new InDimFilter("dim1", Arrays.asList("0", "1", "2", "3", "4"), null), + new BoundDimFilter("dim1", "0", "3", false, true, false, null), + new BoundDimFilter("dim1", "1", "9999", true, false, false, null) + ) + ).build() ) ) .build(); @@ -1465,7 +1464,7 @@ public void testTimeSeriesWithFilter () { final Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder() .dataSource(DATA_SOURCE) .intervals(SEG_SPEC) - .filters(new SelectorDimFilter("dim0", "1", null)) + .filters(filter) .granularity(GRANULARITY) .aggregators(AGGS) .postAggregators(POST_AGGS) @@ -1477,16 +1476,24 @@ client, new TimeseriesQueryQueryToolChest( ) ); + /* + For dim0 (2011-01-01/2011-01-05), the combined range is {[1,1], [222,333]}, so segments [-inf,1], [1,2], [2,3], and + [3,4] is needed + For dim1 (2011-01-06/2011-01-10), the combined range for the bound filters is {(1,3)}, combined this with the in + filter result in {[2,2]}, so segments [1,2] and [2,3] is needed + */ List>> expectedResult = Arrays.asList( - makeTimeResults(new DateTime("2011-01-02"), 10, 1252, - new DateTime("2011-01-03"), 20, 6213), - Collections.>emptyList() + makeTimeResults(new DateTime("2011-01-01"), 50, 5000, + new DateTime("2011-01-02"), 10, 1252, + new DateTime("2011-01-03"), 20, 6213, + new DateTime("2011-01-04"), 30, 743), + makeTimeResults(new DateTime("2011-01-07"), 60, 6020, + new DateTime("2011-01-08"), 70, 250) ); - testQueryCaching( + testQueryCachingWithFilter( runner, 3, - true, builder.build(), expectedResult, new Interval("2011-01-01/2011-01-05"), makeTimeResults(new DateTime("2011-01-01"), 50, 5000), @@ -1497,32 +1504,10 @@ client, new TimeseriesQueryQueryToolChest( new Interval("2011-01-06/2011-01-10"), makeTimeResults(new DateTime("2011-01-06"), 50, 425), new Interval("2011-01-06/2011-01-10"), makeTimeResults(new DateTime("2011-01-07"), 60, 6020), new Interval("2011-01-06/2011-01-10"), makeTimeResults(new DateTime("2011-01-08"), 70, 250), - new Interval("2011-01-06/2011-01-10"), makeTimeResults(new DateTime("2011-01-09"), 23, 85312) + new Interval("2011-01-06/2011-01-10"), makeTimeResults(new DateTime("2011-01-09"), 23, 85312), + new Interval("2011-01-06/2011-01-10"), makeTimeResults(new DateTime("2011-01-10"), 100, 512) ); - -// HashMap context = new HashMap(); -// TestHelper.assertExpectedResults( -// makeRenamedTimeResults( -// //new DateTime("2011-01-01"), 0, 5000, -// new DateTime("2011-01-01"), 100, 1252, -// //new DateTime("2011-01-02"), 20, 6213, -// new DateTime("2011-01-02"), 30, 743, -// //new DateTime("2011-01-02"), 40, 6000, -// //new DateTime("2011-01-02"), 50, 425, -// new DateTime("2011-01-02"), 60, 6020, -// //new DateTime("2011-01-02"), 70, 250, -// new DateTime("2011-01-04"), 23, 85312 -// ), -// runner.run( -// builder.intervals("2011-01-01/2011-01-10") -// .aggregators(RENAMED_AGGS) -// .postAggregators(RENAMED_POST_AGGS) -// .build(), -// context -// ) -// ); - } @Test @@ -1665,39 +1650,178 @@ private Iterable> makeTimeBoundaryResult( ); } - public void testQueryCaching(QueryRunner runner, final Query query, Object... args) + public void parseResults ( + final List queryIntervals, + final List>>> expectedResults, + Object... args) { - testQueryCaching(runner, 3, true, query, null, args); + if (args.length % 2 != 0) { + throw new ISE("args.length must be divisible by two, was %d", args.length); + } + + for (int i = 0; i < args.length; i += 2) { + final Interval interval = (Interval) args[i]; + final Iterable> results = (Iterable>) args[i + 1]; + + if (queryIntervals.size() > 0 && interval.equals(queryIntervals.get(queryIntervals.size() - 1))) { + expectedResults.get(expectedResults.size() - 1).add(results); + } else { + queryIntervals.add(interval); + expectedResults.add(Lists.>>newArrayList(results)); + } + } } @SuppressWarnings("unchecked") - public void testQueryCaching( + public void testQueryCachingWithFilter( final QueryRunner runner, final int numTimesToQuery, - boolean expectBySegment, final Query query, final List>> filteredExpected, Object... args // does this assume query intervals must be ordered? ) { - if (args.length % 2 != 0) { - throw new ISE("args.length must be divisible by two, was %d", args.length); - } - final List queryIntervals = Lists.newArrayListWithCapacity(args.length / 2); final List>>> expectedResults = Lists.newArrayListWithCapacity(queryIntervals.size()); - for (int i = 0; i < args.length; i += 2) { - final Interval interval = (Interval) args[i]; - final Iterable> results = (Iterable>) args[i + 1]; + parseResults(queryIntervals, expectedResults, args); - if (queryIntervals.size() > 0 && interval.equals(queryIntervals.get(queryIntervals.size() - 1))) { - expectedResults.get(expectedResults.size() - 1).add(results); + for (int i = 0; i < queryIntervals.size(); ++i) { + List mocks = Lists.newArrayList(); + mocks.add(serverView); + + final Interval actualQueryInterval = new Interval( + queryIntervals.get(0).getStart(), queryIntervals.get(i).getEnd() + ); + + final List> serverExpectationList = populateTimeline( + queryIntervals, + expectedResults, + i, + mocks + ); + + final Map finalExpectation = serverExpectationList.get( + serverExpectationList.size() - 1 + ); + for (Map.Entry entry : finalExpectation.entrySet()) { + DruidServer server = entry.getKey(); + ServerExpectations expectations = entry.getValue(); + + EasyMock.expect(serverView.getQueryRunner(server)) + .andReturn(expectations.getQueryRunner()) + .times(0, 1); + + final Capture capture = new Capture(); + final Capture context = new Capture(); + QueryRunner queryable = expectations.getQueryRunner(); + + if (query instanceof TimeseriesQuery) { + final List segmentIds = Lists.newArrayList(); + final List>> results = Lists.newArrayList(); + for (ServerExpectation expectation : expectations) { + segmentIds.add(expectation.getSegmentId()); + results.add(expectation.getResults()); + } + EasyMock.expect(queryable.run(EasyMock.capture(capture), EasyMock.capture(context))) + .andAnswer(new IAnswer() + { + @Override + public Sequence answer() throws Throwable + { + return toFilteredQueryableTimeseriesResults((TimeseriesQuery)capture.getValue(), segmentIds, queryIntervals, results); + } + }) + .times(0, 1); + } else { + throw new ISE("Unknown query type[%s]", query.getClass()); + } + } + + final Iterable> expected = new ArrayList<>(); + for (int intervalNo = 0; intervalNo < i + 1; intervalNo++) { + Iterables.addAll((List)expected, filteredExpected.get(intervalNo)); + } + + runWithMocks( + new Runnable() + { + @Override + public void run() + { + HashMap context = new HashMap(); + for (int i = 0; i < numTimesToQuery; ++i) { + TestHelper.assertExpectedResults( + expected, + runner.run( + query.withQuerySegmentSpec( + new MultipleIntervalSegmentSpec( + ImmutableList.of( + actualQueryInterval + ) + ) + ), + context + ) + ); + if (queryCompletedCallback != null) { + queryCompletedCallback.run(); + } + } + } + }, + mocks.toArray() + ); + } + } + + private Sequence> toFilteredQueryableTimeseriesResults( + TimeseriesQuery query, + List segmentIds, + List queryIntervals, + List>> results + ) + { + MultipleSpecificSegmentSpec spec = (MultipleSpecificSegmentSpec)query.getQuerySegmentSpec(); + List> ret = Lists.newArrayList(); + for (SegmentDescriptor descriptor : spec.getDescriptors()) { + String id = String.format("%s_%s", queryIntervals.indexOf(descriptor.getInterval()), descriptor.getPartitionNumber()); + int index = segmentIds.indexOf(id); + if (index != -1) { + ret.add(new Result( + results.get(index).iterator().next().getTimestamp(), + new BySegmentResultValueClass( + Lists.newArrayList(results.get(index)), + id, + descriptor.getInterval() + ) + )); } else { - queryIntervals.add(interval); - expectedResults.add(Lists.>>newArrayList(results)); + throw new ISE("Descriptor %s not found in server", id); } } + return Sequences.simple(ret); + } + + public void testQueryCaching(QueryRunner runner, final Query query, Object... args) + { + testQueryCaching(runner, 3, true, query, args); + } + + @SuppressWarnings("unchecked") + public void testQueryCaching( + final QueryRunner runner, + final int numTimesToQuery, + boolean expectBySegment, + final Query query, + Object... args // does this assume query intervals must be ordered? + ) + { + + final List queryIntervals = Lists.newArrayListWithCapacity(args.length / 2); + final List>>> expectedResults = Lists.newArrayListWithCapacity(queryIntervals.size()); + + parseResults(queryIntervals, expectedResults, args); for (int i = 0; i < queryIntervals.size(); ++i) { List mocks = Lists.newArrayList(); @@ -1820,39 +1944,6 @@ public void testQueryCaching( expectedResultsRangeEnd = i + 1; } - final Iterable> expected; - if (filteredExpected != null) { - expected = new ArrayList<>(); - for (int intervalNo = expectedResultsRangeStart; intervalNo < expectedResultsRangeEnd; intervalNo++) { - Iterables.addAll((List)expected, filteredExpected.get(intervalNo)); - } - } else { - expected = new MergeIterable<>( - Ordering.>natural().nullsFirst(), - FunctionalIterable - .create(new RangeIterable(expectedResultsRangeStart, expectedResultsRangeEnd)) - .transformCat( - new Function>>>() - { - @Override - public Iterable>> apply(@Nullable Integer input) - { - List>> retVal = Lists.newArrayList(); - - final Map exps = serverExpectationList.get(input); - for (ServerExpectations expectations : exps.values()) { - for (ServerExpectation expectation : expectations) { - retVal.add(expectation.getResults()); - } - } - - return retVal; - } - } - ) - ); - } - runWithMocks( new Runnable() { @@ -1862,7 +1953,30 @@ public void run() HashMap context = new HashMap(); for (int i = 0; i < numTimesToQuery; ++i) { TestHelper.assertExpectedResults( - expected, + new MergeIterable<>( + Ordering.>natural().nullsFirst(), + FunctionalIterable + .create(new RangeIterable(expectedResultsRangeStart, expectedResultsRangeEnd)) + .transformCat( + new Function>>>() + { + @Override + public Iterable>> apply(@Nullable Integer input) + { + List>> retVal = Lists.newArrayList(); + + final Map exps = serverExpectationList.get(input); + for (ServerExpectations expectations : exps.values()) { + for (ServerExpectation expectation : expectations) { + retVal.add(expectation.getResults()); + } + } + + return retVal; + } + } + ) + ), runner.run( query.withQuerySegmentSpec( new MultipleIntervalSegmentSpec( @@ -1922,7 +2036,7 @@ private List> populateTimeline( DataSegment mockSegment = makeMock(mocks, DataSegment.class); ServerExpectation expectation = new ServerExpectation( String.format("%s_%s", k, j), // interval/chunk - queryIntervals.get(numQueryIntervals), + queryIntervals.get(k), mockSegment, expectedResults.get(k).get(j) ); @@ -1941,10 +2055,10 @@ private List> populateTimeline( String start = null; String end = null; if (j > 0) { - start = String.valueOf(j - 1); + start = String.valueOf(j); } if (j + 1 < numChunks) { - end = String.valueOf(j); + end = String.valueOf(j+1); } shardSpec = new SingleDimensionShardSpec("dim"+k, start, end, j); } @@ -2383,7 +2497,7 @@ private void runWithMocks(Runnable toRun, Object... mocks) toRun.run(); -// EasyMock.verify(mocks); + EasyMock.verify(mocks); EasyMock.reset(mocks); } From c06e631443fc5c3fdd1a83a213459f7f4484b35e Mon Sep 17 00:00:00 2001 From: Dave Li Date: Thu, 16 Jun 2016 19:11:53 -0700 Subject: [PATCH 5/9] refactor and some comments --- .../timeline/partition/NoneShardSpec.java | 4 +-- .../druid/timeline/partition/ShardSpec.java | 9 +++++-- .../io/druid/timeline/DataSegmentTest.java | 4 +-- .../java/io/druid/query/filter/DimFilter.java | 6 ++++- .../io/druid/query/filter/NotDimFilter.java | 17 +++++++++--- .../druid/client/CachingClusteredClient.java | 27 ++++++++++--------- .../partition/HashBasedNumberedShardSpec.java | 4 +-- .../timeline/partition/LinearShardSpec.java | 4 +-- .../timeline/partition/NumberedShardSpec.java | 4 +-- .../partition/SingleDimensionShardSpec.java | 10 +++---- 10 files changed, 53 insertions(+), 36 deletions(-) diff --git a/api/src/main/java/io/druid/timeline/partition/NoneShardSpec.java b/api/src/main/java/io/druid/timeline/partition/NoneShardSpec.java index c980e32ea6d4..4b894805cf9c 100644 --- a/api/src/main/java/io/druid/timeline/partition/NoneShardSpec.java +++ b/api/src/main/java/io/druid/timeline/partition/NoneShardSpec.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.RangeSet; +import com.google.common.collect.Range; import io.druid.data.input.InputRow; import java.util.List; @@ -75,7 +75,7 @@ public ShardSpec getShardSpec(long timestamp, InputRow row) } @Override - public Map> getDomain() + public Map> getDomain() { return ImmutableMap.of(); } diff --git a/api/src/main/java/io/druid/timeline/partition/ShardSpec.java b/api/src/main/java/io/druid/timeline/partition/ShardSpec.java index 72a88fa28b78..fdf9bce8d4e9 100644 --- a/api/src/main/java/io/druid/timeline/partition/ShardSpec.java +++ b/api/src/main/java/io/druid/timeline/partition/ShardSpec.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.google.common.collect.RangeSet; +import com.google.common.collect.Range; import io.druid.data.input.InputRow; import java.util.List; @@ -44,5 +44,10 @@ public interface ShardSpec public ShardSpecLookup getLookup(List shardSpecs); - public Map> getDomain(); + /** + * Get the possible range of each dimension for the rows this shard contains. + * + * @return map of dimensions to its possible range. Dimensions with unknown possible range are not mapped + */ + public Map> getDomain(); } diff --git a/api/src/test/java/io/druid/timeline/DataSegmentTest.java b/api/src/test/java/io/druid/timeline/DataSegmentTest.java index a958b8a03c4b..1da50c06b32a 100644 --- a/api/src/test/java/io/druid/timeline/DataSegmentTest.java +++ b/api/src/test/java/io/druid/timeline/DataSegmentTest.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.google.common.collect.RangeSet; +import com.google.common.collect.Range; import com.google.common.collect.Sets; import io.druid.TestObjectMapper; import io.druid.data.input.InputRow; @@ -79,7 +79,7 @@ public ShardSpecLookup getLookup(List shardSpecs) } @Override - public Map> getDomain() + public Map> getDomain() { return ImmutableMap.of(); } diff --git a/processing/src/main/java/io/druid/query/filter/DimFilter.java b/processing/src/main/java/io/druid/query/filter/DimFilter.java index 01068928fbe4..a1c595df68d0 100644 --- a/processing/src/main/java/io/druid/query/filter/DimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/DimFilter.java @@ -61,7 +61,11 @@ public interface DimFilter /** * Returns a RangeSet that represents the possible range of the input dimension for this DimFilter.This is * applicable to filters that use dimensions such as select, in, bound, and logical filters such as and, or, not. - * Note that + * + * Null represents that the range cannot be determined, and will be returned for filters such as javascript and regex + * where there's no easy way to determine the filtered range. It is treated the same way as an all range in most + * cases, however there are some subtle difference at logical filters such as not filter, where complement of all + * is nothing while complement of null is still null. * * @param dimension name of the dimension to get range for * @return a RangeSet that represent the possible range of the input dimension, or null if it is not possible to diff --git a/processing/src/main/java/io/druid/query/filter/NotDimFilter.java b/processing/src/main/java/io/druid/query/filter/NotDimFilter.java index ac72d9b9e84f..890b22adfcee 100644 --- a/processing/src/main/java/io/druid/query/filter/NotDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/NotDimFilter.java @@ -38,7 +38,7 @@ public class NotDimFilter implements DimFilter { final private DimFilter field; - final private Function negate = + private static final Function NEGATE = new Function() { public DimFilter apply(DimFilter filter) { return Druids.newNotDimFilterBuilder().field(filter).build(); } }; @@ -78,16 +78,27 @@ public Filter toFilter() return new NotFilter(field.toFilter()); } + /** + * There are some special cases involving null that require special casing for And and Or instead of simply taking + * the complement + * + * Example 1 : "NOT ( [0,INF) OR null)" The inside of NOT would evaluate to null, and the complement would also + * be null. However, by breaking the NOT, this statement is "NOT([0,INF)) AND NOT(null)", which means it should + * actually evaluate to (-INF, 0). + * + * Example 2 : "NOT ( [0,INF) AND null )" The inside of NOT would evaluate to [0,INF), and the complement would be + * (-INF, 0). However the statement is actually "NOT([0,INF)) OR NOT(null)", and it should be evaluated to null. + */ @Override public RangeSet getDimensionRangeSet(String dimension) { if (field instanceof AndDimFilter) { List fields = ((AndDimFilter) field).getFields(); - return Druids.newOrDimFilterBuilder().fields(Lists.transform(fields, negate)).build().getDimensionRangeSet(dimension); + return Druids.newOrDimFilterBuilder().fields(Lists.transform(fields, NEGATE)).build().getDimensionRangeSet(dimension); } if (field instanceof OrDimFilter) { List fields = ((OrDimFilter) field).getFields(); - return Druids.newAndDimFilterBuilder().fields(Lists.transform(fields, negate)).build().getDimensionRangeSet(dimension); + return Druids.newAndDimFilterBuilder().fields(Lists.transform(fields, NEGATE)).build().getDimensionRangeSet(dimension); } if (field instanceof NotDimFilter) { return ((NotDimFilter) field).getField().getDimensionRangeSet(dimension); diff --git a/server/src/main/java/io/druid/client/CachingClusteredClient.java b/server/src/main/java/io/druid/client/CachingClusteredClient.java index 9c4b856e0b8f..49e8f3788aed 100644 --- a/server/src/main/java/io/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/io/druid/client/CachingClusteredClient.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; +import com.google.common.base.Optional; import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; @@ -29,9 +30,9 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Range; import com.google.common.collect.RangeSet; import com.google.common.collect.Sets; -import com.google.common.collect.TreeRangeSet; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -222,8 +223,9 @@ public Sequence run(final Query query, final Map responseC final List> filteredServersLookup = toolChest.filterSegments(query, serversLookup); - Map> dimensionRangeMap = Maps.newHashMap(); - DimFilter filter = query.getFilter(); + // Absent Optional indicate a null rangeset which represents range cannot be determined + final Map>> dimensionRangeMap = Maps.newHashMap(); + final DimFilter filter = query.getFilter(); // Filter unneeded chunks based on partition dimension for (TimelineObjectHolder holder : filteredServersLookup) { @@ -232,17 +234,16 @@ public Sequence run(final Query query, final Map responseC boolean include = true; if (filter != null) { - Map> domain = selector.getSegment().getShardSpec().getDomain(); - for (Map.Entry> entry : domain.entrySet()) { - if (dimensionRangeMap.get(entry.getKey()) == null) { - dimensionRangeMap.put(entry.getKey(), filter.getDimensionRangeSet(entry.getKey())); + Map> domain = selector.getSegment().getShardSpec().getDomain(); + for (Map.Entry> entry : domain.entrySet()) { + Optional> optFilterRangeSet = dimensionRangeMap.get(entry.getKey()); + if (optFilterRangeSet == null) { + RangeSet filterRangeSet = filter.getDimensionRangeSet(entry.getKey()); + optFilterRangeSet = filterRangeSet == null ? Optional.>absent() : Optional.of(filterRangeSet); + dimensionRangeMap.put(entry.getKey(), optFilterRangeSet); } - if (dimensionRangeMap.get(entry.getKey()) != null) { - RangeSet intersectRange = TreeRangeSet.create(dimensionRangeMap.get(entry.getKey())); - intersectRange.removeAll(entry.getValue().complement()); - if (intersectRange.isEmpty()) { - include = false; - } + if (optFilterRangeSet.isPresent() && optFilterRangeSet.get().subRangeSet(entry.getValue()).isEmpty()) { + include = false; } } } diff --git a/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java b/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java index 729d1f973769..b06ba203ff94 100644 --- a/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/HashBasedNumberedShardSpec.java @@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.google.common.collect.RangeSet; +import com.google.common.collect.Range; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; import io.druid.data.input.InputRow; @@ -126,7 +126,7 @@ public ShardSpec getShardSpec(long timestamp, InputRow row) } @Override - public Map> getDomain() + public Map> getDomain() { return ImmutableMap.of(); } diff --git a/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java b/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java index d12d43a3347f..7f019e351935 100644 --- a/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.RangeSet; +import com.google.common.collect.Range; import io.druid.data.input.InputRow; import java.util.List; @@ -61,7 +61,7 @@ public ShardSpec getShardSpec(long timestamp, InputRow row) } @Override - public Map> getDomain() + public Map> getDomain() { return ImmutableMap.of(); } diff --git a/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java b/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java index 1c078118d0c7..e46e5e0bb875 100644 --- a/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.RangeSet; +import com.google.common.collect.Range; import io.druid.data.input.InputRow; import java.util.List; @@ -71,7 +71,7 @@ public ShardSpec getShardSpec(long timestamp, InputRow row) } @Override - public Map> getDomain() + public Map> getDomain() { return ImmutableMap.of(); } diff --git a/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java b/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java index 208a29db45fd..beaed164f948 100644 --- a/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java @@ -20,16 +20,14 @@ package io.druid.timeline.partition; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableRangeSet; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Range; -import com.google.common.collect.RangeSet; import com.metamx.common.ISE; import io.druid.data.input.InputRow; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.TreeMap; /** * Class uses getters/setters to work around http://jira.codehaus.org/browse/MSHADE-92 @@ -119,9 +117,8 @@ public ShardSpec getShardSpec(long timestamp, InputRow row) } @Override - public Map> getDomain() + public Map> getDomain() { - Map> retMap = new HashMap<>(); Range range; if (start == null && end == null) { range = Range.all(); @@ -132,8 +129,7 @@ public Map> getDomain() } else { range = Range.closed(start, end); } - retMap.put(dimension, ImmutableRangeSet.of(range)); - return retMap; + return ImmutableMap.of(dimension, range); } public void setPartitionNum(int partitionNum) From 7fa41d65bb5c1ffb81d50683d0a65cd686c9e340 Mon Sep 17 00:00:00 2001 From: Dave Li Date: Thu, 23 Jun 2016 14:04:25 -0700 Subject: [PATCH 6/9] extract filtershard to helper function --- .../dimension/ListFilteredDimensionSpec.java | 6 +- .../query/dimension/LookupDimensionSpec.java | 10 +- .../dimension/RegexFilteredDimensionSpec.java | 4 +- .../io/druid/query/filter/AndDimFilter.java | 2 +- .../io/druid/query/filter/BoundDimFilter.java | 10 +- .../query/filter/DimFilterCacheHelper.java | 65 ---------- .../io/druid/query/filter/DimFilterUtils.java | 117 ++++++++++++++++++ .../query/filter/ExtractionDimFilter.java | 6 +- .../io/druid/query/filter/InDimFilter.java | 6 +- .../query/filter/JavaScriptDimFilter.java | 6 +- .../io/druid/query/filter/NoopDimFilter.java | 2 +- .../io/druid/query/filter/NotDimFilter.java | 7 +- .../io/druid/query/filter/OrDimFilter.java | 2 +- .../io/druid/query/filter/RegexDimFilter.java | 6 +- .../query/filter/SearchQueryDimFilter.java | 6 +- .../druid/query/filter/SelectorDimFilter.java | 6 +- .../druid/query/filter/SpatialDimFilter.java | 4 +- .../query/filter/DimFilterUtilsTest.java | 117 ++++++++++++++++++ .../filter/GetDimensionRangeSetTest.java | 20 ++- .../druid/client/CachingClusteredClient.java | 51 ++++---- 20 files changed, 315 insertions(+), 138 deletions(-) delete mode 100644 processing/src/main/java/io/druid/query/filter/DimFilterCacheHelper.java create mode 100644 processing/src/main/java/io/druid/query/filter/DimFilterUtils.java create mode 100644 processing/src/test/java/io/druid/query/filter/DimFilterUtilsTest.java diff --git a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java index 1f303bb7a055..82d671efc78c 100644 --- a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java @@ -23,7 +23,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.metamx.common.StringUtils; -import io.druid.query.filter.DimFilterCacheHelper; +import io.druid.query.filter.DimFilterUtils; import io.druid.segment.DimensionSelector; import java.nio.ByteBuffer; @@ -119,10 +119,10 @@ public byte[] getCacheKey() .put(CACHE_TYPE_ID) .put(delegateCacheKey) .put((byte) (isWhitelist ? 1 : 0)) - .put(DimFilterCacheHelper.STRING_SEPARATOR); + .put(DimFilterUtils.STRING_SEPARATOR); for (byte[] bytes : valuesBytes) { filterCacheKey.put(bytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR); + .put(DimFilterUtils.STRING_SEPARATOR); } return filterCacheKey.array(); } diff --git a/processing/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java index 06d9eb91c8c6..c88fc329c2f4 100644 --- a/processing/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java @@ -26,7 +26,7 @@ import com.google.common.base.Strings; import com.metamx.common.StringUtils; import io.druid.query.extraction.ExtractionFn; -import io.druid.query.filter.DimFilterCacheHelper; +import io.druid.query.filter.DimFilterUtils; import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.lookup.LookupExtractor; import io.druid.query.lookup.LookupReferencesManager; @@ -167,13 +167,13 @@ public byte[] getCacheKey() + replaceWithBytes.length) .put(CACHE_TYPE_ID) .put(dimensionBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(outputNameBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(dimExtractionFnBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(replaceWithBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(retainMissingValue == true ? (byte) 1 : (byte) 0) .array(); } diff --git a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java index db3548b76f64..b55a95693f91 100644 --- a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java @@ -23,7 +23,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.metamx.common.StringUtils; -import io.druid.query.filter.DimFilterCacheHelper; +import io.druid.query.filter.DimFilterUtils; import io.druid.segment.DimensionSelector; import java.nio.ByteBuffer; @@ -89,7 +89,7 @@ public byte[] getCacheKey() return ByteBuffer.allocate(2 + delegateCacheKey.length + regexBytes.length) .put(CACHE_TYPE_ID) .put(delegateCacheKey) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(regexBytes) .array(); } diff --git a/processing/src/main/java/io/druid/query/filter/AndDimFilter.java b/processing/src/main/java/io/druid/query/filter/AndDimFilter.java index 716e12831408..33da2555eeec 100644 --- a/processing/src/main/java/io/druid/query/filter/AndDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/AndDimFilter.java @@ -58,7 +58,7 @@ public List getFields() @Override public byte[] getCacheKey() { - return DimFilterCacheHelper.computeCacheKey(DimFilterCacheHelper.AND_CACHE_ID, fields); + return DimFilterUtils.computeCacheKey(DimFilterUtils.AND_CACHE_ID, fields); } @Override diff --git a/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java b/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java index 3afdac5243cd..8296a1f045ed 100644 --- a/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/BoundDimFilter.java @@ -142,18 +142,18 @@ public byte[] getCacheKey() + lowerBytes.length + extractionFnBytes.length ); - boundCacheBuffer.put(DimFilterCacheHelper.BOUND_CACHE_ID) + boundCacheBuffer.put(DimFilterUtils.BOUND_CACHE_ID) .put(boundType) .put(upperStrictByte) .put(lowerStrictByte) .put(AlphaNumericByte) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(dimensionBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(upperBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(lowerBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(extractionFnBytes); return boundCacheBuffer.array(); } diff --git a/processing/src/main/java/io/druid/query/filter/DimFilterCacheHelper.java b/processing/src/main/java/io/druid/query/filter/DimFilterCacheHelper.java deleted file mode 100644 index 706e5ce5471a..000000000000 --- a/processing/src/main/java/io/druid/query/filter/DimFilterCacheHelper.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.query.filter; - -import java.nio.ByteBuffer; -import java.util.List; - -/** - */ -public class DimFilterCacheHelper -{ - static final byte NOOP_CACHE_ID = -0x4; - static final byte SELECTOR_CACHE_ID = 0x0; - static final byte AND_CACHE_ID = 0x1; - static final byte OR_CACHE_ID = 0x2; - static final byte NOT_CACHE_ID = 0x3; - static final byte EXTRACTION_CACHE_ID = 0x4; - static final byte REGEX_CACHE_ID = 0x5; - static final byte SEARCH_QUERY_TYPE_ID = 0x6; - static final byte JAVASCRIPT_CACHE_ID = 0x7; - static final byte SPATIAL_CACHE_ID = 0x8; - static final byte IN_CACHE_ID = 0x9; - public static final byte STRING_SEPARATOR = (byte) 0xFF; - public static byte BOUND_CACHE_ID = 0xA; - - static byte[] computeCacheKey(byte cacheIdKey, List filters) - { - if (filters.size() == 1) { - return filters.get(0).getCacheKey(); - } - - byte[][] cacheKeys = new byte[filters.size()][]; - int totalSize = 0; - int index = 0; - for (DimFilter field : filters) { - cacheKeys[index] = field.getCacheKey(); - totalSize += cacheKeys[index].length; - ++index; - } - - ByteBuffer retVal = ByteBuffer.allocate(1 + totalSize); - retVal.put(cacheIdKey); - for (byte[] cacheKey : cacheKeys) { - retVal.put(cacheKey); - } - return retVal.array(); - } -} diff --git a/processing/src/main/java/io/druid/query/filter/DimFilterUtils.java b/processing/src/main/java/io/druid/query/filter/DimFilterUtils.java new file mode 100644 index 000000000000..bd2f0c025b2e --- /dev/null +++ b/processing/src/main/java/io/druid/query/filter/DimFilterUtils.java @@ -0,0 +1,117 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.filter; + +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import com.google.common.collect.Sets; +import io.druid.timeline.partition.ShardSpec; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + */ +public class DimFilterUtils +{ + static final byte NOOP_CACHE_ID = -0x4; + static final byte SELECTOR_CACHE_ID = 0x0; + static final byte AND_CACHE_ID = 0x1; + static final byte OR_CACHE_ID = 0x2; + static final byte NOT_CACHE_ID = 0x3; + static final byte EXTRACTION_CACHE_ID = 0x4; + static final byte REGEX_CACHE_ID = 0x5; + static final byte SEARCH_QUERY_TYPE_ID = 0x6; + static final byte JAVASCRIPT_CACHE_ID = 0x7; + static final byte SPATIAL_CACHE_ID = 0x8; + static final byte IN_CACHE_ID = 0x9; + public static final byte STRING_SEPARATOR = (byte) 0xFF; + public static byte BOUND_CACHE_ID = 0xA; + + static byte[] computeCacheKey(byte cacheIdKey, List filters) + { + if (filters.size() == 1) { + return filters.get(0).getCacheKey(); + } + + byte[][] cacheKeys = new byte[filters.size()][]; + int totalSize = 0; + int index = 0; + for (DimFilter field : filters) { + cacheKeys[index] = field.getCacheKey(); + totalSize += cacheKeys[index].length; + ++index; + } + + ByteBuffer retVal = ByteBuffer.allocate(1 + totalSize); + retVal.put(cacheIdKey); + for (byte[] cacheKey : cacheKeys) { + retVal.put(cacheKey); + } + return retVal.array(); + } + + /** + * Filter the given iterable of objects by removing any object whose ShardSpec does not fit in the rangeset of the + * dimFilter {@link DimFilter#getDimensionRangeSet(String)}. If you use the same dimFilter for multiple Iterable + * of objects, use {@link #filterShards(DimFilter, Iterable, Function, Map)} instead with a cached map to save + * redundant calls of {@link DimFilter#getDimensionRangeSet(String)} on the same dimension. + */ + public static Set filterShards (DimFilter dimFilter, Iterable input, Function function) + { + return filterShards(dimFilter, input, function, new HashMap>>()); + } + + public static Set filterShards (DimFilter dimFilter, Iterable input, Function function, + Map>> dimensionRangeMap) + { + Set retSet = Sets.newLinkedHashSet(); + + for (T obj : input) { + ShardSpec shard = function.apply(obj); + boolean include = true; + + if (dimFilter != null && shard != null) { + Map> domain = shard.getDomain(); + for (Map.Entry> entry : domain.entrySet()) { + Optional> optFilterRangeSet = dimensionRangeMap.get(entry.getKey()); + if (optFilterRangeSet == null) { + RangeSet filterRangeSet = dimFilter.getDimensionRangeSet(entry.getKey()); + optFilterRangeSet = Optional.fromNullable(filterRangeSet); + dimensionRangeMap.put(entry.getKey(), optFilterRangeSet); + } + if (optFilterRangeSet.isPresent() && optFilterRangeSet.get().subRangeSet(entry.getValue()).isEmpty()) { + include = false; + } + } + } + + if (include) { + retSet.add(obj); + } + } + return retSet; + } +} diff --git a/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java b/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java index 0c7063508f63..8bb339540e7d 100644 --- a/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java @@ -83,11 +83,11 @@ public byte[] getCacheKey() byte[] valueBytes = value == null ? new byte[0] : StringUtils.toUtf8(value); byte[] extractionFnBytes = extractionFn.getCacheKey(); return ByteBuffer.allocate(3 + dimensionBytes.length + valueBytes.length + extractionFnBytes.length) - .put(DimFilterCacheHelper.EXTRACTION_CACHE_ID) + .put(DimFilterUtils.EXTRACTION_CACHE_ID) .put(dimensionBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(valueBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(extractionFnBytes) .array(); } diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index 021503aebc9a..9ee4d5d8da02 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -109,11 +109,11 @@ public byte[] getCacheKey() + dimensionBytes.length + valuesBytesSize + extractionFnBytes.length) - .put(DimFilterCacheHelper.IN_CACHE_ID) + .put(DimFilterUtils.IN_CACHE_ID) .put(dimensionBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(extractionFnBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR); + .put(DimFilterUtils.STRING_SEPARATOR); for (byte[] bytes : valuesBytes) { filterCacheKey.put(bytes) .put((byte) 0xFF); diff --git a/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java b/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java index dcab4147ef14..6442c68da42e 100644 --- a/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java @@ -93,11 +93,11 @@ public byte[] getCacheKey() byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); return ByteBuffer.allocate(3 + dimensionBytes.length + functionBytes.length + extractionFnBytes.length) - .put(DimFilterCacheHelper.JAVASCRIPT_CACHE_ID) + .put(DimFilterUtils.JAVASCRIPT_CACHE_ID) .put(dimensionBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(functionBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(extractionFnBytes) .array(); } diff --git a/processing/src/main/java/io/druid/query/filter/NoopDimFilter.java b/processing/src/main/java/io/druid/query/filter/NoopDimFilter.java index bda0c849d3bd..61949fb9247a 100644 --- a/processing/src/main/java/io/druid/query/filter/NoopDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/NoopDimFilter.java @@ -30,7 +30,7 @@ public class NoopDimFilter implements DimFilter @Override public byte[] getCacheKey() { - return ByteBuffer.allocate(1).put(DimFilterCacheHelper.NOOP_CACHE_ID).array(); + return ByteBuffer.allocate(1).put(DimFilterUtils.NOOP_CACHE_ID).array(); } @Override diff --git a/processing/src/main/java/io/druid/query/filter/NotDimFilter.java b/processing/src/main/java/io/druid/query/filter/NotDimFilter.java index 890b22adfcee..cf4de057c0cd 100644 --- a/processing/src/main/java/io/druid/query/filter/NotDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/NotDimFilter.java @@ -25,7 +25,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.RangeSet; -import com.sun.org.apache.xpath.internal.operations.And; import io.druid.query.Druids; import io.druid.segment.filter.NotFilter; @@ -36,13 +35,13 @@ */ public class NotDimFilter implements DimFilter { - final private DimFilter field; - private static final Function NEGATE = new Function() { public DimFilter apply(DimFilter filter) { return Druids.newNotDimFilterBuilder().field(filter).build(); } }; + final private DimFilter field; + @JsonCreator public NotDimFilter( @JsonProperty("field") DimFilter field @@ -63,7 +62,7 @@ public byte[] getCacheKey() { byte[] subKey = field.getCacheKey(); - return ByteBuffer.allocate(1 + subKey.length).put(DimFilterCacheHelper.NOT_CACHE_ID).put(subKey).array(); + return ByteBuffer.allocate(1 + subKey.length).put(DimFilterUtils.NOT_CACHE_ID).put(subKey).array(); } @Override diff --git a/processing/src/main/java/io/druid/query/filter/OrDimFilter.java b/processing/src/main/java/io/druid/query/filter/OrDimFilter.java index 9928a8c92d34..9e2427a1b511 100644 --- a/processing/src/main/java/io/druid/query/filter/OrDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/OrDimFilter.java @@ -58,7 +58,7 @@ public List getFields() @Override public byte[] getCacheKey() { - return DimFilterCacheHelper.computeCacheKey(DimFilterCacheHelper.OR_CACHE_ID, fields); + return DimFilterUtils.computeCacheKey(DimFilterUtils.OR_CACHE_ID, fields); } @Override diff --git a/processing/src/main/java/io/druid/query/filter/RegexDimFilter.java b/processing/src/main/java/io/druid/query/filter/RegexDimFilter.java index 047aab6b244b..586aea58c6fc 100644 --- a/processing/src/main/java/io/druid/query/filter/RegexDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/RegexDimFilter.java @@ -81,11 +81,11 @@ public byte[] getCacheKey() byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); return ByteBuffer.allocate(3 + dimensionBytes.length + patternBytes.length + extractionFnBytes.length) - .put(DimFilterCacheHelper.REGEX_CACHE_ID) + .put(DimFilterUtils.REGEX_CACHE_ID) .put(dimensionBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(patternBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(extractionFnBytes) .array(); } diff --git a/processing/src/main/java/io/druid/query/filter/SearchQueryDimFilter.java b/processing/src/main/java/io/druid/query/filter/SearchQueryDimFilter.java index 761ee492e22a..f6e17b71a237 100644 --- a/processing/src/main/java/io/druid/query/filter/SearchQueryDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SearchQueryDimFilter.java @@ -77,11 +77,11 @@ public byte[] getCacheKey() byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); return ByteBuffer.allocate(3 + dimensionBytes.length + queryBytes.length + extractionFnBytes.length) - .put(DimFilterCacheHelper.SEARCH_QUERY_TYPE_ID) + .put(DimFilterUtils.SEARCH_QUERY_TYPE_ID) .put(dimensionBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(queryBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(extractionFnBytes) .array(); } diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index 1ca9ef035944..a4826c47f6b6 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -66,11 +66,11 @@ public byte[] getCacheKey() byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); return ByteBuffer.allocate(3 + dimensionBytes.length + valueBytes.length + extractionFnBytes.length) - .put(DimFilterCacheHelper.SELECTOR_CACHE_ID) + .put(DimFilterUtils.SELECTOR_CACHE_ID) .put(dimensionBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(valueBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(extractionFnBytes) .array(); } diff --git a/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java b/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java index 352ffa86fb41..e20dcd28ca7d 100644 --- a/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java @@ -55,9 +55,9 @@ public byte[] getCacheKey() byte[] boundBytes = bound.getCacheKey(); return ByteBuffer.allocate(2 + dimBytes.length + boundBytes.length) - .put(DimFilterCacheHelper.SPATIAL_CACHE_ID) + .put(DimFilterUtils.SPATIAL_CACHE_ID) .put(dimBytes) - .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(DimFilterUtils.STRING_SEPARATOR) .put(boundBytes) .array(); } diff --git a/processing/src/test/java/io/druid/query/filter/DimFilterUtilsTest.java b/processing/src/test/java/io/druid/query/filter/DimFilterUtilsTest.java new file mode 100644 index 000000000000..162879e3c5a8 --- /dev/null +++ b/processing/src/test/java/io/druid/query/filter/DimFilterUtilsTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.filter; + +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableRangeSet; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import com.google.common.collect.Sets; +import io.druid.timeline.partition.ShardSpec; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class DimFilterUtilsTest +{ + private static final Function CONVERTER = new Function() + { + @Nullable + @Override + public ShardSpec apply(@Nullable ShardSpec input) + { + return input; + } + }; + + @Test + public void testFilterShards() { + DimFilter filter1 = EasyMock.createMock(DimFilter.class); + EasyMock.expect(filter1.getDimensionRangeSet("dim1")) + .andReturn(rangeSet(ImmutableList.of(Range.lessThan("abc")))) + .anyTimes(); + EasyMock.expect(filter1.getDimensionRangeSet("dim2")) + .andReturn(null) + .anyTimes(); + + DimFilter filter2 = EasyMock.createMock(DimFilter.class); + EasyMock.expect(filter2.getDimensionRangeSet("dim1")) + .andReturn(rangeSet(ImmutableList.of(Range.singleton("e")))) + .anyTimes(); + EasyMock.expect(filter2.getDimensionRangeSet("dim2")) + .andReturn(rangeSet(ImmutableList.of(Range.singleton("na")))) + .anyTimes(); + + ShardSpec shard1 = shardSpec("dim1", Range.atMost("abc")); + ShardSpec shard2 = shardSpec("dim1", Range.closed("abc", "def")); + ShardSpec shard3 = shardSpec("dim1", Range.atLeast("def")); + ShardSpec shard4 = shardSpec("dim2", Range.atMost("hello")); + ShardSpec shard5 = shardSpec("dim2", Range.closed("hello", "jk")); + ShardSpec shard6 = shardSpec("dim2", Range.closed("jk", "na")); + ShardSpec shard7 = shardSpec("dim2", Range.atLeast("na")); + + List shards = ImmutableList.of(shard1, shard2, shard3, shard4, shard5, shard6, shard7); + EasyMock.replay(filter1, filter2, shard1, shard2, shard3, shard4, shard5, shard6, shard7); + + Set expected1 = ImmutableSet.of(shard1, shard4, shard5, shard6, shard7); + assetFilterResult(filter1, shards, expected1); + + Set expected2 = ImmutableSet.of(shard3, shard6, shard7); + assetFilterResult(filter2, shards, expected2); + } + + private void assetFilterResult(DimFilter filter, Iterable input, Set expected) { + Set result = DimFilterUtils.filterShards(filter, input, CONVERTER); + Assert.assertEquals(expected, result); + + Map>> dimensionRangeMap = Maps.newHashMap(); + result = Sets.newHashSet(); + for (ShardSpec shard : input) { + result.addAll(DimFilterUtils.filterShards(filter, ImmutableList.of(shard), CONVERTER, dimensionRangeMap)); + } + Assert.assertEquals(expected, result); + } + + private static RangeSet rangeSet (List> ranges) { + ImmutableRangeSet.Builder builder = ImmutableRangeSet.builder(); + for (Range range : ranges) { + builder.add(range); + } + return builder.build(); + } + + private static ShardSpec shardSpec (String dimension, Range range) { + ShardSpec shard = EasyMock.createMock(ShardSpec.class); + EasyMock.expect(shard.getDomain()) + .andReturn(ImmutableMap.of(dimension, range)) + .anyTimes(); + return shard; + } +} diff --git a/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java b/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java index a972b0e1a58d..a16202908e91 100644 --- a/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java +++ b/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java @@ -1,5 +1,23 @@ -package io.druid.query.filter; +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.query.filter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableRangeSet; diff --git a/server/src/main/java/io/druid/client/CachingClusteredClient.java b/server/src/main/java/io/druid/client/CachingClusteredClient.java index 49e8f3788aed..09fc40cd2d6b 100644 --- a/server/src/main/java/io/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/io/druid/client/CachingClusteredClient.java @@ -30,7 +30,6 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Range; import com.google.common.collect.RangeSet; import com.google.common.collect.Sets; import com.google.common.util.concurrent.FutureCallback; @@ -63,13 +62,14 @@ import io.druid.query.Result; import io.druid.query.SegmentDescriptor; import io.druid.query.aggregation.MetricManipulatorFns; -import io.druid.query.filter.DimFilter; +import io.druid.query.filter.DimFilterUtils; import io.druid.query.spec.MultipleSpecificSegmentSpec; import io.druid.server.coordination.DruidServerMetadata; import io.druid.timeline.DataSegment; import io.druid.timeline.TimelineLookup; import io.druid.timeline.TimelineObjectHolder; import io.druid.timeline.partition.PartitionChunk; +import io.druid.timeline.partition.ShardSpec; import org.joda.time.Interval; import java.io.IOException; @@ -222,38 +222,29 @@ public Sequence run(final Query query, final Map responseC // Let tool chest filter out unneeded segments final List> filteredServersLookup = toolChest.filterSegments(query, serversLookup); - - // Absent Optional indicate a null rangeset which represents range cannot be determined - final Map>> dimensionRangeMap = Maps.newHashMap(); - final DimFilter filter = query.getFilter(); + Map>> dimensionRangeMap = Maps.newHashMap(); // Filter unneeded chunks based on partition dimension for (TimelineObjectHolder holder : filteredServersLookup) { - for (PartitionChunk chunk : holder.getObject()) { - ServerSelector selector = chunk.getObject(); - boolean include = true; - - if (filter != null) { - Map> domain = selector.getSegment().getShardSpec().getDomain(); - for (Map.Entry> entry : domain.entrySet()) { - Optional> optFilterRangeSet = dimensionRangeMap.get(entry.getKey()); - if (optFilterRangeSet == null) { - RangeSet filterRangeSet = filter.getDimensionRangeSet(entry.getKey()); - optFilterRangeSet = filterRangeSet == null ? Optional.>absent() : Optional.of(filterRangeSet); - dimensionRangeMap.put(entry.getKey(), optFilterRangeSet); - } - if (optFilterRangeSet.isPresent() && optFilterRangeSet.get().subRangeSet(entry.getValue()).isEmpty()) { - include = false; + final Set> filteredChunks = DimFilterUtils.filterShards( + query.getFilter(), + holder.getObject(), + new Function, ShardSpec>() + { + @Override + public ShardSpec apply(PartitionChunk input) + { + return input.getObject().getSegment().getShardSpec(); } - } - } - - if (include) { - final SegmentDescriptor descriptor = new SegmentDescriptor( - holder.getInterval(), holder.getVersion(), chunk.getChunkNumber() - ); - segments.add(Pair.of(selector, descriptor)); - } + }, + dimensionRangeMap + ); + for (PartitionChunk chunk : filteredChunks) { + ServerSelector selector = chunk.getObject(); + final SegmentDescriptor descriptor = new SegmentDescriptor( + holder.getInterval(), holder.getVersion(), chunk.getChunkNumber() + ); + segments.add(Pair.of(selector, descriptor)); } } From 75b11cabb51f2fe63dd74ffc36cdd6342d9c690e Mon Sep 17 00:00:00 2001 From: Dave Li Date: Thu, 23 Jun 2016 15:17:03 -0700 Subject: [PATCH 7/9] fixup --- .../java/io/druid/client/CachingClusteredClientTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java index e98a1907b41c..571236386e76 100644 --- a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java +++ b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java @@ -2678,7 +2678,11 @@ public List getMetrics() @JsonProperty public ShardSpec getShardSpec() { - return baseSegment.getShardSpec(); + try { + return baseSegment.getShardSpec(); + } catch (IllegalStateException e) { + return NoneShardSpec.instance(); + } } @Override From 09f85f26d839a65f28429ab7ac1a7ef3cce1ecae Mon Sep 17 00:00:00 2001 From: Dave Li Date: Fri, 24 Jun 2016 12:03:33 -0700 Subject: [PATCH 8/9] minor changes --- .../java/io/druid/query/filter/DimFilter.java | 2 +- .../io/druid/query/filter/DimFilterUtils.java | 26 ++++++++++++------- .../query/filter/DimFilterUtilsTest.java | 10 +++---- .../filter/GetDimensionRangeSetTest.java | 1 - .../druid/client/CachingClusteredClient.java | 4 +-- .../client/CachingClusteredClientTest.java | 13 +++++----- 6 files changed, 31 insertions(+), 25 deletions(-) diff --git a/processing/src/main/java/io/druid/query/filter/DimFilter.java b/processing/src/main/java/io/druid/query/filter/DimFilter.java index a1c595df68d0..7c8676462f38 100644 --- a/processing/src/main/java/io/druid/query/filter/DimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/DimFilter.java @@ -71,5 +71,5 @@ public interface DimFilter * @return a RangeSet that represent the possible range of the input dimension, or null if it is not possible to * determine for this DimFilter. */ - public RangeSet getDimensionRangeSet (String dimension); + public RangeSet getDimensionRangeSet(String dimension); } diff --git a/processing/src/main/java/io/druid/query/filter/DimFilterUtils.java b/processing/src/main/java/io/druid/query/filter/DimFilterUtils.java index bd2f0c025b2e..d3a4e1cb7424 100644 --- a/processing/src/main/java/io/druid/query/filter/DimFilterUtils.java +++ b/processing/src/main/java/io/druid/query/filter/DimFilterUtils.java @@ -74,33 +74,39 @@ static byte[] computeCacheKey(byte cacheIdKey, List filters) } /** - * Filter the given iterable of objects by removing any object whose ShardSpec does not fit in the rangeset of the - * dimFilter {@link DimFilter#getDimensionRangeSet(String)}. If you use the same dimFilter for multiple Iterable - * of objects, use {@link #filterShards(DimFilter, Iterable, Function, Map)} instead with a cached map to save + * Filter the given iterable of objects by removing any object whose ShardSpec, obtained from the converter function, + * does not fit in the rangeset of the dimFilter {@link DimFilter#getDimensionRangeSet(String)}. A set will be + * returned containing the filtered objects, in the same order as they are passed in. + * + * If you use the same dimFilter for multiple Iterables of objects, consider using + * {@link #filterShards(DimFilter, Iterable, Function, Map)} instead with a cached map to save * redundant calls of {@link DimFilter#getDimensionRangeSet(String)} on the same dimension. */ - public static Set filterShards (DimFilter dimFilter, Iterable input, Function function) + public static Set filterShards(DimFilter dimFilter, Iterable input, Function converter) { - return filterShards(dimFilter, input, function, new HashMap>>()); + return filterShards(dimFilter, input, converter, new HashMap>>()); } - public static Set filterShards (DimFilter dimFilter, Iterable input, Function function, - Map>> dimensionRangeMap) + /** + * DimensionRangedCache can be re-used between calls with the same dimFilter. + */ + public static Set filterShards(DimFilter dimFilter, Iterable input, Function converter, + Map>> dimensionRangeCache) { Set retSet = Sets.newLinkedHashSet(); for (T obj : input) { - ShardSpec shard = function.apply(obj); + ShardSpec shard = converter.apply(obj); boolean include = true; if (dimFilter != null && shard != null) { Map> domain = shard.getDomain(); for (Map.Entry> entry : domain.entrySet()) { - Optional> optFilterRangeSet = dimensionRangeMap.get(entry.getKey()); + Optional> optFilterRangeSet = dimensionRangeCache.get(entry.getKey()); if (optFilterRangeSet == null) { RangeSet filterRangeSet = dimFilter.getDimensionRangeSet(entry.getKey()); optFilterRangeSet = Optional.fromNullable(filterRangeSet); - dimensionRangeMap.put(entry.getKey(), optFilterRangeSet); + dimensionRangeCache.put(entry.getKey(), optFilterRangeSet); } if (optFilterRangeSet.isPresent() && optFilterRangeSet.get().subRangeSet(entry.getValue()).isEmpty()) { include = false; diff --git a/processing/src/test/java/io/druid/query/filter/DimFilterUtilsTest.java b/processing/src/test/java/io/druid/query/filter/DimFilterUtilsTest.java index 162879e3c5a8..fb881c097e2f 100644 --- a/processing/src/test/java/io/druid/query/filter/DimFilterUtilsTest.java +++ b/processing/src/test/java/io/druid/query/filter/DimFilterUtilsTest.java @@ -81,13 +81,13 @@ public void testFilterShards() { EasyMock.replay(filter1, filter2, shard1, shard2, shard3, shard4, shard5, shard6, shard7); Set expected1 = ImmutableSet.of(shard1, shard4, shard5, shard6, shard7); - assetFilterResult(filter1, shards, expected1); + assertFilterResult(filter1, shards, expected1); Set expected2 = ImmutableSet.of(shard3, shard6, shard7); - assetFilterResult(filter2, shards, expected2); + assertFilterResult(filter2, shards, expected2); } - private void assetFilterResult(DimFilter filter, Iterable input, Set expected) { + private void assertFilterResult(DimFilter filter, Iterable input, Set expected) { Set result = DimFilterUtils.filterShards(filter, input, CONVERTER); Assert.assertEquals(expected, result); @@ -99,7 +99,7 @@ private void assetFilterResult(DimFilter filter, Iterable input, Set< Assert.assertEquals(expected, result); } - private static RangeSet rangeSet (List> ranges) { + private static RangeSet rangeSet(List> ranges) { ImmutableRangeSet.Builder builder = ImmutableRangeSet.builder(); for (Range range : ranges) { builder.add(range); @@ -107,7 +107,7 @@ private static RangeSet rangeSet (List> ranges) { return builder.build(); } - private static ShardSpec shardSpec (String dimension, Range range) { + private static ShardSpec shardSpec(String dimension, Range range) { ShardSpec shard = EasyMock.createMock(ShardSpec.class); EasyMock.expect(shard.getDomain()) .andReturn(ImmutableMap.of(dimension, range)) diff --git a/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java b/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java index a16202908e91..c1a1545fc9d8 100644 --- a/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java +++ b/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java @@ -28,7 +28,6 @@ import io.druid.query.search.search.ContainsSearchQuerySpec; import org.junit.Assert; import org.junit.Test; -import org.mozilla.javascript.tools.debugger.Dim; import java.util.Arrays; import java.util.List; diff --git a/server/src/main/java/io/druid/client/CachingClusteredClient.java b/server/src/main/java/io/druid/client/CachingClusteredClient.java index 09fc40cd2d6b..66dfeb524c3d 100644 --- a/server/src/main/java/io/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/io/druid/client/CachingClusteredClient.java @@ -222,7 +222,7 @@ public Sequence run(final Query query, final Map responseC // Let tool chest filter out unneeded segments final List> filteredServersLookup = toolChest.filterSegments(query, serversLookup); - Map>> dimensionRangeMap = Maps.newHashMap(); + Map>> dimensionRangeCache = Maps.newHashMap(); // Filter unneeded chunks based on partition dimension for (TimelineObjectHolder holder : filteredServersLookup) { @@ -237,7 +237,7 @@ public ShardSpec apply(PartitionChunk input) return input.getObject().getSegment().getShardSpec(); } }, - dimensionRangeMap + dimensionRangeCache ); for (PartitionChunk chunk : filteredChunks) { ServerSelector selector = chunk.getObject(); diff --git a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java index 571236386e76..5616c8d053d3 100644 --- a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java +++ b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java @@ -1439,8 +1439,8 @@ public void testTimeBoundaryCaching() throws Exception } @Test - public void testTimeSeriesWithFilter () { - + public void testTimeSeriesWithFilter() throws Exception + { DimFilter filter = Druids.newAndDimFilterBuilder() .fields( Arrays.asList( @@ -1511,8 +1511,8 @@ For dim1 (2011-01-06/2011-01-10), the combined range for the bound filters is {( } @Test - public void testSingleDimensionPruning () { - + public void testSingleDimensionPruning() throws Exception + { DimFilter filter = Druids.newAndDimFilterBuilder() .fields( Arrays.asList( @@ -1650,10 +1650,11 @@ private Iterable> makeTimeBoundaryResult( ); } - public void parseResults ( + public void parseResults( final List queryIntervals, final List>>> expectedResults, - Object... args) + Object... args + ) { if (args.length % 2 != 0) { throw new ISE("args.length must be divisible by two, was %d", args.length); From 20aa962b62aec7b1ff5e72f1e39f50882514e8eb Mon Sep 17 00:00:00 2001 From: Dave Li Date: Fri, 24 Jun 2016 13:11:17 -0700 Subject: [PATCH 9/9] update javadoc --- .../io/druid/query/filter/DimFilterUtils.java | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/processing/src/main/java/io/druid/query/filter/DimFilterUtils.java b/processing/src/main/java/io/druid/query/filter/DimFilterUtils.java index d3a4e1cb7424..a8841580f8f2 100644 --- a/processing/src/main/java/io/druid/query/filter/DimFilterUtils.java +++ b/processing/src/main/java/io/druid/query/filter/DimFilterUtils.java @@ -75,12 +75,17 @@ static byte[] computeCacheKey(byte cacheIdKey, List filters) /** * Filter the given iterable of objects by removing any object whose ShardSpec, obtained from the converter function, - * does not fit in the rangeset of the dimFilter {@link DimFilter#getDimensionRangeSet(String)}. A set will be - * returned containing the filtered objects, in the same order as they are passed in. + * does not fit in the RangeSet of the dimFilter {@link DimFilter#getDimensionRangeSet(String)}. The returned set + * contains the filtered objects in the same order as they appear in input. * - * If you use the same dimFilter for multiple Iterables of objects, consider using - * {@link #filterShards(DimFilter, Iterable, Function, Map)} instead with a cached map to save - * redundant calls of {@link DimFilter#getDimensionRangeSet(String)} on the same dimension. + * If you plan to call this multiple times with the same dimFilter, consider using + * {@link #filterShards(DimFilter, Iterable, Function, Map)} instead with a cached map + * + * @param dimFilter The filter to use + * @param input The iterable of objects to be filtered + * @param converter The function to convert T to ShardSpec that can be filtered by + * @param This can be any type, as long as transform function is provided to convert this to ShardSpec + * @return The set of filtered object, in the same order as input */ public static Set filterShards(DimFilter dimFilter, Iterable input, Function converter) { @@ -88,7 +93,20 @@ public static Set filterShards(DimFilter dimFilter, Iterable input, Fu } /** - * DimensionRangedCache can be re-used between calls with the same dimFilter. + * Filter the given iterable of objects by removing any object whose ShardSpec, obtained from the converter function, + * does not fit in the RangeSet of the dimFilter {@link DimFilter#getDimensionRangeSet(String)}. The returned set + * contains the filtered objects in the same order as they appear in input. + * + * DimensionRangedCache stores the RangeSets of different dimensions for the dimFilter. It should be re-used + * between calls with the same dimFilter to save redundant calls of {@link DimFilter#getDimensionRangeSet(String)} + * on same dimensions. + * + * @param dimFilter The filter to use + * @param input The iterable of objects to be filtered + * @param converter The function to convert T to ShardSpec that can be filtered by + * @param dimensionRangeCache The cache of RangeSets of different dimensions for the dimFilter + * @param This can be any type, as long as transform function is provided to convert this to ShardSpec + * @return The set of filtered object, in the same order as input */ public static Set filterShards(DimFilter dimFilter, Iterable input, Function converter, Map>> dimensionRangeCache)