Skip to content

add method getRequiredColumns for DimFilter #5872

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jon-wei merged 2 commits into from
Jun 27, 2018
Merged

add method getRequiredColumns for DimFilter #5872

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e810da0
add method getRequiredColumns for DimFilter
Jun 11, 2018
9a3bf55
deal with the NullPointerException when DimFilter is null
Jun 12, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 4 additions & 62 deletions ...d-view-selection/src/main/java/io/druid/query/materializedview/MaterializedViewUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,24 +20,12 @@
package io.druid.query.materializedview;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import io.druid.java.util.common.Intervals;
import io.druid.java.util.common.JodaUtils;
import io.druid.query.Query;
import io.druid.query.aggregation.AggregatorFactory;
import io.druid.query.aggregation.FilteredAggregatorFactory;
import io.druid.query.dimension.DimensionSpec;
import io.druid.query.filter.AndDimFilter;
import io.druid.query.filter.BoundDimFilter;
import io.druid.query.filter.DimFilter;
import io.druid.query.filter.InDimFilter;
import io.druid.query.filter.IntervalDimFilter;
import io.druid.query.filter.LikeDimFilter;
import io.druid.query.filter.NotDimFilter;
import io.druid.query.filter.OrDimFilter;
import io.druid.query.filter.RegexDimFilter;
import io.druid.query.filter.SearchQueryDimFilter;
import io.druid.query.filter.SelectorDimFilter;
import io.druid.query.groupby.GroupByQuery;
import io.druid.query.timeseries.TimeseriesQuery;
import io.druid.query.topn.TopNQuery;
Expand All @@ -60,10 +48,9 @@ public class MaterializedViewUtils
public static Set<String> getRequiredFields(Query query)
{
Set<String> dimensions = new HashSet<>();
Set<String> dimsInFilter = getDimensionsInFilter(query.getFilter());
if (dimsInFilter != null) {
dimensions.addAll(dimsInFilter);
}
Set<String> dimsInFilter = null == query.getFilter() ? new HashSet<String>() : query.getFilter().getRequiredColumns();
dimensions.addAll(dimsInFilter);

if (query instanceof TopNQuery) {
TopNQuery q = (TopNQuery) query;
dimensions.addAll(extractFieldsFromAggregations(q.getAggregatorSpecs()));
Expand All @@ -90,58 +77,13 @@ private static Set<String> extractFieldsFromAggregations(List<AggregatorFactory>
for (AggregatorFactory agg : aggs) {
if (agg instanceof FilteredAggregatorFactory) {
FilteredAggregatorFactory fagg = (FilteredAggregatorFactory) agg;
ret.addAll(getDimensionsInFilter(fagg.getFilter()));
ret.addAll(fagg.getFilter().getRequiredColumns());
}
ret.addAll(agg.requiredFields());
}
return ret;
}

private static Set<String> getDimensionsInFilter(DimFilter dimFilter)
{
if (dimFilter instanceof AndDimFilter) {
AndDimFilter d = (AndDimFilter) dimFilter;
Set<String> ret = new HashSet<>();
for (DimFilter filter : d.getFields()) {
ret.addAll(getDimensionsInFilter(filter));
}
return ret;
} else if (dimFilter instanceof OrDimFilter) {
OrDimFilter d = (OrDimFilter) dimFilter;
Set<String> ret = new HashSet<>();
for (DimFilter filter : d.getFields()) {
ret.addAll(getDimensionsInFilter(filter));
}
return ret;
} else if (dimFilter instanceof NotDimFilter) {
NotDimFilter d = (NotDimFilter) dimFilter;
return getDimensionsInFilter(d.getField());
} else if (dimFilter instanceof BoundDimFilter) {
BoundDimFilter d = (BoundDimFilter) dimFilter;
return Sets.newHashSet(d.getDimension());
} else if (dimFilter instanceof InDimFilter) {
InDimFilter d = (InDimFilter) dimFilter;
return Sets.newHashSet(d.getDimension());
} else if (dimFilter instanceof IntervalDimFilter) {
IntervalDimFilter d = (IntervalDimFilter) dimFilter;
return Sets.newHashSet(d.getDimension());
} else if (dimFilter instanceof LikeDimFilter) {
LikeDimFilter d = (LikeDimFilter) dimFilter;
return Sets.newHashSet(d.getDimension());
} else if (dimFilter instanceof RegexDimFilter) {
RegexDimFilter d = (RegexDimFilter) dimFilter;
return Sets.newHashSet(d.getDimension());
} else if (dimFilter instanceof SearchQueryDimFilter) {
SearchQueryDimFilter d = (SearchQueryDimFilter) dimFilter;
return Sets.newHashSet(d.getDimension());
} else if (dimFilter instanceof SelectorDimFilter) {
SelectorDimFilter d = (SelectorDimFilter) dimFilter;
return Sets.newHashSet(d.getDimension());
} else {
return null;
}
}

/**
* caculate the intervals which are covered by interval2, but not covered by interval1.
* result intervals = interval2 - interval1 ∩ interval2
Expand Down
11 changes: 11 additions & 0 deletions processing/src/main/java/io/druid/query/filter/AndDimFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,14 @@
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeRangeSet;
import io.druid.java.util.common.StringUtils;
import io.druid.segment.filter.AndFilter;
import io.druid.segment.filter.Filters;

import java.util.Arrays;
import java.util.HashSet;
import java.util.List;

/**
Expand Down Expand Up @@ -97,6 +99,15 @@ public RangeSet<String> getDimensionRangeSet(String dimension)
return retSet;
}

@Override
public HashSet<String> getRequiredColumns()
{
HashSet<String> requiredColumns = Sets.newHashSet();
fields.stream()
.forEach(field -> requiredColumns.addAll(field.getRequiredColumns()));
return requiredColumns;
}

@Override
public boolean equals(Object o)
{
Expand Down
8 changes: 8 additions & 0 deletions processing/src/main/java/io/druid/query/filter/BoundDimFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import com.google.common.collect.BoundType;
import com.google.common.collect.Range;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeRangeSet;
import com.google.common.primitives.Doubles;
import com.google.common.primitives.Floats;
Expand All @@ -40,6 +41,7 @@
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.Objects;

public class BoundDimFilter implements DimFilter
Expand Down Expand Up @@ -246,6 +248,12 @@ && getExtractionFn() == null
return retSet;
}

@Override
public HashSet<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
}

@Override
public boolean equals(Object o)
{
Expand Down
12 changes: 12 additions & 0 deletions processing/src/main/java/io/druid/query/filter/ColumnComparisonDimFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,14 @@
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import io.druid.query.cache.CacheKeyBuilder;
import io.druid.query.dimension.DimensionSpec;
import io.druid.segment.filter.ColumnComparisonFilter;

import java.util.HashSet;
import java.util.List;
import java.util.stream.Collectors;

/**
*/
Expand Down Expand Up @@ -103,6 +106,15 @@ public RangeSet<String> getDimensionRangeSet(String dimension)
return null;
}

@Override
public HashSet<String> getRequiredColumns()
{
return Sets.newHashSet(dimensions.stream()
.map(DimensionSpec::getDimension)
.collect(Collectors.toSet())
);
}

@Override
public int hashCode()
{
Expand Down
7 changes: 7 additions & 0 deletions processing/src/main/java/io/druid/query/filter/DimFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
import com.google.common.collect.RangeSet;
import io.druid.java.util.common.Cacheable;

import java.util.HashSet;

/**
*/
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
Expand Down Expand Up @@ -75,4 +77,9 @@ public interface DimFilter extends Cacheable
* determine for this DimFilter.
*/
RangeSet<String> getDimensionRangeSet(String dimension);

/**
* @return a HashSet that represents all columns' name which the DimFilter required to do filter.
*/
HashSet<String> getRequiredColumns();
Copy link
Copy Markdown
Contributor

@himanshug himanshug Aug 28, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we have the return type be Set<String> instead so that single column cases can just return ImmutableSet.of(col) ?

Copy link
Copy Markdown
Member

@clintropolis clintropolis Aug 5, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm making this change to use Set as part of #8209.

}
8 changes: 8 additions & 0 deletions processing/src/main/java/io/druid/query/filter/ExpressionDimFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,14 @@
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import io.druid.math.expr.Expr;
import io.druid.math.expr.ExprMacroTable;
import io.druid.math.expr.Parser;
import io.druid.query.cache.CacheKeyBuilder;
import io.druid.segment.filter.ExpressionFilter;

import java.util.HashSet;
import java.util.Objects;

public class ExpressionDimFilter implements DimFilter
Expand Down Expand Up @@ -70,6 +72,12 @@ public RangeSet<String> getDimensionRangeSet(final String dimension)
return null;
}

@Override
public HashSet<String> getRequiredColumns()
{
return Sets.newHashSet(Parser.findRequiredBindings(parsed));
}

@Override
public byte[] getCacheKey()
{
Expand Down
8 changes: 8 additions & 0 deletions processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,12 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import io.druid.java.util.common.StringUtils;
import io.druid.query.extraction.ExtractionFn;

import java.nio.ByteBuffer;
import java.util.HashSet;

/**
* This class is deprecated, use SelectorDimFilter instead: {@link SelectorDimFilter}
Expand Down Expand Up @@ -110,6 +112,12 @@ public RangeSet<String> getDimensionRangeSet(String dimension)
return null;
}

@Override
public HashSet<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
}

@Override
public String toString()
{
Expand Down
8 changes: 8 additions & 0 deletions processing/src/main/java/io/druid/query/filter/InDimFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import com.google.common.collect.Iterables;
import com.google.common.collect.Range;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeRangeSet;
import com.google.common.primitives.Doubles;
import com.google.common.primitives.Floats;
Expand All @@ -48,6 +49,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
Expand Down Expand Up @@ -215,6 +217,12 @@ public RangeSet<String> getDimensionRangeSet(String dimension)
return retSet;
}

@Override
public HashSet<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
}

@Override
public boolean equals(Object o)
{
Expand Down
8 changes: 8 additions & 0 deletions processing/src/main/java/io/druid/query/filter/IntervalDimFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.primitives.Longs;
import io.druid.java.util.common.JodaUtils;
import io.druid.java.util.common.Pair;
Expand All @@ -34,6 +35,7 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;

public class IntervalDimFilter implements DimFilter
Expand Down Expand Up @@ -122,6 +124,12 @@ public RangeSet<String> getDimensionRangeSet(String dimension)
return convertedFilter.getDimensionRangeSet(dimension);
}

@Override
public HashSet<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
}

@Override
public boolean equals(Object o)
{
Expand Down
8 changes: 8 additions & 0 deletions processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import io.druid.java.util.common.StringUtils;
import io.druid.js.JavaScriptConfig;
import io.druid.query.extraction.ExtractionFn;
Expand All @@ -34,6 +35,7 @@
import org.mozilla.javascript.ScriptableObject;

import java.nio.ByteBuffer;
import java.util.HashSet;

public class JavaScriptDimFilter implements DimFilter
{
Expand Down Expand Up @@ -134,6 +136,12 @@ public RangeSet<String> getDimensionRangeSet(String dimension)
return null;
}

@Override
public HashSet<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
}

@Override
public String toString()
{
Expand Down
8 changes: 8 additions & 0 deletions processing/src/main/java/io/druid/query/filter/LikeDimFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.google.common.base.Predicate;
import com.google.common.base.Strings;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.io.BaseEncoding;
import com.google.common.primitives.Chars;
import io.druid.java.util.common.StringUtils;
Expand All @@ -34,6 +35,7 @@

import javax.annotation.Nullable;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.regex.Pattern;

public class LikeDimFilter implements DimFilter
Expand Down Expand Up @@ -306,6 +308,12 @@ public RangeSet<String> getDimensionRangeSet(String dimension)
return null;
}

@Override
public HashSet<String> getRequiredColumns()
{
return Sets.newHashSet(dimension);
}

@Override
public boolean equals(Object o)
{
Expand Down
7 changes: 7 additions & 0 deletions processing/src/main/java/io/druid/query/filter/NoopDimFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import com.google.common.collect.RangeSet;

import java.nio.ByteBuffer;
import java.util.HashSet;

/**
*/
Expand Down Expand Up @@ -50,4 +51,10 @@ public RangeSet<String> getDimensionRangeSet(String dimension)
{
return null;
}

@Override
public HashSet<String> getRequiredColumns()
{
return null;
}
}
Loading