Skip to content

Add segment pruning based on secondary partition dimension #2982

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fjy merged 9 commits into from
Jun 24, 2016
Merged

Add segment pruning based on secondary partition dimension #2982

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
baef042
add get dimension rangeset to filters
acslk May 10, 2016
d6bbc5c
add get domain to ShardSpec and added chunk filter in caching cluster…
acslk May 11, 2016
15d2517
add null check and modified not filter, started with unit test
acslk May 12, 2016
e0e1d9f
add filter test with caching
acslk May 17, 2016
c06e631
refactor and some comments
acslk Jun 17, 2016
7fa41d6
extract filtershard to helper function
acslk Jun 23, 2016
75b11ca
fixup
acslk Jun 23, 2016
09f85f2
minor changes
acslk Jun 24, 2016
20aa962
update javadoc
acslk Jun 24, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions api/src/main/java/io/druid/timeline/partition/NoneShardSpec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,12 @@
package io.druid.timeline.partition;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Range;
import io.druid.data.input.InputRow;

import java.util.List;
import java.util.Map;

/**
*/
Expand Down Expand Up @@ -71,6 +74,12 @@ public ShardSpec getShardSpec(long timestamp, InputRow row)
};
}

@Override
public Map<String, Range<String>> getDomain()
{
return ImmutableMap.of();
}

@Override
public boolean equals(Object obj)
{
Expand Down
9 changes: 9 additions & 0 deletions api/src/main/java/io/druid/timeline/partition/ShardSpec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@

import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.google.common.collect.Range;
import io.druid.data.input.InputRow;

import java.util.List;
import java.util.Map;

/**
* A Marker interface that exists to combine ShardSpec objects together for Jackson
Expand All @@ -41,4 +43,11 @@ public interface ShardSpec
public int getPartitionNum();

public ShardSpecLookup getLookup(List<ShardSpec> shardSpecs);

/**
* Get the possible range of each dimension for the rows this shard contains.
*
* @return map of dimensions to its possible range. Dimensions with unknown possible range are not mapped
*/
public Map<String, Range<String>> getDomain();
}
7 changes: 7 additions & 0 deletions api/src/test/java/io/druid/timeline/DataSegmentTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
import io.druid.TestObjectMapper;
import io.druid.data.input.InputRow;
Expand Down Expand Up @@ -76,6 +77,12 @@ public ShardSpecLookup getLookup(List<ShardSpec> shardSpecs)
{
return null;
}

@Override
public Map<String, Range<String>> getDomain()
{
return ImmutableMap.of();
}
};
}

Expand Down
3 changes: 3 additions & 0 deletions processing/src/main/java/io/druid/query/Query.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.google.common.collect.Ordering;
import com.metamx.common.guava.Sequence;
import io.druid.query.datasourcemetadata.DataSourceMetadataQuery;
import io.druid.query.filter.DimFilter;
import io.druid.query.groupby.GroupByQuery;
import io.druid.query.metadata.metadata.SegmentMetadataQuery;
import io.druid.query.search.search.SearchQuery;
Expand Down Expand Up @@ -65,6 +66,8 @@ public interface Query<T>

boolean hasFilters();

DimFilter getFilter();
Copy link
Copy Markdown
Member

@nishantmonu51 nishantmonu51 May 18, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

suggestion : change return type to Optional since not all queries have filters.

Copy link
Copy Markdown
Contributor

@gianm gianm May 18, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IMO we have a long and proud history of null return for various Query things, and this is not the right PR to change those to Optional.

Also I'm not totally sure Guava Optional actually makes the code significantly better, it's missing stuff like foreach / ifPresent, map, and flatMap that make Scala Options and Java 8 Optionals really usable.


String getType();

Sequence<T> run(QuerySegmentWalker walker, Map<String, Object> context);
Expand Down
7 changes: 7 additions & 0 deletions processing/src/main/java/io/druid/query/datasourcemetadata/DataSourceMetadataQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import io.druid.query.DataSource;
import io.druid.query.Query;
import io.druid.query.Result;
import io.druid.query.filter.DimFilter;
import io.druid.query.spec.MultipleIntervalSegmentSpec;
import io.druid.query.spec.QuerySegmentSpec;
import org.joda.time.DateTime;
Expand Down Expand Up @@ -66,6 +67,12 @@ public boolean hasFilters()
return false;
}

@Override
public DimFilter getFilter()
{
return null;
}

@Override
public String getType()
{
Expand Down
6 changes: 3 additions & 3 deletions processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.metamx.common.StringUtils;
import io.druid.query.filter.DimFilterCacheHelper;
import io.druid.query.filter.DimFilterUtils;
import io.druid.segment.DimensionSelector;

import java.nio.ByteBuffer;
Expand Down Expand Up @@ -119,10 +119,10 @@ public byte[] getCacheKey()
.put(CACHE_TYPE_ID)
.put(delegateCacheKey)
.put((byte) (isWhitelist ? 1 : 0))
.put(DimFilterCacheHelper.STRING_SEPARATOR);
.put(DimFilterUtils.STRING_SEPARATOR);
for (byte[] bytes : valuesBytes) {
filterCacheKey.put(bytes)
.put(DimFilterCacheHelper.STRING_SEPARATOR);
.put(DimFilterUtils.STRING_SEPARATOR);
}
return filterCacheKey.array();
}
Expand Down
10 changes: 5 additions & 5 deletions processing/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import com.google.common.base.Strings;
import com.metamx.common.StringUtils;
import io.druid.query.extraction.ExtractionFn;
import io.druid.query.filter.DimFilterCacheHelper;
import io.druid.query.filter.DimFilterUtils;
import io.druid.query.lookup.LookupExtractionFn;
import io.druid.query.lookup.LookupExtractor;
import io.druid.query.lookup.LookupReferencesManager;
Expand Down Expand Up @@ -167,13 +167,13 @@ public byte[] getCacheKey()
+ replaceWithBytes.length)
.put(CACHE_TYPE_ID)
.put(dimensionBytes)
.put(DimFilterCacheHelper.STRING_SEPARATOR)
.put(DimFilterUtils.STRING_SEPARATOR)
.put(outputNameBytes)
.put(DimFilterCacheHelper.STRING_SEPARATOR)
.put(DimFilterUtils.STRING_SEPARATOR)
.put(dimExtractionFnBytes)
.put(DimFilterCacheHelper.STRING_SEPARATOR)
.put(DimFilterUtils.STRING_SEPARATOR)
.put(replaceWithBytes)
.put(DimFilterCacheHelper.STRING_SEPARATOR)
.put(DimFilterUtils.STRING_SEPARATOR)
.put(retainMissingValue == true ? (byte) 1 : (byte) 0)
.array();
}
Expand Down
4 changes: 2 additions & 2 deletions processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.metamx.common.StringUtils;
import io.druid.query.filter.DimFilterCacheHelper;
import io.druid.query.filter.DimFilterUtils;
import io.druid.segment.DimensionSelector;

import java.nio.ByteBuffer;
Expand Down Expand Up @@ -89,7 +89,7 @@ public byte[] getCacheKey()
return ByteBuffer.allocate(2 + delegateCacheKey.length + regexBytes.length)
.put(CACHE_TYPE_ID)
.put(delegateCacheKey)
.put(DimFilterCacheHelper.STRING_SEPARATOR)
.put(DimFilterUtils.STRING_SEPARATOR)
.put(regexBytes)
.array();
}
Expand Down
21 changes: 20 additions & 1 deletion processing/src/main/java/io/druid/query/filter/AndDimFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.RangeSet;
import com.google.common.collect.TreeRangeSet;
import io.druid.query.Druids;
import io.druid.segment.filter.AndFilter;
import io.druid.segment.filter.Filters;
Expand Down Expand Up @@ -56,7 +58,7 @@ public List<DimFilter> getFields()
@Override
public byte[] getCacheKey()
{
return DimFilterCacheHelper.computeCacheKey(DimFilterCacheHelper.AND_CACHE_ID, fields);
return DimFilterUtils.computeCacheKey(DimFilterUtils.AND_CACHE_ID, fields);
}

@Override
Expand All @@ -72,6 +74,23 @@ public Filter toFilter()
return new AndFilter(Filters.toFilters(fields));
}

@Override
public RangeSet<String> getDimensionRangeSet(String dimension)
{
RangeSet<String> retSet = null;
for (DimFilter field : fields) {
RangeSet<String> rangeSet = field.getDimensionRangeSet(dimension);
if (rangeSet != null) {
if (retSet == null) {
retSet = TreeRangeSet.create(rangeSet);
} else {
retSet.removeAll(rangeSet.complement());
}
}
}
return retSet;
}

@Override
public boolean equals(Object o)
{
Expand Down
35 changes: 30 additions & 5 deletions processing/src/main/java/io/druid/query/filter/BoundDimFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,16 @@
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import com.google.common.collect.BoundType;
import com.google.common.collect.Range;
import com.google.common.collect.RangeSet;
import com.google.common.collect.TreeRangeSet;
import com.metamx.common.StringUtils;
import io.druid.query.extraction.ExtractionFn;
import io.druid.segment.filter.BoundFilter;

import java.nio.ByteBuffer;
import java.util.Objects;

public class BoundDimFilter implements DimFilter
{
Expand Down Expand Up @@ -137,18 +142,18 @@ public byte[] getCacheKey()
+ lowerBytes.length
+ extractionFnBytes.length
);
boundCacheBuffer.put(DimFilterCacheHelper.BOUND_CACHE_ID)
boundCacheBuffer.put(DimFilterUtils.BOUND_CACHE_ID)
.put(boundType)
.put(upperStrictByte)
.put(lowerStrictByte)
.put(AlphaNumericByte)
.put(DimFilterCacheHelper.STRING_SEPARATOR)
.put(DimFilterUtils.STRING_SEPARATOR)
.put(dimensionBytes)
.put(DimFilterCacheHelper.STRING_SEPARATOR)
.put(DimFilterUtils.STRING_SEPARATOR)
.put(upperBytes)
.put(DimFilterCacheHelper.STRING_SEPARATOR)
.put(DimFilterUtils.STRING_SEPARATOR)
.put(lowerBytes)
.put(DimFilterCacheHelper.STRING_SEPARATOR)
.put(DimFilterUtils.STRING_SEPARATOR)
.put(extractionFnBytes);
return boundCacheBuffer.array();
}
Expand All @@ -165,6 +170,26 @@ public Filter toFilter()
return new BoundFilter(this);
}

@Override
public RangeSet<String> getDimensionRangeSet(String dimension)
{
if (!Objects.equals(getDimension(), dimension) || getExtractionFn() != null || alphaNumeric) {
return null;
}
RangeSet<String> retSet = TreeRangeSet.create();
Range<String> range;
if (getLower() == null) {
range = isUpperStrict() ? Range.lessThan(getUpper()) : Range.atMost(getUpper());
} else if (getUpper() == null) {
range = isLowerStrict() ? Range.greaterThan(getLower()) : Range.atLeast(getLower());
} else {
range = Range.range(getLower(), isLowerStrict() ? BoundType.OPEN : BoundType.CLOSED,
getUpper(), isUpperStrict() ? BoundType.OPEN : BoundType.CLOSED);
}
retSet.add(range);
return retSet;
}

@Override
public boolean equals(Object o)
{
Expand Down
16 changes: 16 additions & 0 deletions processing/src/main/java/io/druid/query/filter/DimFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.google.common.collect.RangeSet;

/**
*/
Expand Down Expand Up @@ -56,4 +57,19 @@ public interface DimFilter
* @return a Filter that implements this DimFilter, or null if this DimFilter is a no-op.
*/
public Filter toFilter();

/**
* Returns a RangeSet that represents the possible range of the input dimension for this DimFilter.This is
* applicable to filters that use dimensions such as select, in, bound, and logical filters such as and, or, not.
*
* Null represents that the range cannot be determined, and will be returned for filters such as javascript and regex
* where there's no easy way to determine the filtered range. It is treated the same way as an all range in most
* cases, however there are some subtle difference at logical filters such as not filter, where complement of all
* is nothing while complement of null is still null.
*
* @param dimension name of the dimension to get range for
* @return a RangeSet that represent the possible range of the input dimension, or null if it is not possible to
* determine for this DimFilter.
*/
public RangeSet<String> getDimensionRangeSet(String dimension);
}
65 changes: 0 additions & 65 deletions processing/src/main/java/io/druid/query/filter/DimFilterCacheHelper.java
View file
Open in desktop

This file was deleted.

Loading