From a8173fecf829746e84e0d1d2f7d51b139922d29f Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 6 Jun 2018 12:08:51 -0700 Subject: [PATCH 1/2] Lazy-ify IncrementalIndex filtering too. Follow-up to #5403, which only lazy-ified cursor-based filtering on QueryableIndex. --- .../druid/segment/DimensionSelectorUtils.java | 2 +- .../druid/segment/StringDimensionIndexer.java | 20 ++++++++++++++----- .../column/SimpleDictionaryEncodedColumn.java | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java b/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java index 092675c85510..b1df53cd9045 100644 --- a/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java +++ b/processing/src/main/java/io/druid/segment/DimensionSelectorUtils.java @@ -52,7 +52,7 @@ public static ValueMatcher makeValueMatcherGeneric(DimensionSelector selector, S if (idLookup != null) { return makeDictionaryEncodedValueMatcherGeneric(selector, idLookup.lookupId(value), value == null); } else if (selector.getValueCardinality() >= 0 && selector.nameLookupPossibleInAdvance()) { - // Employ precomputed BitSet optimization + // Employ caching BitSet optimization return makeDictionaryEncodedValueMatcherGeneric(selector, Predicates.equalTo(value)); } else { return makeNonDictionaryEncodedValueMatcherGeneric(selector, value); diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index df28aa43f2e8..58ed13db73ee 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -177,7 +177,7 @@ public SortedDimensionDictionary(List idToValue, int length) this.idToIndex = new int[length]; this.indexToId = new int[length]; int index = 0; - for (IntIterator iterator = sortedMap.values().iterator(); iterator.hasNext();) { + for (IntIterator iterator = sortedMap.values().iterator(); iterator.hasNext(); ) { int id = iterator.nextInt(); idToIndex[id] = index; indexToId[index] = id; @@ -496,7 +496,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) return BooleanValueMatcher.of(false); } } else { - // Employ precomputed BitSet optimization + // Employ caching BitSet optimization return makeValueMatcher(Predicates.equalTo(value)); } } @@ -504,8 +504,11 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public ValueMatcher makeValueMatcher(final Predicate predicate) { - final BitSet predicateMatchingValueIds = DimensionSelectorUtils.makePredicateMatchingSet(this, predicate); + final BitSet checkedIds = new BitSet(maxId); + final BitSet matchingIds = new BitSet(maxId); final boolean matchNull = predicate.apply(null); + + // Lazy matcher; only check an id if matches() is called. return new ValueMatcher() { @Override @@ -522,8 +525,15 @@ public boolean matches() } for (int id : dimsInt) { - if (predicateMatchingValueIds.get(id)) { - return true; + if (checkedIds.get(id)) { + return matchingIds.get(id); + } else { + final boolean matches = predicate.apply(lookupName(id)); + checkedIds.set(id); + if (matches) { + matchingIds.set(id); + return true; + } } } return false; diff --git a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java index ab12164720b3..3b3d8fe4f717 100644 --- a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java @@ -259,7 +259,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) return BooleanValueMatcher.of(false); } } else { - // Employ precomputed BitSet optimization + // Employ caching BitSet optimization return makeValueMatcher(Predicates.equalTo(value)); } } From 4519a3bf1c29f5caee9b346514b3fc7bad309bca Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 6 Jun 2018 19:04:21 -0400 Subject: [PATCH 2/2] Fix logic error. --- .../main/java/io/druid/segment/StringDimensionIndexer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index 58ed13db73ee..e36b223731f5 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -526,7 +526,9 @@ public boolean matches() for (int id : dimsInt) { if (checkedIds.get(id)) { - return matchingIds.get(id); + if (matchingIds.get(id)) { + return true; + } } else { final boolean matches = predicate.apply(lookupName(id)); checkedIds.set(id);