diff --git a/docs/content/development/extensions-core/bloom-filter.md b/docs/content/development/extensions-core/bloom-filter.md index 140111d123a3..baff804ae9df 100644 --- a/docs/content/development/extensions-core/bloom-filter.md +++ b/docs/content/development/extensions-core/bloom-filter.md @@ -23,7 +23,7 @@ Internally, this implementation of bloom filter uses Murmur3 fast non-cryptograp { "type" : "bloom", "dimension" : , - "bloomKFilter" : , + "bloomFilter" : , "extractionFn" : } ``` @@ -32,14 +32,14 @@ Internally, this implementation of bloom filter uses Murmur3 fast non-cryptograp |-------------------------|------------------------------|----------------------------------| |`type` |Filter Type. Should always be `bloom`|yes| |`dimension` |The dimension to filter over. | yes | -|`bloomKFilter` |Base64 encoded Binary representation of `org.apache.hive.common.util.BloomKFilter`| yes | +|`bloomFilter` |Base64 encoded Binary representation of `org.apache.hive.common.util.BloomFilter`| yes | |`extractionFn`|[Extraction function](./../dimensionspecs.html#extraction-functions) to apply to the dimension values |no| -### Serialized Format for BloomKFilter - Serialized BloomKFilter format: +### Serialized Format for BloomFilter + Serialized BloomFilter format: - 1 byte for the number of hash functions. - 1 big endian int(That is how OutputStream works) for the number of longs in the bitset - - big endian longs in the BloomKFilter bitset + - big endian longs in the BloomFilter bitset -Note: `org.apache.hive.common.util.BloomKFilter` provides a serialize method which can be used to serialize bloom filters to outputStream. \ No newline at end of file +Note: `org.apache.hive.common.util.BloomFilter` provides a serialize method which can be used to serialize bloom filters to outputStream. \ No newline at end of file diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/guice/BloomFilterSerializersModule.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/guice/BloomFilterSerializersModule.java index 0072ff38d275..5ac758e01ba0 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/guice/BloomFilterSerializersModule.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/guice/BloomFilterSerializersModule.java @@ -27,8 +27,8 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.ser.std.StdSerializer; import org.apache.druid.query.filter.BloomDimFilter; -import org.apache.druid.query.filter.BloomKFilterHolder; -import org.apache.hive.common.util.BloomKFilter; +import org.apache.druid.query.filter.BloomFilterHolder; +import org.apache.hive.common.util.BloomFilter; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -41,65 +41,66 @@ public class BloomFilterSerializersModule extends SimpleModule public BloomFilterSerializersModule() { registerSubtypes(new NamedType(BloomDimFilter.class, BLOOM_FILTER_TYPE_NAME)); - addSerializer(BloomKFilter.class, new BloomKFilterSerializer()); - addDeserializer(BloomKFilter.class, new BloomKFilterDeserializer()); - addDeserializer(BloomKFilterHolder.class, new BloomKFilterHolderDeserializer()); + addSerializer(BloomFilter.class, new BloomFilterSerializer()); + addDeserializer(BloomFilter.class, new BloomFilterDeserializer()); + addDeserializer(BloomFilterHolder.class, new BloomFilterHolderDeserializer()); } - private static class BloomKFilterSerializer extends StdSerializer + private static class BloomFilterSerializer extends StdSerializer { - BloomKFilterSerializer() + BloomFilterSerializer() { - super(BloomKFilter.class); + super(BloomFilter.class); } @Override - public void serialize(BloomKFilter bloomKFilter, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) + public void serialize(BloomFilter bloomFilter, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { - jsonGenerator.writeBinary(bloomKFilterToBytes(bloomKFilter)); + jsonGenerator.writeBinary(bloomFilterToBytes(bloomFilter)); } } - private static class BloomKFilterDeserializer extends StdDeserializer + private static class BloomFilterDeserializer extends StdDeserializer { - BloomKFilterDeserializer() + BloomFilterDeserializer() { - super(BloomKFilter.class); + super(BloomFilter.class); } @Override - public BloomKFilter deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + public BloomFilter deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { - return bloomKFilterFromBytes(jsonParser.getBinaryValue()); + return bloomFilterFromBytes(jsonParser.getBinaryValue()); } } - private static class BloomKFilterHolderDeserializer extends StdDeserializer + private static class BloomFilterHolderDeserializer extends StdDeserializer { - BloomKFilterHolderDeserializer() + BloomFilterHolderDeserializer() { - super(BloomKFilterHolder.class); + super(BloomFilterHolder.class); } @Override - public BloomKFilterHolder deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + public BloomFilterHolder deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { - return BloomKFilterHolder.fromBytes(jsonParser.getBinaryValue()); + byte[] bytes = jsonParser.getBinaryValue(); + return BloomFilterHolder.fromBytes(bytes); } } - public static byte[] bloomKFilterToBytes(BloomKFilter bloomKFilter) throws IOException + public static byte[] bloomFilterToBytes(BloomFilter bloomFilter) throws IOException { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - BloomKFilter.serialize(byteArrayOutputStream, bloomKFilter); + BloomFilter.serialize(byteArrayOutputStream, bloomFilter); return byteArrayOutputStream.toByteArray(); } - public static BloomKFilter bloomKFilterFromBytes(byte[] bytes) throws IOException + public static BloomFilter bloomFilterFromBytes(byte[] bytes) throws IOException { - return BloomKFilter.deserialize(new ByteArrayInputStream(bytes)); + return BloomFilter.deserialize(new ByteArrayInputStream(bytes)); } } diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java index 383488cc41ac..d5492ae7aff0 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomDimFilter.java @@ -30,7 +30,7 @@ import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.segment.filter.DimensionPredicateFilter; -import org.apache.hive.common.util.BloomKFilter; +import org.apache.hive.common.util.BloomFilter; import java.util.HashSet; @@ -40,22 +40,22 @@ public class BloomDimFilter implements DimFilter { private final String dimension; - private final BloomKFilter bloomKFilter; + private final BloomFilter bloomFilter; private final HashCode hash; private final ExtractionFn extractionFn; @JsonCreator public BloomDimFilter( @JsonProperty("dimension") String dimension, - @JsonProperty("bloomKFilter") BloomKFilterHolder bloomKFilterHolder, + @JsonProperty("bloomFilter") BloomFilterHolder bloomFilterHolder, @JsonProperty("extractionFn") ExtractionFn extractionFn ) { Preconditions.checkArgument(dimension != null, "dimension must not be null"); - Preconditions.checkNotNull(bloomKFilterHolder); + Preconditions.checkNotNull(bloomFilterHolder); this.dimension = dimension; - this.bloomKFilter = bloomKFilterHolder.getFilter(); - this.hash = bloomKFilterHolder.getFilterHash(); + this.bloomFilter = bloomFilterHolder.getFilter(); + this.hash = bloomFilterHolder.getFilterHash(); this.extractionFn = extractionFn; } @@ -90,9 +90,9 @@ public Predicate makeStringPredicate() { return str -> { if (str == null) { - return bloomKFilter.testBytes(null, 0, 0); + return bloomFilter.testBytes(null, 0, 0); } - return bloomKFilter.testString(str); + return bloomFilter.testString(str); }; } @@ -104,13 +104,13 @@ public DruidLongPredicate makeLongPredicate() @Override public boolean applyLong(long input) { - return bloomKFilter.testLong(input); + return bloomFilter.testLong(input); } @Override public boolean applyNull() { - return bloomKFilter.testBytes(null, 0, 0); + return bloomFilter.testBytes(null, 0, 0); } }; } @@ -123,13 +123,13 @@ public DruidFloatPredicate makeFloatPredicate() @Override public boolean applyFloat(float input) { - return bloomKFilter.testFloat(input); + return bloomFilter.testDouble(input); } @Override public boolean applyNull() { - return bloomKFilter.testBytes(null, 0, 0); + return bloomFilter.testBytes(null, 0, 0); } }; } @@ -142,13 +142,13 @@ public DruidDoublePredicate makeDoublePredicate() @Override public boolean applyDouble(double input) { - return bloomKFilter.testDouble(input); + return bloomFilter.testDouble(input); } @Override public boolean applyNull() { - return bloomKFilter.testBytes(null, 0, 0); + return bloomFilter.testBytes(null, 0, 0); } }; } @@ -164,9 +164,9 @@ public String getDimension() } @JsonProperty - public BloomKFilter getBloomKFilter() + public BloomFilter getBloomFilter() { - return bloomKFilter; + return bloomFilter; } @JsonProperty diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomKFilterHolder.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomFilterHolder.java similarity index 69% rename from extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomKFilterHolder.java rename to extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomFilterHolder.java index e06632f0a60c..801b2464fb0f 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomKFilterHolder.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/BloomFilterHolder.java @@ -22,23 +22,23 @@ import com.google.common.hash.HashCode; import com.google.common.hash.Hashing; import org.apache.druid.guice.BloomFilterSerializersModule; -import org.apache.hive.common.util.BloomKFilter; +import org.apache.hive.common.util.BloomFilter; import java.io.IOException; import java.util.Objects; -public class BloomKFilterHolder +public class BloomFilterHolder { - private final BloomKFilter filter; + private final BloomFilter filter; private final HashCode hash; - public BloomKFilterHolder(BloomKFilter filter, HashCode hash) + public BloomFilterHolder(BloomFilter filter, HashCode hash) { this.filter = filter; this.hash = hash; } - BloomKFilter getFilter() + BloomFilter getFilter() { return filter; } @@ -48,17 +48,17 @@ HashCode getFilterHash() return hash; } - public static BloomKFilterHolder fromBloomKFilter(BloomKFilter filter) throws IOException + public static BloomFilterHolder fromBloomFilter(BloomFilter filter) throws IOException { - byte[] bytes = BloomFilterSerializersModule.bloomKFilterToBytes(filter); + byte[] bytes = BloomFilterSerializersModule.bloomFilterToBytes(filter); - return new BloomKFilterHolder(filter, Hashing.sha512().hashBytes(bytes)); + return new BloomFilterHolder(filter, Hashing.sha512().hashBytes(bytes)); } - public static BloomKFilterHolder fromBytes(byte[] bytes) throws IOException + public static BloomFilterHolder fromBytes(byte[] bytes) throws IOException { - return new BloomKFilterHolder( - BloomFilterSerializersModule.bloomKFilterFromBytes(bytes), + return new BloomFilterHolder( + BloomFilterSerializersModule.bloomFilterFromBytes(bytes), Hashing.sha512().hashBytes(bytes) ); } @@ -73,7 +73,7 @@ public boolean equals(Object o) return false; } - BloomKFilterHolder that = (BloomKFilterHolder) o; + BloomFilterHolder that = (BloomFilterHolder) o; return Objects.equals(this.hash, that.hash); } diff --git a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/filter/BloomDimFilterTest.java b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/filter/BloomDimFilterTest.java index 81a272f5f95a..c8fec2ab8701 100644 --- a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/filter/BloomDimFilterTest.java +++ b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/filter/BloomDimFilterTest.java @@ -41,7 +41,7 @@ import org.apache.druid.segment.StorageAdapter; import org.apache.druid.segment.filter.BaseFilterTest; import org.apache.druid.segment.incremental.IncrementalIndexSchema; -import org.apache.hive.common.util.BloomKFilter; +import org.apache.hive.common.util.BloomFilter; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -128,9 +128,9 @@ public static void tearDown() throws Exception @Test public void testSerde() throws IOException { - BloomKFilter bloomFilter = new BloomKFilter(1500); + BloomFilter bloomFilter = new BloomFilter(1500); bloomFilter.addString("myTestString"); - BloomKFilterHolder holder = new BloomKFilterHolder(bloomFilter, null); + BloomFilterHolder holder = new BloomFilterHolder(bloomFilter, null); BloomDimFilter bloomDimFilter = new BloomDimFilter( "abc", holder, @@ -141,8 +141,8 @@ public void testSerde() throws IOException BloomDimFilter serde = (BloomDimFilter) filter; Assert.assertEquals(bloomDimFilter.getDimension(), serde.getDimension()); Assert.assertEquals(bloomDimFilter.getExtractionFn(), serde.getExtractionFn()); - Assert.assertTrue(bloomDimFilter.getBloomKFilter().testString("myTestString")); - Assert.assertFalse(bloomDimFilter.getBloomKFilter().testString("not_match")); + Assert.assertTrue(bloomDimFilter.getBloomFilter().testString("myTestString")); + Assert.assertFalse(bloomDimFilter.getBloomFilter().testString("not_match")); } @Test @@ -150,22 +150,22 @@ public void testWithTimeExtractionFnNull() throws IOException { assertFilterMatches(new BloomDimFilter( "dim0", - bloomKFilter(1000, null, ""), + createBloomFilterHolder(1000, null, ""), new TimeDimExtractionFn("yyyy-MM-dd", "yyyy-MM", true) ), ImmutableList.of()); assertFilterMatches(new BloomDimFilter( "dim6", - bloomKFilter(1000, null, ""), + createBloomFilterHolder(1000, null, ""), new TimeDimExtractionFn("yyyy-MM-dd", "yyyy-MM", true) ), ImmutableList.of("3", "4", "5")); assertFilterMatches(new BloomDimFilter( "dim6", - bloomKFilter(1000, "2017-07"), + createBloomFilterHolder(1000, "2017-07"), new TimeDimExtractionFn("yyyy-MM-dd", "yyyy-MM", true) ), ImmutableList.of("0", "1")); assertFilterMatches(new BloomDimFilter( "dim6", - bloomKFilter(1000, "2017-05"), + createBloomFilterHolder(1000, "2017-05"), new TimeDimExtractionFn("yyyy-MM-dd", "yyyy-MM", true) ), ImmutableList.of("2")); } @@ -173,27 +173,27 @@ public void testWithTimeExtractionFnNull() throws IOException @Test public void testSingleValueStringColumnWithoutNulls() throws IOException { - assertFilterMatches(new BloomDimFilter("dim0", bloomKFilter(1000, (String) null), null), ImmutableList.of()); - assertFilterMatches(new BloomDimFilter("dim0", bloomKFilter(1000, ""), null), ImmutableList.of()); - assertFilterMatches(new BloomDimFilter("dim0", bloomKFilter(1000, "0"), null), ImmutableList.of("0")); - assertFilterMatches(new BloomDimFilter("dim0", bloomKFilter(1000, "1"), null), ImmutableList.of("1")); + assertFilterMatches(new BloomDimFilter("dim0", createBloomFilterHolder(1000, (String) null), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("dim0", createBloomFilterHolder(1000, ""), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("dim0", createBloomFilterHolder(1000, "0"), null), ImmutableList.of("0")); + assertFilterMatches(new BloomDimFilter("dim0", createBloomFilterHolder(1000, "1"), null), ImmutableList.of("1")); } @Test public void testSingleValueStringColumnWithNulls() throws IOException { if (NullHandling.replaceWithDefault()) { - assertFilterMatches(new BloomDimFilter("dim1", bloomKFilter(1000, (String) null), null), ImmutableList.of("0")); + assertFilterMatches(new BloomDimFilter("dim1", createBloomFilterHolder(1000, (String) null), null), ImmutableList.of("0")); } else { - assertFilterMatches(new BloomDimFilter("dim1", bloomKFilter(1000, (String) null), null), ImmutableList.of()); - assertFilterMatches(new BloomDimFilter("dim1", bloomKFilter(1000, ""), null), ImmutableList.of("0")); + assertFilterMatches(new BloomDimFilter("dim1", createBloomFilterHolder(1000, (String) null), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("dim1", createBloomFilterHolder(1000, ""), null), ImmutableList.of("0")); } - assertFilterMatches(new BloomDimFilter("dim1", bloomKFilter(1000, "10"), null), ImmutableList.of("1")); - assertFilterMatches(new BloomDimFilter("dim1", bloomKFilter(1000, "2"), null), ImmutableList.of("2")); - assertFilterMatches(new BloomDimFilter("dim1", bloomKFilter(1000, "1"), null), ImmutableList.of("3")); - assertFilterMatches(new BloomDimFilter("dim1", bloomKFilter(1000, "def"), null), ImmutableList.of("4")); - assertFilterMatches(new BloomDimFilter("dim1", bloomKFilter(1000, "abc"), null), ImmutableList.of("5")); - assertFilterMatches(new BloomDimFilter("dim1", bloomKFilter(1000, "ab"), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("dim1", createBloomFilterHolder(1000, "10"), null), ImmutableList.of("1")); + assertFilterMatches(new BloomDimFilter("dim1", createBloomFilterHolder(1000, "2"), null), ImmutableList.of("2")); + assertFilterMatches(new BloomDimFilter("dim1", createBloomFilterHolder(1000, "1"), null), ImmutableList.of("3")); + assertFilterMatches(new BloomDimFilter("dim1", createBloomFilterHolder(1000, "def"), null), ImmutableList.of("4")); + assertFilterMatches(new BloomDimFilter("dim1", createBloomFilterHolder(1000, "abc"), null), ImmutableList.of("5")); + assertFilterMatches(new BloomDimFilter("dim1", createBloomFilterHolder(1000, "ab"), null), ImmutableList.of()); } @Test @@ -201,66 +201,66 @@ public void testMultiValueStringColumn() throws IOException { if (NullHandling.replaceWithDefault()) { assertFilterMatches( - new BloomDimFilter("dim2", bloomKFilter(1000, (String) null), null), + new BloomDimFilter("dim2", createBloomFilterHolder(1000, (String) null), null), ImmutableList.of("1", "2", "5") ); } else { assertFilterMatches( - new BloomDimFilter("dim2", bloomKFilter(1000, (String) null), null), + new BloomDimFilter("dim2", createBloomFilterHolder(1000, (String) null), null), ImmutableList.of("1", "5") ); - assertFilterMatches(new BloomDimFilter("dim2", bloomKFilter(1000, ""), null), ImmutableList.of("2")); + assertFilterMatches(new BloomDimFilter("dim2", createBloomFilterHolder(1000, ""), null), ImmutableList.of("2")); } - assertFilterMatches(new BloomDimFilter("dim2", bloomKFilter(1000, "a"), null), ImmutableList.of("0", "3")); - assertFilterMatches(new BloomDimFilter("dim2", bloomKFilter(1000, "b"), null), ImmutableList.of("0")); - assertFilterMatches(new BloomDimFilter("dim2", bloomKFilter(1000, "c"), null), ImmutableList.of("4")); - assertFilterMatches(new BloomDimFilter("dim2", bloomKFilter(1000, "d"), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("dim2", createBloomFilterHolder(1000, "a"), null), ImmutableList.of("0", "3")); + assertFilterMatches(new BloomDimFilter("dim2", createBloomFilterHolder(1000, "b"), null), ImmutableList.of("0")); + assertFilterMatches(new BloomDimFilter("dim2", createBloomFilterHolder(1000, "c"), null), ImmutableList.of("4")); + assertFilterMatches(new BloomDimFilter("dim2", createBloomFilterHolder(1000, "d"), null), ImmutableList.of()); } @Test public void testMissingColumnSpecifiedInDimensionList() throws IOException { assertFilterMatches( - new BloomDimFilter("dim3", bloomKFilter(1000, (String) null), null), + new BloomDimFilter("dim3", createBloomFilterHolder(1000, (String) null), null), ImmutableList.of("0", "1", "2", "3", "4", "5") ); - assertFilterMatches(new BloomDimFilter("dim3", bloomKFilter(1000, ""), null), ImmutableList.of()); - assertFilterMatches(new BloomDimFilter("dim3", bloomKFilter(1000, "a"), null), ImmutableList.of()); - assertFilterMatches(new BloomDimFilter("dim3", bloomKFilter(1000, "b"), null), ImmutableList.of()); - assertFilterMatches(new BloomDimFilter("dim3", bloomKFilter(1000, "c"), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("dim3", createBloomFilterHolder(1000, ""), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("dim3", createBloomFilterHolder(1000, "a"), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("dim3", createBloomFilterHolder(1000, "b"), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("dim3", createBloomFilterHolder(1000, "c"), null), ImmutableList.of()); } @Test public void testMissingColumnNotSpecifiedInDimensionList() throws IOException { assertFilterMatches( - new BloomDimFilter("dim4", bloomKFilter(1000, (String) null), null), + new BloomDimFilter("dim4", createBloomFilterHolder(1000, (String) null), null), ImmutableList.of("0", "1", "2", "3", "4", "5") ); - assertFilterMatches(new BloomDimFilter("dim4", bloomKFilter(1000, ""), null), ImmutableList.of()); - assertFilterMatches(new BloomDimFilter("dim4", bloomKFilter(1000, "a"), null), ImmutableList.of()); - assertFilterMatches(new BloomDimFilter("dim4", bloomKFilter(1000, "b"), null), ImmutableList.of()); - assertFilterMatches(new BloomDimFilter("dim4", bloomKFilter(1000, "c"), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("dim4", createBloomFilterHolder(1000, ""), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("dim4", createBloomFilterHolder(1000, "a"), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("dim4", createBloomFilterHolder(1000, "b"), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("dim4", createBloomFilterHolder(1000, "c"), null), ImmutableList.of()); } @Test public void testExpressionVirtualColumn() throws IOException { assertFilterMatches( - new BloomDimFilter("expr", bloomKFilter(1000, 1.1F), null), + new BloomDimFilter("expr", createBloomFilterHolder(1000, 1.1F), null), ImmutableList.of("0", "1", "2", "3", "4", "5") ); - assertFilterMatches(new BloomDimFilter("expr", bloomKFilter(1000, 1.2F), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("expr", createBloomFilterHolder(1000, 1.2F), null), ImmutableList.of()); assertFilterMatches( - new BloomDimFilter("exprDouble", bloomKFilter(1000, 2.1D), null), + new BloomDimFilter("exprDouble", createBloomFilterHolder(1000, 2.1D), null), ImmutableList.of("0", "1", "2", "3", "4", "5") ); - assertFilterMatches(new BloomDimFilter("exprDouble", bloomKFilter(1000, 2.2D), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("exprDouble", createBloomFilterHolder(1000, 2.2D), null), ImmutableList.of()); assertFilterMatches( - new BloomDimFilter("exprLong", bloomKFilter(1000, 3L), null), + new BloomDimFilter("exprLong", createBloomFilterHolder(1000, 3L), null), ImmutableList.of("0", "1", "2", "3", "4", "5") ); - assertFilterMatches(new BloomDimFilter("exprLong", bloomKFilter(1000, 4L), null), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("exprLong", createBloomFilterHolder(1000, 4L), null), ImmutableList.of()); } @Test @@ -275,33 +275,33 @@ public void testSelectorWithLookupExtractionFn() throws IOException LookupExtractor mapExtractor = new MapLookupExtractor(stringMap, false); LookupExtractionFn lookupFn = new LookupExtractionFn(mapExtractor, false, "UNKNOWN", false, true); - assertFilterMatches(new BloomDimFilter("dim0", bloomKFilter(1000, "HELLO"), lookupFn), ImmutableList.of("1")); + assertFilterMatches(new BloomDimFilter("dim0", createBloomFilterHolder(1000, "HELLO"), lookupFn), ImmutableList.of("1")); assertFilterMatches( - new BloomDimFilter("dim0", bloomKFilter(1000, "UNKNOWN"), lookupFn), + new BloomDimFilter("dim0", createBloomFilterHolder(1000, "UNKNOWN"), lookupFn), ImmutableList.of("0", "2", "3", "4", "5") ); - assertFilterMatches(new BloomDimFilter("dim1", bloomKFilter(1000, "HELLO"), lookupFn), ImmutableList.of("3", "4")); + assertFilterMatches(new BloomDimFilter("dim1", createBloomFilterHolder(1000, "HELLO"), lookupFn), ImmutableList.of("3", "4")); assertFilterMatches( - new BloomDimFilter("dim1", bloomKFilter(1000, "UNKNOWN"), lookupFn), + new BloomDimFilter("dim1", createBloomFilterHolder(1000, "UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "5") ); - assertFilterMatches(new BloomDimFilter("dim2", bloomKFilter(1000, "HELLO"), lookupFn), ImmutableList.of("0", "3")); + assertFilterMatches(new BloomDimFilter("dim2", createBloomFilterHolder(1000, "HELLO"), lookupFn), ImmutableList.of("0", "3")); assertFilterMatches( - new BloomDimFilter("dim2", bloomKFilter(1000, "UNKNOWN"), lookupFn), + new BloomDimFilter("dim2", createBloomFilterHolder(1000, "UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "4", "5") ); - assertFilterMatches(new BloomDimFilter("dim3", bloomKFilter(1000, "HELLO"), lookupFn), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("dim3", createBloomFilterHolder(1000, "HELLO"), lookupFn), ImmutableList.of()); assertFilterMatches( - new BloomDimFilter("dim3", bloomKFilter(1000, "UNKNOWN"), lookupFn), + new BloomDimFilter("dim3", createBloomFilterHolder(1000, "UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "3", "4", "5") ); - assertFilterMatches(new BloomDimFilter("dim4", bloomKFilter(1000, "HELLO"), lookupFn), ImmutableList.of()); + assertFilterMatches(new BloomDimFilter("dim4", createBloomFilterHolder(1000, "HELLO"), lookupFn), ImmutableList.of()); assertFilterMatches( - new BloomDimFilter("dim4", bloomKFilter(1000, "UNKNOWN"), lookupFn), + new BloomDimFilter("dim4", createBloomFilterHolder(1000, "UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "3", "4", "5") ); @@ -310,7 +310,7 @@ public void testSelectorWithLookupExtractionFn() throws IOException ); LookupExtractor mapExtractor2 = new MapLookupExtractor(stringMap2, false); LookupExtractionFn lookupFn2 = new LookupExtractionFn(mapExtractor2, true, null, false, true); - assertFilterMatches(new BloomDimFilter("dim0", bloomKFilter(1000, "5"), lookupFn2), ImmutableList.of("2", "5")); + assertFilterMatches(new BloomDimFilter("dim0", createBloomFilterHolder(1000, "5"), lookupFn2), ImmutableList.of("2", "5")); final Map stringMap3 = ImmutableMap.of( "1", "" @@ -320,16 +320,16 @@ public void testSelectorWithLookupExtractionFn() throws IOException if (NullHandling.replaceWithDefault()) { // Nulls and empty strings are considered equivalent assertFilterMatches( - new BloomDimFilter("dim0", bloomKFilter(1000, (String) null), lookupFn3), + new BloomDimFilter("dim0", createBloomFilterHolder(1000, (String) null), lookupFn3), ImmutableList.of("0", "1", "2", "3", "4", "5") ); } else { assertFilterMatches( - new BloomDimFilter("dim0", bloomKFilter(1000, (String) null), lookupFn3), + new BloomDimFilter("dim0", createBloomFilterHolder(1000, (String) null), lookupFn3), ImmutableList.of("0", "2", "3", "4", "5") ); assertFilterMatches( - new BloomDimFilter("dim0", bloomKFilter(1000, ""), lookupFn3), + new BloomDimFilter("dim0", createBloomFilterHolder(1000, ""), lookupFn3), ImmutableList.of("1") ); } @@ -338,11 +338,11 @@ public void testSelectorWithLookupExtractionFn() throws IOException @Test public void testCacheKeyIsNotGiantIfFilterIsGiant() throws IOException { - BloomKFilter bloomFilter = new BloomKFilter(10_000_000); + BloomFilter bloomFilter = new BloomFilter(10_000_000); // FILL IT UP! bloomFilter.addString("myTestString"); - BloomKFilterHolder holder = BloomKFilterHolder.fromBloomKFilter(bloomFilter); + BloomFilterHolder holder = BloomFilterHolder.fromBloomFilter(bloomFilter); BloomDimFilter bloomDimFilter = new BloomDimFilter( "abc", @@ -350,7 +350,7 @@ public void testCacheKeyIsNotGiantIfFilterIsGiant() throws IOException new TimeDimExtractionFn("yyyy-MM-dd", "yyyy-MM", true) ); - byte[] bloomFilterBytes = BloomFilterSerializersModule.bloomKFilterToBytes(bloomFilter); + byte[] bloomFilterBytes = BloomFilterSerializersModule.bloomFilterToBytes(bloomFilter); // serialized filter can be quite large for high capacity bloom filters... Assert.assertTrue(bloomFilterBytes.length > 7794000); @@ -360,9 +360,9 @@ public void testCacheKeyIsNotGiantIfFilterIsGiant() throws IOException Assert.assertTrue(actualSize < 100); } - private static BloomKFilterHolder bloomKFilter(int expectedEntries, String... values) throws IOException + private static BloomFilterHolder createBloomFilterHolder(int expectedEntries, String... values) throws IOException { - BloomKFilter filter = new BloomKFilter(expectedEntries); + BloomFilter filter = new BloomFilter(expectedEntries); for (String value : values) { if (value == null) { filter.addBytes(null, 0, 0); @@ -371,25 +371,25 @@ private static BloomKFilterHolder bloomKFilter(int expectedEntries, String... va } } - return BloomKFilterHolder.fromBloomKFilter(filter); + return BloomFilterHolder.fromBloomFilter(filter); } - private static BloomKFilterHolder bloomKFilter(int expectedEntries, Float... values) throws IOException + private static BloomFilterHolder createBloomFilterHolder(int expectedEntries, Float... values) throws IOException { - BloomKFilter filter = new BloomKFilter(expectedEntries); + BloomFilter filter = new BloomFilter(expectedEntries); for (Float value : values) { if (value == null) { filter.addBytes(null, 0, 0); } else { - filter.addFloat(value); + filter.addDouble(value); } } - return BloomKFilterHolder.fromBloomKFilter(filter); + return BloomFilterHolder.fromBloomFilter(filter); } - private static BloomKFilterHolder bloomKFilter(int expectedEntries, Double... values) throws IOException + private static BloomFilterHolder createBloomFilterHolder(int expectedEntries, Double... values) throws IOException { - BloomKFilter filter = new BloomKFilter(expectedEntries); + BloomFilter filter = new BloomFilter(expectedEntries); for (Double value : values) { if (value == null) { filter.addBytes(null, 0, 0); @@ -397,12 +397,12 @@ private static BloomKFilterHolder bloomKFilter(int expectedEntries, Double... va filter.addDouble(value); } } - return BloomKFilterHolder.fromBloomKFilter(filter); + return BloomFilterHolder.fromBloomFilter(filter); } - private static BloomKFilterHolder bloomKFilter(int expectedEntries, Long... values) throws IOException + private static BloomFilterHolder createBloomFilterHolder(int expectedEntries, Long... values) throws IOException { - BloomKFilter filter = new BloomKFilter(expectedEntries); + BloomFilter filter = new BloomFilter(expectedEntries); for (Long value : values) { if (value == null) { filter.addBytes(null, 0, 0); @@ -410,6 +410,6 @@ private static BloomKFilterHolder bloomKFilter(int expectedEntries, Long... valu filter.addLong(value); } } - return BloomKFilterHolder.fromBloomKFilter(filter); + return BloomFilterHolder.fromBloomFilter(filter); } }