diff --git a/docs/content/querying/dimensionspecs.md b/docs/content/querying/dimensionspecs.md index 5a65a50856a5..1587326ae36d 100644 --- a/docs/content/querying/dimensionspecs.md +++ b/docs/content/querying/dimensionspecs.md @@ -252,7 +252,7 @@ It is illegal to set `retainMissingValue = true` and also specify a `replaceMiss A property of `injective` specifies if optimizations can be used which assume there is no combining of multiple names into one. For example: If ABC123 is the only key that maps to SomeCompany, that can be optimized since it is a unique lookup. But if both ABC123 and DEF456 BOTH map to SomeCompany, then that is NOT a unique lookup. Setting this value to true and setting `retainMissingValue` to FALSE (the default) may cause undesired behavior. -A property `optimize` can be supplied to allow optimization of lookup based extraction filter (by default `optimize = false`). +A property `optimize` can be supplied to allow optimization of lookup based extraction filter (by default `optimize = true`). The optimization layer will run on the broker and it will rewrite the extraction filter as clause of selector filters. For instance the following filter @@ -393,3 +393,41 @@ or without setting "locale" (in this case, the current value of the default loca "type" : "lower" } ``` + +### Lookup DimensionSpecs + +Lookup DimensionSpecs can be used to define directly a lookup implementation as dimension spec. +Generally speaking there is two different kind of lookups implementations. +The first kind is passed at the query time like `map` implementation. + +```json +{ + "type":"lookup", + "dimension":"dimensionName", + "outputName":"dimensionOutputName", + "replaceMissingValuesWith":"missing_value", + "retainMissingValue":false, + "lookup":{"type": "map", "map":{"key":"value"}, "isOneToOne":false} +} +``` + +A property of `retainMissingValue` and `replaceMissingValueWith` can be specified at query time to hint how to handle missing values. Setting `replaceMissingValueWith` to `""` has the same effect as setting it to `null` or omitting the property. +Setting `retainMissingValue` to true will use the dimension's original value if it is not found in the lookup. +The default values are `replaceMissingValueWith = null` and `retainMissingValue = false` which causes missing values to be treated as missing. + +It is illegal to set `retainMissingValue = true` and also specify a `replaceMissingValueWith`. + +A property of `injective` specifies if optimizations can be used which assume there is no combining of multiple names into one. For example: If ABC123 is the only key that maps to SomeCompany, that can be optimized since it is a unique lookup. But if both ABC123 and DEF456 BOTH map to SomeCompany, then that is NOT a unique lookup. Setting this value to true and setting `retainMissingValue` to FALSE (the default) may cause undesired behavior. + +A property `optimize` can be supplied to allow optimization of lookup based extraction filter (by default `optimize = true`). + +The second kind where it is not possible to pass at query time due to their size, will be based on an external lookup table or resource that is already registered via configuration file or/and coordinator. + +```json +{ + "type":"lookup" + "dimension":"dimensionName" + "outputName":"dimensionOutputName" + "name":"lookupName" +} +``` diff --git a/extensions/namespace-lookup/src/main/java/io/druid/query/extraction/NamespacedExtractor.java b/extensions/namespace-lookup/src/main/java/io/druid/query/extraction/NamespacedExtractor.java index 659adb3f8f6b..276c80f13a9b 100644 --- a/extensions/namespace-lookup/src/main/java/io/druid/query/extraction/NamespacedExtractor.java +++ b/extensions/namespace-lookup/src/main/java/io/druid/query/extraction/NamespacedExtractor.java @@ -87,4 +87,5 @@ public List unapply(@NotNull String value) { return reverseExtractionFunction.apply(value); } + } diff --git a/pom.xml b/pom.xml index c5c84cb2fe5d..cacd1aad2a5e 100644 --- a/pom.xml +++ b/pom.xml @@ -585,6 +585,12 @@ 1.3 test + + pl.pragmatists + JUnitParams + 1.0.4 + test + diff --git a/processing/pom.xml b/processing/pom.xml index d28dce7e5ac1..026cfe9a1bf0 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -101,6 +101,11 @@ caliper test + + pl.pragmatists + JUnitParams + test + diff --git a/processing/src/main/java/io/druid/query/dimension/DimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/DimensionSpec.java index 807da4d2c9b3..21b9201d8c1c 100644 --- a/processing/src/main/java/io/druid/query/dimension/DimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/DimensionSpec.java @@ -31,7 +31,8 @@ @JsonSubTypes.Type(name = "default", value = DefaultDimensionSpec.class), @JsonSubTypes.Type(name = "extraction", value = ExtractionDimensionSpec.class), @JsonSubTypes.Type(name = "regexFiltered", value = RegexFilteredDimensionSpec.class), - @JsonSubTypes.Type(name = "listFiltered", value = ListFilteredDimensionSpec.class) + @JsonSubTypes.Type(name = "listFiltered", value = ListFilteredDimensionSpec.class), + @JsonSubTypes.Type(name = "lookup", value = LookupDimensionSpec.class) }) public interface DimensionSpec { diff --git a/processing/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java new file mode 100644 index 000000000000..31b9118c9a3f --- /dev/null +++ b/processing/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java @@ -0,0 +1,235 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.dimension; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.metamx.common.StringUtils; +import io.druid.query.extraction.ExtractionFn; +import io.druid.query.extraction.LookupExtractionFn; +import io.druid.query.extraction.LookupExtractor; +import io.druid.query.extraction.LookupReferencesManager; +import io.druid.query.filter.DimFilterCacheHelper; +import io.druid.segment.DimensionSelector; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +public class LookupDimensionSpec implements DimensionSpec +{ + private static final byte CACHE_TYPE_ID = 0x4; + + @JsonProperty + private final String dimension; + + @JsonProperty + private final String outputName; + + @JsonProperty + private final LookupExtractor lookup; + + @JsonProperty + private final boolean retainMissingValue; + + @JsonProperty + private final String replaceMissingValueWith; + + @JsonProperty + private final String name; + + @JsonProperty + private final boolean optimize; + + private final LookupReferencesManager lookupReferencesManager; + + @JsonCreator + public LookupDimensionSpec( + @JsonProperty("dimension") String dimension, + @JsonProperty("outputName") String outputName, + @JsonProperty("lookup") LookupExtractor lookup, + @JsonProperty("retainMissingValue") boolean retainMissingValue, + @JsonProperty("replaceMissingValueWith") String replaceMissingValueWith, + @JsonProperty("name") String name, + @JacksonInject LookupReferencesManager lookupReferencesManager, + @JsonProperty("optimize") Boolean optimize + ) + { + this.retainMissingValue = retainMissingValue; + this.optimize = optimize == null ? true : optimize; + this.replaceMissingValueWith = Strings.emptyToNull(replaceMissingValueWith); + this.dimension = Preconditions.checkNotNull(dimension, "dimension can not be Null"); + this.outputName = Preconditions.checkNotNull(outputName, "outputName can not be Null"); + this.lookupReferencesManager = lookupReferencesManager; + this.name = name; + this.lookup = lookup; + Preconditions.checkArgument( + Strings.isNullOrEmpty(name) ^ (lookup == null), + "name [%s] and lookup [%s] are mutually exclusive please provide either a name or a lookup", name, lookup + ); + + if (!Strings.isNullOrEmpty(name)) { + Preconditions.checkNotNull( + this.lookupReferencesManager, + "The system is not configured to allow for lookups, please read about configuring a lookup manager in the docs" + ); + } + } + + @Override + @JsonProperty + public String getDimension() + { + return dimension; + } + + @Override + @JsonProperty + public String getOutputName() + { + return outputName; + } + + @JsonProperty + @Nullable + public LookupExtractor getLookup() + { + return lookup; + } + + @JsonProperty + @Nullable + public String getName() + { + return name; + } + + @Override + public ExtractionFn getExtractionFn() + { + final LookupExtractor lookupExtractor = Strings.isNullOrEmpty(name) + ? this.lookup + : Preconditions.checkNotNull( + this.lookupReferencesManager.get(name).get(), + "can not find lookup with name [%s]", + name + ); + return new LookupExtractionFn( + lookupExtractor, + retainMissingValue, + replaceMissingValueWith, + lookupExtractor.isOneToOne(), + optimize + ); + + } + + @Override + public DimensionSelector decorate(DimensionSelector selector) + { + return selector; + } + + @Override + public byte[] getCacheKey() + { + byte[] dimensionBytes = StringUtils.toUtf8(dimension); + byte[] dimExtractionFnBytes = Strings.isNullOrEmpty(name) + ? getLookup().getCacheKey() + : StringUtils.toUtf8(name); + byte[] outputNameBytes = StringUtils.toUtf8(outputName); + byte[] replaceWithBytes = StringUtils.toUtf8(Strings.nullToEmpty(replaceMissingValueWith)); + + + return ByteBuffer.allocate(6 + + dimensionBytes.length + + outputNameBytes.length + + dimExtractionFnBytes.length + + replaceWithBytes.length) + .put(CACHE_TYPE_ID) + .put(dimensionBytes) + .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(outputNameBytes) + .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(dimExtractionFnBytes) + .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(replaceWithBytes) + .put(DimFilterCacheHelper.STRING_SEPARATOR) + .put(retainMissingValue == true ? (byte) 1 : (byte) 0) + .array(); + } + + @Override + public boolean preservesOrdering() + { + return getExtractionFn().preservesOrdering(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof LookupDimensionSpec)) { + return false; + } + + LookupDimensionSpec that = (LookupDimensionSpec) o; + + if (retainMissingValue != that.retainMissingValue) { + return false; + } + if (optimize != that.optimize) { + return false; + } + if (!getDimension().equals(that.getDimension())) { + return false; + } + if (!getOutputName().equals(that.getOutputName())) { + return false; + } + if (getLookup() != null ? !getLookup().equals(that.getLookup()) : that.getLookup() != null) { + return false; + } + if (replaceMissingValueWith != null + ? !replaceMissingValueWith.equals(that.replaceMissingValueWith) + : that.replaceMissingValueWith != null) { + return false; + } + return getName() != null ? getName().equals(that.getName()) : that.getName() == null; + + } + + @Override + public int hashCode() + { + int result = getDimension().hashCode(); + result = 31 * result + getOutputName().hashCode(); + result = 31 * result + (getLookup() != null ? getLookup().hashCode() : 0); + result = 31 * result + (retainMissingValue ? 1 : 0); + result = 31 * result + (replaceMissingValueWith != null ? replaceMissingValueWith.hashCode() : 0); + result = 31 * result + (getName() != null ? getName().hashCode() : 0); + result = 31 * result + (optimize ? 1 : 0); + return result; + } +} diff --git a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java index c092b41d4a14..db3548b76f64 100644 --- a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java @@ -65,10 +65,6 @@ public DimensionSelector decorate(final DimensionSelector selector) return selector; } - if (selector == null) { - return selector; - } - int count = 0; final Map forwardMapping = new HashMap<>(); diff --git a/processing/src/main/java/io/druid/query/extraction/LookupExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/LookupExtractionFn.java index d12b545116eb..d099e9e7b9a0 100644 --- a/processing/src/main/java/io/druid/query/extraction/LookupExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/LookupExtractionFn.java @@ -60,7 +60,7 @@ public String apply(String input) injective ); this.lookup = lookup; - this.optimize = optimize == null ? false : optimize; + this.optimize = optimize == null ? true : optimize; } diff --git a/processing/src/main/java/io/druid/query/extraction/LookupExtractor.java b/processing/src/main/java/io/druid/query/extraction/LookupExtractor.java index 2de937df05ef..2aeb31d40468 100644 --- a/processing/src/main/java/io/druid/query/extraction/LookupExtractor.java +++ b/processing/src/main/java/io/druid/query/extraction/LookupExtractor.java @@ -106,6 +106,11 @@ public Map> unapplyAll(Iterable values) * @return A byte array that can be used to uniquely identify if results of a prior lookup can use the cached values */ - @Nullable public abstract byte[] getCacheKey(); + + // make this abstract again once @drcrallen fix the metmax lookup implementation. + public boolean isOneToOne() + { + return false; + } } diff --git a/processing/src/main/java/io/druid/query/extraction/LookupExtractorFactory.java b/processing/src/main/java/io/druid/query/extraction/LookupExtractorFactory.java new file mode 100644 index 000000000000..c7a9969efdac --- /dev/null +++ b/processing/src/main/java/io/druid/query/extraction/LookupExtractorFactory.java @@ -0,0 +1,50 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.extraction; + +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.base.Supplier; + +/** + * Users of Lookup Extraction need to implement a {@link LookupExtractorFactory} supplier of type {@link LookupExtractor}. + * Such factory will manage the state and life cycle of an given lookup. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +public interface LookupExtractorFactory extends Supplier +{ + /** + *

+ * This method will be called to start the LookupExtractor upon registered + * Calling start multiple times should not lead to any failure and suppose to return true in both cases. + *

+ * + * @return true if start successfully started the {@link LookupExtractor} + */ + public boolean start(); + + /** + *

+ * This method will be called to stop the LookupExtractor upon deletion. + * Calling this method multiple times should not lead to any failure. + *

+ * @return true if successfully closed the {@link LookupExtractor} + */ + public boolean close(); +} diff --git a/processing/src/main/java/io/druid/query/extraction/LookupReferencesManager.java b/processing/src/main/java/io/druid/query/extraction/LookupReferencesManager.java new file mode 100644 index 000000000000..14eaac9bfda3 --- /dev/null +++ b/processing/src/main/java/io/druid/query/extraction/LookupReferencesManager.java @@ -0,0 +1,183 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.extraction; + + +import com.google.common.collect.Maps; +import com.metamx.common.ISE; +import com.metamx.common.lifecycle.LifecycleStart; +import com.metamx.common.lifecycle.LifecycleStop; +import com.metamx.common.logger.Logger; +import io.druid.guice.ManageLifecycle; + +import javax.annotation.Nullable; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * This class provide a basic {@link LookupExtractorFactory} references manager. + * It allows basic operations fetching, listing, adding and deleting of {@link LookupExtractor} objects + * It is be used by queries to fetch the lookup reference. + * It is used by Lookup configuration manager to add/remove or list lookups configuration via HTTP or other protocols. + */ + +@ManageLifecycle +public class LookupReferencesManager +{ + private static final Logger LOGGER = new Logger(LookupReferencesManager.class); + private final ConcurrentMap lookupMap = new ConcurrentHashMap(); + private final Object lock = new Object(); + private final AtomicBoolean started = new AtomicBoolean(false); + + @LifecycleStart + public void start() + { + synchronized (lock) { + if (!started.getAndSet(true)) { + LOGGER.info("Started lookup factory references manager"); + } + } + } + + @LifecycleStop + public void stop() + { + synchronized (lock) { + if (started.getAndSet(false)) { + LOGGER.info("Stopped lookup factory references manager"); + for (String lookupName : lookupMap.keySet()) { + remove(lookupName); + } + } + } + } + + /** + * @param lookupName name of the lookupExtractorFactory object + * @param lookupExtractorFactory {@link LookupExtractorFactory} implementation reference. + * + * @return true if the lookup is added otherwise false. + * + * @throws IllegalStateException If the manager is closed or if start of lookup returns false. + */ + public boolean put(String lookupName, final LookupExtractorFactory lookupExtractorFactory) + { + synchronized (lock) { + assertStarted(); + if (lookupMap.containsKey(lookupName)) { + LOGGER.warn("lookup [%s] is not add, another lookup with the same name already exist", lookupName); + return false; + } + if (!lookupExtractorFactory.start()) { + throw new ISE("start method returned false for lookup [%s]", lookupName); + } + return (null == lookupMap.putIfAbsent(lookupName, lookupExtractorFactory)); + } + } + + /** + * @param lookups {@link Map} containing all the lookup as one batch. + * + * @throws IllegalStateException if the manager is closed or if {@link LookupExtractorFactory#start()} returns false + */ + public void put(Map lookups) + { + Map faildExtractorFactoryMap = new HashMap<>(); + synchronized (lock) { + assertStarted(); + for (Map.Entry entry : lookups.entrySet()) { + final String lookupName = entry.getKey(); + final LookupExtractorFactory lookupExtractorFactory = entry.getValue(); + if (lookupMap.containsKey(lookupName)) { + LOGGER.warn("lookup [%s] is not add, another lookup with the same name already exist", lookupName); + continue; + } + if (!lookupExtractorFactory.start()) { + faildExtractorFactoryMap.put(lookupName, lookupExtractorFactory); + continue; + } + lookupMap.put(lookupName, lookupExtractorFactory); + } + if (!faildExtractorFactoryMap.isEmpty()) { + throw new ISE( + "was not able to start the following lookup(s) [%s]", + faildExtractorFactoryMap.keySet().toString() + ); + } + } + } + + /** + * @param lookupName name of {@link LookupExtractorFactory} to delete from the reference registry. + * this function does call the cleaning method {@link LookupExtractorFactory#close()} + * + * @return true only if {@code lookupName} is removed and the lookup correctly stopped + */ + public boolean remove(String lookupName) + { + final LookupExtractorFactory lookupExtractorFactory = lookupMap.remove(lookupName); + if (lookupExtractorFactory != null) { + LOGGER.debug("Removing lookup [%s]", lookupName); + return lookupExtractorFactory.close(); + } + return false; + } + + /** + * @param lookupName key to fetch the reference of the object {@link LookupExtractor} + * + * @return reference of {@link LookupExtractorFactory} that correspond the {@code lookupName} or null if absent + * + * @throws IllegalStateException if the {@link LookupReferencesManager} is closed or did not start yet + */ + @Nullable + public LookupExtractorFactory get(String lookupName) + { + final LookupExtractorFactory lookupExtractorFactory = lookupMap.get(lookupName); + assertStarted(); + return lookupExtractorFactory; + } + + /** + * @return Returns {@link Map} containing a copy of the current state. + * + * @throws ISE if the is is closed or did not start yet. + */ + public Map getAll() + { + assertStarted(); + return Maps.newHashMap(lookupMap); + } + + private void assertStarted() throws ISE + { + if (isClosed()) { + throw new ISE("lookup manager is closed"); + } + } + + public boolean isClosed() + { + return !started.get(); + } +} diff --git a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java index ac2066f50cf3..fdd3b5b46139 100644 --- a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java +++ b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java @@ -43,12 +43,16 @@ public class MapLookupExtractor extends LookupExtractor { private final Map map; + private final boolean isOneToOne; + @JsonCreator public MapLookupExtractor( - @JsonProperty("map") Map map + @JsonProperty("map") Map map, + @JsonProperty("isOneToOne") boolean isOneToOne ) { this.map = Preconditions.checkNotNull(map, "map"); + this.isOneToOne = isOneToOne; } @JsonProperty @@ -77,6 +81,13 @@ public List unapply(final String value) } + @Override + @JsonProperty("isOneToOne") + public boolean isOneToOne() + { + return isOneToOne; + } + @Override public byte[] getCacheKey() { @@ -122,4 +133,5 @@ public int hashCode() { return map.hashCode(); } + } diff --git a/processing/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java b/processing/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java new file mode 100644 index 000000000000..769a5a82f24a --- /dev/null +++ b/processing/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java @@ -0,0 +1,231 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.dimension; + +import com.fasterxml.jackson.databind.InjectableValues; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.extraction.ExtractionFn; +import io.druid.query.extraction.LookupExtractor; +import io.druid.query.extraction.LookupExtractorFactory; +import io.druid.query.extraction.LookupReferencesManager; +import io.druid.query.extraction.MapLookupExtractor; +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; + +@RunWith(JUnitParamsRunner.class) +public class LookupDimensionSpecTest +{ + private static final Map STRING_MAP = ImmutableMap.of("key", "value", "key2", "value2"); + private static LookupExtractor MAP_LOOKUP_EXTRACTOR = new MapLookupExtractor( + STRING_MAP, true); + + private static final LookupReferencesManager LOOKUP_REF_MANAGER = EasyMock.createMock(LookupReferencesManager.class); + + static { + EasyMock.expect(LOOKUP_REF_MANAGER.get(EasyMock.eq("lookupName"))).andReturn(new LookupExtractorFactory() + { + @Override + public boolean start() + { + return true; + } + + @Override + public boolean close() + { + return true; + } + + @Override + public LookupExtractor get() + { + return MAP_LOOKUP_EXTRACTOR; + } + }).anyTimes(); + EasyMock.replay(LOOKUP_REF_MANAGER); + } + + private final DimensionSpec lookupDimSpec = new LookupDimensionSpec("dimName", "outputName", MAP_LOOKUP_EXTRACTOR, false, null, null, null, + true + ); + + + @Parameters + @Test + public void testSerDesr(DimensionSpec lookupDimSpec) throws IOException + { + ObjectMapper mapper = new DefaultObjectMapper(); + InjectableValues injectableValues = new InjectableValues.Std().addValue( + LookupReferencesManager.class, + LOOKUP_REF_MANAGER + ); + String serLookup = mapper.writeValueAsString(lookupDimSpec); + Assert.assertEquals(lookupDimSpec, mapper.reader(DimensionSpec.class).with(injectableValues).readValue(serLookup)); + } + + private Object[] parametersForTestSerDesr() + { + return new Object[]{ + new LookupDimensionSpec("dimName", "outputName", MAP_LOOKUP_EXTRACTOR, true, null, null, null, true), + new LookupDimensionSpec("dimName", "outputName", MAP_LOOKUP_EXTRACTOR, false, "Missing_value", null, null, true), + new LookupDimensionSpec("dimName", "outputName", MAP_LOOKUP_EXTRACTOR, false, null, null, null, true), + new LookupDimensionSpec("dimName", "outputName", null, false, null, "name", LOOKUP_REF_MANAGER, true) + }; + } + + @Test(expected = Exception.class) + public void testExceptionWhenNameAndLookupNotNull() + { + new LookupDimensionSpec("dimName", "outputName", MAP_LOOKUP_EXTRACTOR, false, "replace", "name", null, true); + } + + @Test(expected = Exception.class) + public void testExceptionWhenNameAndLookupNull() + { + new LookupDimensionSpec("dimName", "outputName", null, false, "replace", "", null, true); + } + + @Test + public void testGetDimension() + { + Assert.assertEquals("dimName", lookupDimSpec.getDimension()); + } + + @Test + public void testGetOutputName() + { + Assert.assertEquals("outputName", lookupDimSpec.getOutputName()); + } + + public Object[] parametersForTestApply() + { + return new Object[]{ + new Object[]{ + new LookupDimensionSpec("dimName", "outputName", null, true, null, "lookupName", LOOKUP_REF_MANAGER, true), + STRING_MAP + }, + new Object[]{ + new LookupDimensionSpec("dimName", "outputName", MAP_LOOKUP_EXTRACTOR, true, null, null, null, true), + STRING_MAP + }, + new Object[]{ + new LookupDimensionSpec("dimName", "outputName", MAP_LOOKUP_EXTRACTOR, false, null, null, null, true), + ImmutableMap.of("not there", "") + }, + new Object[]{ + new LookupDimensionSpec("dimName", "outputName", null, false, null, "lookupName", LOOKUP_REF_MANAGER, true), + ImmutableMap.of("not there", "") + }, + new Object[]{ + new LookupDimensionSpec("dimName", "outputName", MAP_LOOKUP_EXTRACTOR, false, "Missing_value", null, null, + true + ), + ImmutableMap.of("not there", "Missing_value") + }, + new Object[]{ + new LookupDimensionSpec("dimName", "outputName", null, false, "Missing_value", "lookupName", LOOKUP_REF_MANAGER, + true + ), + ImmutableMap.of("not there", "Missing_value") + }, + new Object[]{ + new LookupDimensionSpec("dimName", "outputName", null, true, null, "lookupName", LOOKUP_REF_MANAGER, true), + ImmutableMap.of("not there", "not there") + }, + new Object[]{ + new LookupDimensionSpec("dimName", "outputName", MAP_LOOKUP_EXTRACTOR, true, null, "", null, true), + ImmutableMap.of("not there", "not there") + } + + }; + } + + @Test + @Parameters + public void testApply(DimensionSpec dimensionSpec, Map map) + { + for (Map.Entry entry : map.entrySet() + ) { + Assert.assertEquals(Strings.emptyToNull(entry.getValue()), dimensionSpec.getExtractionFn().apply(entry.getKey())); + } + } + + public Object[] parametersForTestGetCacheKey() + { + return new Object[]{ + new Object[]{ + new LookupDimensionSpec("dimName", "outputName", MAP_LOOKUP_EXTRACTOR, true, null, null, null, true), + false + }, + new Object[]{ + new LookupDimensionSpec("dimName", "outputName", MAP_LOOKUP_EXTRACTOR, false, "Missing_value", null, null, + true + ), + false + }, + new Object[]{ + new LookupDimensionSpec("dimName", "outputName2", MAP_LOOKUP_EXTRACTOR, false, null, null, null, true), + false + }, + new Object[]{ + new LookupDimensionSpec("dimName2", "outputName2", MAP_LOOKUP_EXTRACTOR, false, null, null, null, true), + false + }, + new Object[]{ + new LookupDimensionSpec("dimName", "outputName", MAP_LOOKUP_EXTRACTOR, false, null, null, null, true), + true + }, + new Object[]{ + new LookupDimensionSpec("dimName", "outputName", null, false, null, "name", LOOKUP_REF_MANAGER, true), + false + } + }; + } + + @Test + @Parameters + public void testGetCacheKey(DimensionSpec dimensionSpec, boolean expectedResult) + { + Assert.assertEquals(expectedResult, Arrays.equals(lookupDimSpec.getCacheKey(), dimensionSpec.getCacheKey())); + } + + @Test + public void testPreservesOrdering() + { + Assert.assertFalse(lookupDimSpec.preservesOrdering()); + } + + @Test + public void testIsOneToOne() + { + Assert.assertEquals(lookupDimSpec.getExtractionFn().getExtractionType(), ExtractionFn.ExtractionType.ONE_TO_ONE); + } +} diff --git a/processing/src/test/java/io/druid/query/extraction/LookupExtractorTest.java b/processing/src/test/java/io/druid/query/extraction/LookupExtractorTest.java index 7b08082184a5..7fb35119fa68 100644 --- a/processing/src/test/java/io/druid/query/extraction/LookupExtractorTest.java +++ b/processing/src/test/java/io/druid/query/extraction/LookupExtractorTest.java @@ -21,11 +21,14 @@ package io.druid.query.extraction; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.druid.jackson.DefaultObjectMapper; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -54,7 +57,14 @@ public class LookupExtractorTest "emptyString", Arrays.asList("") ); - LookupExtractor lookupExtractor = new MapLookupExtractor(EXPECTED_MAP); + LookupExtractor lookupExtractor = new MapLookupExtractor(EXPECTED_MAP, false); + + @Test + public void testSerDes() throws IOException + { + ObjectMapper mapper = new DefaultObjectMapper(); + Assert.assertEquals(lookupExtractor, mapper.reader(LookupExtractor.class).readValue(mapper.writeValueAsBytes(lookupExtractor))); + } @Test public void testApplyAll() diff --git a/processing/src/test/java/io/druid/query/extraction/LookupReferencesManagerTest.java b/processing/src/test/java/io/druid/query/extraction/LookupReferencesManagerTest.java new file mode 100644 index 000000000000..fc7056ad09f0 --- /dev/null +++ b/processing/src/test/java/io/druid/query/extraction/LookupReferencesManagerTest.java @@ -0,0 +1,172 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.extraction; + +import com.google.common.collect.ImmutableMap; +import com.metamx.common.ISE; +import org.easymock.EasyMock; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +public class LookupReferencesManagerTest +{ + LookupReferencesManager lookupReferencesManager; + + @Before + public void setUp() + { + lookupReferencesManager = new LookupReferencesManager(); + Assert.assertTrue("must be closed before start call", lookupReferencesManager.isClosed()); + lookupReferencesManager.start(); + Assert.assertFalse("must start after start call", lookupReferencesManager.isClosed()); + } + + @After + public void tearDown() + { + lookupReferencesManager.stop(); + Assert.assertTrue("stop call should close it", lookupReferencesManager.isClosed()); + } + + @Test(expected = ISE.class) + public void testGetExceptionWhenClosed() + { + lookupReferencesManager.stop(); + lookupReferencesManager.get("test"); + } + + @Test(expected = ISE.class) + public void testAddExceptionWhenClosed() + { + lookupReferencesManager.stop(); + lookupReferencesManager.put("test", EasyMock.createMock(LookupExtractorFactory.class)); + } + + @Test + public void testPutGetRemove() + { + LookupExtractorFactory lookupExtractorFactory = EasyMock.createMock(LookupExtractorFactory.class); + EasyMock.expect(lookupExtractorFactory.start()).andReturn(true).once(); + EasyMock.expect(lookupExtractorFactory.close()).andReturn(true).once(); + EasyMock.replay(lookupExtractorFactory); + Assert.assertNull(lookupReferencesManager.get("test")); + lookupReferencesManager.put("test", lookupExtractorFactory); + Assert.assertEquals(lookupExtractorFactory, lookupReferencesManager.get("test")); + Assert.assertTrue(lookupReferencesManager.remove("test")); + Assert.assertNull(lookupReferencesManager.get("test")); + } + + @Test + public void testCloseIsCalledAfterStopping() throws IOException + { + LookupExtractorFactory lookupExtractorFactory = EasyMock.createStrictMock(LookupExtractorFactory.class); + EasyMock.expect(lookupExtractorFactory.start()).andReturn(true).once(); + EasyMock.expect(lookupExtractorFactory.close()).andReturn(true).once(); + EasyMock.replay(lookupExtractorFactory); + lookupReferencesManager.put("testMock", lookupExtractorFactory); + lookupReferencesManager.stop(); + EasyMock.verify(lookupExtractorFactory); + } + + @Test + public void testCloseIsCalledAfterRemove() throws IOException + { + LookupExtractorFactory lookupExtractorFactory = EasyMock.createStrictMock(LookupExtractorFactory.class); + EasyMock.expect(lookupExtractorFactory.start()).andReturn(true).once(); + EasyMock.expect(lookupExtractorFactory.close()).andReturn(true).once(); + EasyMock.replay(lookupExtractorFactory); + lookupReferencesManager.put("testMock", lookupExtractorFactory); + lookupReferencesManager.remove("testMock"); + EasyMock.verify(lookupExtractorFactory); + } + + @Test + public void testRemoveInExisting() + { + Assert.assertFalse(lookupReferencesManager.remove("notThere")); + } + + @Test + public void testGetNotThere() + { + Assert.assertNull(lookupReferencesManager.get("notThere")); + } + + @Test + public void testAddingWithSameLookupName() + { + LookupExtractorFactory lookupExtractorFactory = EasyMock.createNiceMock(LookupExtractorFactory.class); + EasyMock.expect(lookupExtractorFactory.start()).andReturn(true).once(); + LookupExtractorFactory lookupExtractorFactory2 = EasyMock.createNiceMock(LookupExtractorFactory.class); + EasyMock.expect(lookupExtractorFactory2.start()).andReturn(true).times(2); + EasyMock.replay(lookupExtractorFactory, lookupExtractorFactory2); + Assert.assertTrue(lookupReferencesManager.put("testName", lookupExtractorFactory)); + Assert.assertFalse(lookupReferencesManager.put("testName", lookupExtractorFactory2)); + ImmutableMap extractorImmutableMap = ImmutableMap.of( + "testName", + lookupExtractorFactory2 + ); + lookupReferencesManager.put(extractorImmutableMap); + Assert.assertEquals(lookupExtractorFactory, lookupReferencesManager.get("testName")); + } + + @Test + public void testAddLookupsThenGetAll() + { + LookupExtractorFactory lookupExtractorFactory = EasyMock.createNiceMock(LookupExtractorFactory.class); + EasyMock.expect(lookupExtractorFactory.start()).andReturn(true).once(); + LookupExtractorFactory lookupExtractorFactory2 = EasyMock.createNiceMock(LookupExtractorFactory.class); + EasyMock.expect(lookupExtractorFactory2.start()).andReturn(true).once(); + EasyMock.replay(lookupExtractorFactory, lookupExtractorFactory2); + ImmutableMap extractorImmutableMap = ImmutableMap.of( + "name1", + lookupExtractorFactory, + "name2", + lookupExtractorFactory2 + ); + lookupReferencesManager.put(extractorImmutableMap); + Assert.assertEquals(extractorImmutableMap, lookupReferencesManager.getAll()); + } + + @Test(expected = ISE.class) + public void testExceptionWhenStartFail() + { + LookupExtractorFactory lookupExtractorFactory = EasyMock.createStrictMock(LookupExtractorFactory.class); + EasyMock.expect(lookupExtractorFactory.start()).andReturn(false).once(); + EasyMock.replay(lookupExtractorFactory); + lookupReferencesManager.put("testMock", lookupExtractorFactory); + } + + @Test(expected = ISE.class) + public void testputAllExceptionWhenStartFail() + { + LookupExtractorFactory lookupExtractorFactory = EasyMock.createStrictMock(LookupExtractorFactory.class); + EasyMock.expect(lookupExtractorFactory.start()).andReturn(false).once(); + ImmutableMap extractorImmutableMap = ImmutableMap.of( + "name1", + lookupExtractorFactory + ); + lookupReferencesManager.put(extractorImmutableMap); + } +} diff --git a/processing/src/test/java/io/druid/query/extraction/MapLookupExtractorTest.java b/processing/src/test/java/io/druid/query/extraction/MapLookupExtractorTest.java index b54a7a849676..507bdcc08652 100644 --- a/processing/src/test/java/io/druid/query/extraction/MapLookupExtractorTest.java +++ b/processing/src/test/java/io/druid/query/extraction/MapLookupExtractorTest.java @@ -32,7 +32,7 @@ public class MapLookupExtractorTest { private final Map lookupMap = ImmutableMap.of("foo", "bar", "null", "", "empty String", "", "","empty_string"); - private final MapLookupExtractor fn = new MapLookupExtractor(lookupMap); + private final MapLookupExtractor fn = new MapLookupExtractor(lookupMap, false); @Test public void testUnApply() @@ -62,33 +62,33 @@ public void testApply() throws Exception @Test public void testGetCacheKey() throws Exception { - final MapLookupExtractor fn2 = new MapLookupExtractor(ImmutableMap.copyOf(lookupMap)); + final MapLookupExtractor fn2 = new MapLookupExtractor(ImmutableMap.copyOf(lookupMap), false); Assert.assertArrayEquals(fn.getCacheKey(), fn2.getCacheKey()); - final MapLookupExtractor fn3 = new MapLookupExtractor(ImmutableMap.of("foo2", "bar")); + final MapLookupExtractor fn3 = new MapLookupExtractor(ImmutableMap.of("foo2", "bar"), false); Assert.assertFalse(Arrays.equals(fn.getCacheKey(), fn3.getCacheKey())); - final MapLookupExtractor fn4 = new MapLookupExtractor(ImmutableMap.of("foo", "bar2")); + final MapLookupExtractor fn4 = new MapLookupExtractor(ImmutableMap.of("foo", "bar2"), false); Assert.assertFalse(Arrays.equals(fn.getCacheKey(), fn4.getCacheKey())); } @Test public void testEquals() throws Exception { - final MapLookupExtractor fn2 = new MapLookupExtractor(ImmutableMap.copyOf(lookupMap)); + final MapLookupExtractor fn2 = new MapLookupExtractor(ImmutableMap.copyOf(lookupMap), false); Assert.assertEquals(fn, fn2); - final MapLookupExtractor fn3 = new MapLookupExtractor(ImmutableMap.of("foo2", "bar")); + final MapLookupExtractor fn3 = new MapLookupExtractor(ImmutableMap.of("foo2", "bar"), false); Assert.assertNotEquals(fn, fn3); - final MapLookupExtractor fn4 = new MapLookupExtractor(ImmutableMap.of("foo", "bar2")); + final MapLookupExtractor fn4 = new MapLookupExtractor(ImmutableMap.of("foo", "bar2"), false); Assert.assertNotEquals(fn, fn4); } @Test public void testHashCode() throws Exception { - final MapLookupExtractor fn2 = new MapLookupExtractor(ImmutableMap.copyOf(lookupMap)); + final MapLookupExtractor fn2 = new MapLookupExtractor(ImmutableMap.copyOf(lookupMap), false); Assert.assertEquals(fn.hashCode(), fn2.hashCode()); - final MapLookupExtractor fn3 = new MapLookupExtractor(ImmutableMap.of("foo2", "bar")); + final MapLookupExtractor fn3 = new MapLookupExtractor(ImmutableMap.of("foo2", "bar"), false); Assert.assertNotEquals(fn.hashCode(), fn3.hashCode()); - final MapLookupExtractor fn4 = new MapLookupExtractor(ImmutableMap.of("foo", "bar2")); + final MapLookupExtractor fn4 = new MapLookupExtractor(ImmutableMap.of("foo", "bar2"), false); Assert.assertNotEquals(fn.hashCode(), fn4.hashCode()); } } diff --git a/processing/src/test/java/io/druid/query/extraction/extraction/LookupExtractionFnExpectationsTest.java b/processing/src/test/java/io/druid/query/extraction/extraction/LookupExtractionFnExpectationsTest.java index 80113b6cac03..7675445a0cb1 100644 --- a/processing/src/test/java/io/druid/query/extraction/extraction/LookupExtractionFnExpectationsTest.java +++ b/processing/src/test/java/io/druid/query/extraction/extraction/LookupExtractionFnExpectationsTest.java @@ -34,7 +34,7 @@ public class LookupExtractionFnExpectationsTest public void testMissingKeyIsNull() { final LookupExtractionFn lookupExtractionFn = new LookupExtractionFn( - new MapLookupExtractor(ImmutableMap.of("foo", "bar")), + new MapLookupExtractor(ImmutableMap.of("foo", "bar"), false), true, null, false, @@ -47,7 +47,7 @@ public void testMissingKeyIsNull() public void testMissingKeyIsReplaced() { final LookupExtractionFn lookupExtractionFn = new LookupExtractionFn( - new MapLookupExtractor(ImmutableMap.of("foo", "bar")), + new MapLookupExtractor(ImmutableMap.of("foo", "bar"), false), false, "REPLACE", false, @@ -60,7 +60,7 @@ public void testMissingKeyIsReplaced() public void testNullKeyIsMappable() { final LookupExtractionFn lookupExtractionFn = new LookupExtractionFn( - new MapLookupExtractor(ImmutableMap.of("", "bar")), + new MapLookupExtractor(ImmutableMap.of("", "bar"), false), false, "REPLACE", false, @@ -73,7 +73,7 @@ public void testNullKeyIsMappable() public void testNullValue() { final LookupExtractionFn lookupExtractionFn = new LookupExtractionFn( - new MapLookupExtractor(ImmutableMap.of("foo", "")), + new MapLookupExtractor(ImmutableMap.of("foo", ""), false), false, "REPLACE", false, diff --git a/processing/src/test/java/io/druid/query/extraction/extraction/LookupExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/extraction/LookupExtractionFnTest.java index 190a4d722f0e..46f761edde4a 100644 --- a/processing/src/test/java/io/druid/query/extraction/extraction/LookupExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/extraction/LookupExtractionFnTest.java @@ -87,14 +87,14 @@ public void testEqualsAndHash(){ return; } final LookupExtractionFn lookupExtractionFn1 = new LookupExtractionFn( - new MapLookupExtractor(ImmutableMap.of("foo", "bar")), + new MapLookupExtractor(ImmutableMap.of("foo", "bar"), false), retainMissing, replaceMissing, injective, false ); final LookupExtractionFn lookupExtractionFn2 = new LookupExtractionFn( - new MapLookupExtractor(ImmutableMap.of("foo", "bar")), + new MapLookupExtractor(ImmutableMap.of("foo", "bar"), false), retainMissing, replaceMissing, injective, @@ -103,7 +103,7 @@ public void testEqualsAndHash(){ final LookupExtractionFn lookupExtractionFn3 = new LookupExtractionFn( - new MapLookupExtractor(ImmutableMap.of("foo", "bar2")), + new MapLookupExtractor(ImmutableMap.of("foo", "bar2"), false), retainMissing, replaceMissing, injective, @@ -124,7 +124,7 @@ public void testSimpleSerDe() throws IOException return; } final LookupExtractionFn lookupExtractionFn = new LookupExtractionFn( - new MapLookupExtractor(ImmutableMap.of("foo", "bar")), + new MapLookupExtractor(ImmutableMap.of("foo", "bar"), false), retainMissing, replaceMissing, injective, @@ -151,7 +151,7 @@ public void testIllegalArgs() { if (retainMissing && !Strings.isNullOrEmpty(replaceMissing)) { final LookupExtractionFn lookupExtractionFn = new LookupExtractionFn( - new MapLookupExtractor(ImmutableMap.of("foo", "bar")), + new MapLookupExtractor(ImmutableMap.of("foo", "bar"), false), retainMissing, Strings.emptyToNull(replaceMissing), injective, @@ -173,7 +173,7 @@ public void testCacheKey() weirdMap.put("foobar", null); final LookupExtractionFn lookupExtractionFn = new LookupExtractionFn( - new MapLookupExtractor(ImmutableMap.of("foo", "bar")), + new MapLookupExtractor(ImmutableMap.of("foo", "bar"), false), retainMissing, replaceMissing, injective, @@ -210,7 +210,7 @@ public void testCacheKey() Arrays.equals( lookupExtractionFn.getCacheKey(), new LookupExtractionFn( - new MapLookupExtractor(weirdMap), + new MapLookupExtractor(weirdMap, false), lookupExtractionFn.isRetainMissingValue(), lookupExtractionFn.getReplaceMissingValueWith(), lookupExtractionFn.isInjective(), diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index 431ab836e240..ff7e230a9880 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -268,7 +268,7 @@ public void testGroupByWithRebucketRename() .setDimensions( Lists.newArrayList( new ExtractionDimensionSpec( - "quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map), false, null, false, false), null + "quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map, false), false, null, false, false), null ) ) ) @@ -344,7 +344,7 @@ public void testGroupByWithSimpleRenameRetainMissingNonInjective() .setDimensions( Lists.newArrayList( new ExtractionDimensionSpec( - "quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map), true, null, false, false), null + "quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map, false), true, null, false, false), null ) ) ) @@ -420,7 +420,7 @@ public void testGroupByWithSimpleRenameRetainMissing() .setDimensions( Lists.newArrayList( new ExtractionDimensionSpec( - "quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map), true, null, true, false), null + "quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map, false), true, null, true, false), null ) ) ) @@ -498,7 +498,7 @@ public void testGroupByWithSimpleRenameAndMissingString() new ExtractionDimensionSpec( "quality", "alias", - new LookupExtractionFn(new MapLookupExtractor(map), false, "MISSING", true, false), + new LookupExtractionFn(new MapLookupExtractor(map, false), false, "MISSING", true, false), null ) ) @@ -574,7 +574,7 @@ public void testGroupByWithSimpleRename() .setDimensions( Lists.newArrayList( new ExtractionDimensionSpec( - "quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map), false, null, true, false), null + "quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map, false), false, null, true, false), null ) ) ) @@ -3922,7 +3922,8 @@ public void testBySegmentResultsUnOptimizedDimextraction() ImmutableMap.of( "mezzanine", "mezzanine0" - ) + ), + false ), false, null, false, false ), @@ -3996,7 +3997,8 @@ public void testBySegmentResultsOptimizedDimextraction() ImmutableMap.of( "mezzanine", "mezzanine0" - ) + ), + false ), false, null, true, false ), @@ -4043,7 +4045,7 @@ public void testGroupByWithExtractionDimFilter() extractionMap.put("mezzanine", "automotiveAndBusinessAndNewsAndMezzanine"); extractionMap.put("news", "automotiveAndBusinessAndNewsAndMezzanine"); - MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap); + MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); List dimFilters = Lists.newArrayList( @@ -4115,7 +4117,7 @@ public void testGroupByWithExtractionDimFilterCaseMappingValueIsNullOrEmpty() extractionMap.put("technology", "technology0"); extractionMap.put("travel", "travel0"); - MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap); + MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) @@ -4151,7 +4153,7 @@ public void testGroupByWithExtractionDimFilterCaseMappingValueIsNullOrEmpty() public void testGroupByWithExtractionDimFilterWhenSearchValueNotInTheMap() { Map extractionMap = new HashMap<>(); - MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap); + MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) @@ -4192,7 +4194,7 @@ public void testGroupByWithExtractionDimFilterKeyisNull() Map extractionMap = new HashMap<>(); extractionMap.put("", "NULLorEMPTY"); - MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap); + MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) @@ -4244,7 +4246,7 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() extractionMap.put("technology", "technology0"); extractionMap.put("travel", "travel0"); - MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap); + MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, "missing", true, false); DimFilter filter = new ExtractionDimFilter("quality","mezzanineANDnews",lookupExtractionFn,null); GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) @@ -4304,7 +4306,7 @@ public void testGroupByWithExtractionDimFilterOptimazitionManyToOne() extractionMap.put("mezzanine", "newsANDmezzanine"); extractionMap.put("news", "newsANDmezzanine"); - MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap); + MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) @@ -4330,7 +4332,7 @@ public void testGroupByWithExtractionDimFilterOptimazitionManyToOne() Map extractionMap = new HashMap<>(); extractionMap.put("", "EMPTY"); - MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap); + MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java index 530aaff87c6c..23da15615770 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java @@ -254,7 +254,7 @@ public void testSearchWithExtractionFilter1() final LookupExtractionFn lookupExtractionFn = new LookupExtractionFn( - new MapLookupExtractor(ImmutableMap.of("automotive", automotiveSnowman)), + new MapLookupExtractor(ImmutableMap.of("automotive", automotiveSnowman), false), true, null, true, diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java index a77d4e41d365..b1b9d68cc643 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java @@ -1629,7 +1629,8 @@ public void testTopNDimExtractionFastTopNOptimalWithReplaceMissing() "spot", "2spot0", "total_market", "1total_market0", "upfront", "3upfront0" - ) + ), + false ), false, "MISSING", true, false ), @@ -1693,7 +1694,8 @@ public void testTopNDimExtractionFastTopNUnOptimalWithReplaceMissing() "spot", "2spot0", "total_market", "1total_market0", "upfront", "3upfront0" - ) + ), + false ), false, "MISSING", false, false ), @@ -1758,7 +1760,8 @@ public void testTopNDimExtractionFastTopNOptimal() "spot", "2spot0", "total_market", "1total_market0", "upfront", "3upfront0" - ) + ), + false ), true, null, true, false ), @@ -1825,7 +1828,8 @@ public void testTopNDimExtractionFastTopNUnOptimal() "total_market0", "upfront", "upfront0" - ) + ), + false ), true, null, false, false ), @@ -1891,7 +1895,8 @@ public void testTopNLexicographicDimExtractionOptimalNamespace() "3total_market", "upfront", "1upfront" - ) + ), + false ), true, null, true, false ), @@ -1957,7 +1962,8 @@ public void testTopNLexicographicDimExtractionUnOptimalNamespace() "3total_market", "upfront", "1upfront" - ) + ), + false ), true, null, false, false ), @@ -2024,7 +2030,8 @@ public void testTopNLexicographicDimExtractionOptimalNamespaceWithRunner() "3total_market", "upfront", "1upfront" - ) + ), + false ), true, null, true, false ), @@ -3165,7 +3172,7 @@ public void testTopNWithExtractionFilter() { Map extractionMap = new HashMap<>(); extractionMap.put("spot", "spot0"); - MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap); + MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); TopNQuery query = new TopNQueryBuilder().dataSource(QueryRunnerTestHelper.dataSource) @@ -3215,7 +3222,7 @@ public void testTopNWithExtractionFilterAndFilteredAggregatorCaseNoExistingValue Map extractionMap = new HashMap<>(); extractionMap.put("", "NULL"); - MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap); + MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); DimFilter extractionFilter = new ExtractionDimFilter("null_column", "NULL", lookupExtractionFn, null); TopNQueryBuilder topNQueryBuilder = new TopNQueryBuilder() @@ -3284,7 +3291,7 @@ public void testTopNWithExtractionFilterNoExistingValue() Map extractionMap = new HashMap<>(); extractionMap.put("","NULL"); - MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap); + MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); DimFilter extractionFilter = new ExtractionDimFilter("null_column", "NULL", lookupExtractionFn, null); TopNQueryBuilder topNQueryBuilder = new TopNQueryBuilder() diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryTest.java index 5c9a43f6e6da..0d82238dc67d 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryTest.java @@ -93,7 +93,7 @@ public void testQuerySerdeWithLookupExtractionFn() throws IOException new ExtractionDimensionSpec( marketDimension, marketDimension, - new LookupExtractionFn(new MapLookupExtractor(ImmutableMap.of("foo", "bar")), true, null, false, false), + new LookupExtractionFn(new MapLookupExtractor(ImmutableMap.of("foo", "bar"), false), true, null, false, false), null ) ) diff --git a/services/src/main/java/io/druid/cli/CliBroker.java b/services/src/main/java/io/druid/cli/CliBroker.java index f5748e67b67d..8af4bd40df20 100644 --- a/services/src/main/java/io/druid/cli/CliBroker.java +++ b/services/src/main/java/io/druid/cli/CliBroker.java @@ -38,10 +38,12 @@ import io.druid.guice.JsonConfigProvider; import io.druid.guice.LazySingleton; import io.druid.guice.LifecycleModule; +import io.druid.guice.ManageLifecycle; import io.druid.query.MapQueryToolChestWarehouse; import io.druid.query.QuerySegmentWalker; import io.druid.query.QueryToolChestWarehouse; import io.druid.query.RetryQueryRunnerConfig; +import io.druid.query.extraction.LookupReferencesManager; import io.druid.server.ClientInfoResource; import io.druid.server.ClientQuerySegmentWalker; import io.druid.server.QueryResource; @@ -105,6 +107,7 @@ public void configure(Binder binder) Jerseys.addResource(binder, ClientInfoResource.class); LifecycleModule.register(binder, QueryResource.class); LifecycleModule.register(binder, DruidBroker.class); + LifecycleModule.register(binder, LookupReferencesManager.class); MetricsModule.register(binder, CacheMonitor.class); diff --git a/services/src/main/java/io/druid/cli/CliHistorical.java b/services/src/main/java/io/druid/cli/CliHistorical.java index 56c2025ebecb..42ea5b9d8cd6 100644 --- a/services/src/main/java/io/druid/cli/CliHistorical.java +++ b/services/src/main/java/io/druid/cli/CliHistorical.java @@ -35,6 +35,7 @@ import io.druid.guice.ManageLifecycle; import io.druid.guice.NodeTypeConfig; import io.druid.query.QuerySegmentWalker; +import io.druid.query.extraction.LookupReferencesManager; import io.druid.server.QueryResource; import io.druid.server.coordination.ServerManager; import io.druid.server.coordination.ZkCoordinator; @@ -83,7 +84,7 @@ public void configure(Binder binder) Jerseys.addResource(binder, QueryResource.class); Jerseys.addResource(binder, HistoricalResource.class); LifecycleModule.register(binder, QueryResource.class); - + LifecycleModule.register(binder, LookupReferencesManager.class); LifecycleModule.register(binder, ZkCoordinator.class); JsonConfigProvider.bind(binder, "druid.historical.cache", CacheConfig.class); diff --git a/services/src/main/java/io/druid/cli/CliPeon.java b/services/src/main/java/io/druid/cli/CliPeon.java index ebe675409102..c75ad6d1e0c3 100644 --- a/services/src/main/java/io/druid/cli/CliPeon.java +++ b/services/src/main/java/io/druid/cli/CliPeon.java @@ -62,6 +62,7 @@ import io.druid.indexing.worker.executor.ExecutorLifecycleConfig; import io.druid.metadata.IndexerSQLMetadataStorageCoordinator; import io.druid.query.QuerySegmentWalker; +import io.druid.query.extraction.LookupReferencesManager; import io.druid.segment.loading.DataSegmentArchiver; import io.druid.segment.loading.DataSegmentKiller; import io.druid.segment.loading.DataSegmentMover; @@ -190,7 +191,7 @@ public void configure(Binder binder) Jerseys.addResource(binder, QueryResource.class); Jerseys.addResource(binder, ChatHandlerResource.class); LifecycleModule.register(binder, QueryResource.class); - + LifecycleModule.register(binder, LookupReferencesManager.class); binder.bind(NodeTypeConfig.class).toInstance(new NodeTypeConfig(nodeType)); LifecycleModule.register(binder, Server.class); diff --git a/services/src/main/java/io/druid/cli/CliRouter.java b/services/src/main/java/io/druid/cli/CliRouter.java index 982786579924..b65fa9a2328b 100644 --- a/services/src/main/java/io/druid/cli/CliRouter.java +++ b/services/src/main/java/io/druid/cli/CliRouter.java @@ -36,6 +36,7 @@ import io.druid.guice.ManageLifecycle; import io.druid.guice.annotations.Self; import io.druid.guice.http.JettyHttpClientModule; +import io.druid.query.extraction.LookupReferencesManager; import io.druid.server.initialization.jetty.JettyServerInitializer; import io.druid.server.router.CoordinatorRuleManager; import io.druid.server.router.QueryHostFinder; @@ -91,6 +92,7 @@ public void configure(Binder binder) LifecycleModule.register(binder, Server.class); DiscoveryModule.register(binder, Self.class); + LifecycleModule.register(binder, LookupReferencesManager.class); } @Provides diff --git a/services/src/main/java/io/druid/guice/RealtimeModule.java b/services/src/main/java/io/druid/guice/RealtimeModule.java index 1000b94f998e..2c716cf5a636 100644 --- a/services/src/main/java/io/druid/guice/RealtimeModule.java +++ b/services/src/main/java/io/druid/guice/RealtimeModule.java @@ -29,6 +29,7 @@ import io.druid.client.coordinator.CoordinatorClient; import io.druid.metadata.MetadataSegmentPublisher; import io.druid.query.QuerySegmentWalker; +import io.druid.query.extraction.LookupReferencesManager; import io.druid.segment.realtime.FireDepartment; import io.druid.segment.realtime.NoopSegmentPublisher; import io.druid.segment.realtime.RealtimeManager; @@ -106,6 +107,7 @@ public void configure(Binder binder) Jerseys.addResource(binder, QueryResource.class); Jerseys.addResource(binder, ChatHandlerResource.class); LifecycleModule.register(binder, QueryResource.class); + LifecycleModule.register(binder, LookupReferencesManager.class); LifecycleModule.register(binder, Server.class); } }