From 2c79d535890028d6340e8dfef5f59995ba6927ff Mon Sep 17 00:00:00 2001 From: Christian Senet Date: Wed, 13 Mar 2019 15:47:17 +0100 Subject: [PATCH 1/2] #5884 Avro Maps can't be flattened in Kafka Indexing on master --- .../data/input/avro/GenericAvroJsonProvider.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/extensions-core/avro-extensions/src/main/java/org/apache/druid/data/input/avro/GenericAvroJsonProvider.java b/extensions-core/avro-extensions/src/main/java/org/apache/druid/data/input/avro/GenericAvroJsonProvider.java index 4beb7578dcd7..01925fe3c402 100644 --- a/extensions-core/avro-extensions/src/main/java/org/apache/druid/data/input/avro/GenericAvroJsonProvider.java +++ b/extensions-core/avro-extensions/src/main/java/org/apache/druid/data/input/avro/GenericAvroJsonProvider.java @@ -23,6 +23,7 @@ import com.jayway.jsonpath.spi.json.JsonProvider; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; +import org.apache.avro.util.Utf8; import java.io.InputStream; import java.util.ArrayList; @@ -137,7 +138,15 @@ public void setArrayIndex(final Object o, final int i, final Object o1) throw new UnsupportedOperationException(); } } - + + private Object transformAvroMapKey(String keyToTransform, Map avroMapObj) { + Object key = avroMapObj.keySet().iterator().next(); + if (key instanceof Utf8) { + return new Utf8(keyToTransform); + } + return keyToTransform; + } + @Override public Object getMapValue(final Object o, final String s) { @@ -146,7 +155,7 @@ public Object getMapValue(final Object o, final String s) } else if (o instanceof GenericRecord) { return ((GenericRecord) o).get(s); } else if (o instanceof Map) { - return ((Map) o).get(s); + return ((Map) o).get(transformAvroMapKey(s,(Map) o)); } else { throw new UnsupportedOperationException(o.getClass().getName()); } From 643f33a2d2be8cedefd3d75964e97e1e29f6c13f Mon Sep 17 00:00:00 2001 From: Christian Senet Date: Thu, 14 Mar 2019 14:21:04 +0100 Subject: [PATCH 2/2] correct typo and reduce code size --- .../input/avro/GenericAvroJsonProvider.java | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/extensions-core/avro-extensions/src/main/java/org/apache/druid/data/input/avro/GenericAvroJsonProvider.java b/extensions-core/avro-extensions/src/main/java/org/apache/druid/data/input/avro/GenericAvroJsonProvider.java index 01925fe3c402..bfb6e2224a98 100644 --- a/extensions-core/avro-extensions/src/main/java/org/apache/druid/data/input/avro/GenericAvroJsonProvider.java +++ b/extensions-core/avro-extensions/src/main/java/org/apache/druid/data/input/avro/GenericAvroJsonProvider.java @@ -138,15 +138,7 @@ public void setArrayIndex(final Object o, final int i, final Object o1) throw new UnsupportedOperationException(); } } - - private Object transformAvroMapKey(String keyToTransform, Map avroMapObj) { - Object key = avroMapObj.keySet().iterator().next(); - if (key instanceof Utf8) { - return new Utf8(keyToTransform); - } - return keyToTransform; - } - + @Override public Object getMapValue(final Object o, final String s) { @@ -155,7 +147,13 @@ public Object getMapValue(final Object o, final String s) } else if (o instanceof GenericRecord) { return ((GenericRecord) o).get(s); } else if (o instanceof Map) { - return ((Map) o).get(transformAvroMapKey(s,(Map) o)); + final Map theMap = (Map) o; + if (theMap.containsKey(s)) { + return theMap.get(s); + } else { + final Utf8 utf8Key = new Utf8(s); + return theMap.get(utf8Key); + } } else { throw new UnsupportedOperationException(o.getClass().getName()); }