From 205480722215aeab23aefe0ff8e3c1d3ea54d48a Mon Sep 17 00:00:00 2001 From: Kenji Noguchi Date: Thu, 13 Apr 2017 19:12:15 -0700 Subject: [PATCH 01/14] add jq expression in the flattenSpec --- api/pom.xml | 6 +- .../druid/data/input/impl/JSONParseSpec.java | 3 + .../data/input/impl/JSONPathFieldSpec.java | 5 ++ .../data/input/impl/JSONPathFieldType.java | 3 +- .../data/input/impl/JSONPathSpecTest.java | 18 ++++ java-util/pom.xml | 5 ++ .../java/util/common/parsers/FlattenExpr.java | 60 +++++++++++++ .../util/common/parsers/JSONPathParser.java | 88 ++++++++++++++++--- .../common/parsers/JSONPathParserTest.java | 10 +++ 9 files changed, 184 insertions(+), 14 deletions(-) create mode 100644 java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java diff --git a/api/pom.xml b/api/pom.xml index d494af174e4c..bda9be4658b2 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -87,7 +87,11 @@ com.google.code.findbugs jsr305 - + + net.thisptr + jackson-jq + 0.0.7 + junit diff --git a/api/src/main/java/io/druid/data/input/impl/JSONParseSpec.java b/api/src/main/java/io/druid/data/input/impl/JSONParseSpec.java index 81ce73b94a44..53fe138c59f4 100644 --- a/api/src/main/java/io/druid/data/input/impl/JSONParseSpec.java +++ b/api/src/main/java/io/druid/data/input/impl/JSONParseSpec.java @@ -115,6 +115,9 @@ private List convertFieldSpecs(List case PATH: type = JSONPathParser.FieldType.PATH; break; + case JQ: + type = JSONPathParser.FieldType.JQ; + break; default: throw new IllegalArgumentException("Invalid type for field " + druidSpec.getName()); } diff --git a/api/src/main/java/io/druid/data/input/impl/JSONPathFieldSpec.java b/api/src/main/java/io/druid/data/input/impl/JSONPathFieldSpec.java index 2825d92652f4..af436c941c83 100644 --- a/api/src/main/java/io/druid/data/input/impl/JSONPathFieldSpec.java +++ b/api/src/main/java/io/druid/data/input/impl/JSONPathFieldSpec.java @@ -69,6 +69,11 @@ public static JSONPathFieldSpec createNestedField(String name, String expr) return new JSONPathFieldSpec(JSONPathFieldType.PATH, name, expr); } + public static JSONPathFieldSpec createJqField(String name, String expr) + { + return new JSONPathFieldSpec(JSONPathFieldType.JQ, name, expr); + } + public static JSONPathFieldSpec createRootField(String name) { return new JSONPathFieldSpec(JSONPathFieldType.ROOT, name, null); diff --git a/api/src/main/java/io/druid/data/input/impl/JSONPathFieldType.java b/api/src/main/java/io/druid/data/input/impl/JSONPathFieldType.java index d99ad77c44e6..a69165837fcd 100644 --- a/api/src/main/java/io/druid/data/input/impl/JSONPathFieldType.java +++ b/api/src/main/java/io/druid/data/input/impl/JSONPathFieldType.java @@ -25,7 +25,8 @@ public enum JSONPathFieldType { ROOT, - PATH; + PATH, + JQ; @JsonValue @Override diff --git a/api/src/test/java/io/druid/data/input/impl/JSONPathSpecTest.java b/api/src/test/java/io/druid/data/input/impl/JSONPathSpecTest.java index 5f405409d86c..ec7dd41da08b 100644 --- a/api/src/test/java/io/druid/data/input/impl/JSONPathSpecTest.java +++ b/api/src/test/java/io/druid/data/input/impl/JSONPathSpecTest.java @@ -41,6 +41,9 @@ public void testSerde() throws IOException fields.add(JSONPathFieldSpec.createNestedField("hey0barx", "$.hey[0].barx")); fields.add(JSONPathFieldSpec.createRootField("timestamp")); fields.add(JSONPathFieldSpec.createRootField("foo.bar1")); + fields.add(JSONPathFieldSpec.createJqField("foobar1", ".foo.bar1")); + fields.add(JSONPathFieldSpec.createJqField("baz0", ".baz[0]")); + fields.add(JSONPathFieldSpec.createJqField("hey0barx", ".hey[0].barx")); JSONPathSpec flattenSpec = new JSONPathSpec(true, fields); @@ -55,6 +58,9 @@ public void testSerde() throws IOException JSONPathFieldSpec hey0barx = serdeFields.get(2); JSONPathFieldSpec timestamp = serdeFields.get(3); JSONPathFieldSpec foodotbar1 = serdeFields.get(4); + JSONPathFieldSpec jqFoobar1 = serdeFields.get(5); + JSONPathFieldSpec jqBaz0 = serdeFields.get(6); + JSONPathFieldSpec jqHey0barx = serdeFields.get(7); Assert.assertEquals(JSONPathFieldType.PATH, foobar1.getType()); Assert.assertEquals("foobar1", foobar1.getName()); @@ -68,6 +74,18 @@ public void testSerde() throws IOException Assert.assertEquals("hey0barx", hey0barx.getName()); Assert.assertEquals("$.hey[0].barx", hey0barx.getExpr()); + Assert.assertEquals(JSONPathFieldType.JQ, jqFoobar1.getType()); + Assert.assertEquals("foobar1", jqFoobar1.getName()); + Assert.assertEquals(".foo.bar1", jqFoobar1.getExpr()); + + Assert.assertEquals(JSONPathFieldType.JQ, jqBaz0.getType()); + Assert.assertEquals("baz0", jqBaz0.getName()); + Assert.assertEquals(".baz[0]", jqBaz0.getExpr()); + + Assert.assertEquals(JSONPathFieldType.JQ, jqHey0barx.getType()); + Assert.assertEquals("hey0barx", jqHey0barx.getName()); + Assert.assertEquals(".hey[0].barx", jqHey0barx.getExpr()); + Assert.assertEquals(JSONPathFieldType.ROOT, timestamp.getType()); Assert.assertEquals("timestamp", timestamp.getName()); Assert.assertEquals(null, timestamp.getExpr()); diff --git a/java-util/pom.xml b/java-util/pom.xml index 91225da71152..aed51349be62 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -107,6 +107,11 @@ test true + + net.thisptr + jackson-jq + RELEASE + diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java new file mode 100644 index 000000000000..0a424949ecf2 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java @@ -0,0 +1,60 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.common.parsers; + +import com.fasterxml.jackson.databind.JsonNode; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.JsonPath; +import net.thisptr.jackson.jq.JsonQuery; +import net.thisptr.jackson.jq.exception.JsonQueryException; + +import java.util.Map; + + +public class FlattenExpr +{ + private JsonPath jsonPathExpr; + private JsonQuery jsonQueryExpr; + + + FlattenExpr(JsonPath jsonPathExpr) { + this.jsonPathExpr = jsonPathExpr; + } + + FlattenExpr(JsonQuery jsonQueryExpr) { + this.jsonQueryExpr = jsonQueryExpr; + } + + public Object read(Map document, Configuration jsonConfig) { + return this.jsonPathExpr.read(document, jsonConfig); + } + + public JsonNode read(JsonNode document) { + try { + return this.jsonQueryExpr.apply(document).get(0); + } + catch (JsonQueryException e) { + e.printStackTrace(); + } + return null; + } + + +} diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java index 2717d2db915a..df993c51deb5 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java @@ -20,6 +20,7 @@ package io.druid.java.util.common.parsers; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Charsets; import com.jayway.jsonpath.Configuration; @@ -29,11 +30,14 @@ import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; +import net.thisptr.jackson.jq.JsonQuery; +import net.thisptr.jackson.jq.exception.JsonQueryException; import java.math.BigInteger; import java.nio.charset.CharsetEncoder; import java.util.ArrayList; import java.util.EnumSet; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -43,7 +47,7 @@ */ public class JSONPathParser implements Parser { - private final Map> fieldPathMap; + private final Map> fieldPathMap; private final boolean useFieldDiscovery; private final ObjectMapper mapper; private final CharsetEncoder enc = Charsets.UTF_8.newEncoder(); @@ -100,20 +104,25 @@ public Map parse(String input) { } ); - for (Map.Entry> entry : fieldPathMap.entrySet()) { + JsonNode jsonNode = mapper.readValue(input, JsonNode.class); + + for (Map.Entry> entry : fieldPathMap.entrySet()) { String fieldName = entry.getKey(); - Pair pair = entry.getValue(); - JsonPath path = pair.rhs; + Pair pair = entry.getValue(); + FlattenExpr path = pair.rhs; Object parsedVal; if (pair.lhs == FieldType.ROOT) { - parsedVal = document.get(fieldName); + parsedVal = valueConversionFunction(document.get(fieldName)); + } else if (pair.lhs == FieldType.PATH) { + parsedVal = valueConversionFunction(path.read(document, jsonPathConfig)); + } else if (pair.lhs == FieldType.JQ) { + parsedVal = jsonNodeValueConversionFunction(path.read(jsonNode)); } else { - parsedVal = path.read(document, jsonPathConfig); + throw new ParseException("Unknown FieldType", pair.lhs); } if (parsedVal == null) { continue; } - parsedVal = valueConversionFunction(parsedVal); map.put(fieldName, parsedVal); } if (useFieldDiscovery) { @@ -126,16 +135,26 @@ public Map parse(String input) } } - private Map> generateFieldPaths(List fieldSpecs) + private Map> generateFieldPaths(List fieldSpecs) { - Map> map = new LinkedHashMap<>(); + Map> map = new LinkedHashMap<>(); for (FieldSpec fieldSpec : fieldSpecs) { String fieldName = fieldSpec.getName(); if (map.get(fieldName) != null) { throw new IllegalArgumentException("Cannot have duplicate field definition: " + fieldName); } - JsonPath path = fieldSpec.getType() == FieldType.PATH ? JsonPath.compile(fieldSpec.getExpr()) : null; - Pair pair = new Pair<>(fieldSpec.getType(), path); + FlattenExpr path = null; + if (fieldSpec.getType() == FieldType.PATH) { + path = new FlattenExpr(JsonPath.compile(fieldSpec.getExpr())); + } else if (fieldSpec.getType() == FieldType.JQ) { + try { + path = new FlattenExpr(JsonQuery.compile(fieldSpec.getExpr())); + } + catch (JsonQueryException e) { + throw new ParseException(e, "Unable to flatten expression row [%s]", fieldSpec.getExpr()); + } + } + Pair pair = new Pair<>(fieldSpec.getType(), path); map.put(fieldName, pair); } return map; @@ -166,6 +185,10 @@ private void discoverFields(Map map, Map documen private Object valueConversionFunction(Object val) { + if (val == null) { + return null; + } + if (val instanceof Integer) { return Long.valueOf((Integer) val); } @@ -198,6 +221,42 @@ private Object valueConversionFunction(Object val) return val; } + private Object jsonNodeValueConversionFunction(JsonNode val) + { + if (val.isInt() || val.isLong()) { + return val.asLong(); + } + + if (val.isNumber()) { + return val.asDouble(); + } + + if (val.isTextual()) { + return charsetFix(val.asText()); + } + + if (val.isArray()) { + List newList = new ArrayList<>(); + for (Iterator it = val.iterator(); it.hasNext(); ) { + JsonNode entry = it.next(); + newList.add(jsonNodeValueConversionFunction(entry)); + } + return newList; + } + + if (val.isObject()) { + Map newMap = new LinkedHashMap<>(); + Map valMap = (Map) val; + for (Iterator> it = val.fields(); it.hasNext(); ) { + Map.Entry entry = it.next(); + newMap.put(entry.getKey(), jsonNodeValueConversionFunction(entry.getValue())); + } + return newMap; + } + + return val; + } + private String charsetFix(String s) { if (s != null && !enc.canEncode(s)) { @@ -233,7 +292,12 @@ public enum FieldType /** * A PATH field uses a JsonPath expression to retrieve the field value */ - PATH; + PATH, + + /** + * A JQ field uses a JsonQuery expression to retrieve the field value + */ + JQ; } /** diff --git a/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java b/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java index 671cf83acc3d..c9ba0f64a641 100644 --- a/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java +++ b/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java @@ -106,6 +106,11 @@ public void testNestingWithFieldDiscovery() fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.ROOT, "INVALID_ROOT", "INVALID_ROOT_EXPR")); fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "INVALID_PATH", "INVALID_PATH_EXPR")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-nested-foo.bar1", ".foo.bar1")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-nested-foo.bar2", ".foo.bar2")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-heybarx0", ".hey[0].barx")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-met-array", ".met.a")); + final Parser jsonParser = new JSONPathParser(fields, true, null); final Map jsonMap = jsonParser.parse(nestedJson); @@ -139,6 +144,11 @@ public void testNestingWithFieldDiscovery() Assert.assertEquals("asdf", jsonMap.get("heybarx0")); Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("met-array")); + Assert.assertEquals("aaa", jsonMap.get("jq-nested-foo.bar1")); + Assert.assertEquals("bbb", jsonMap.get("jq-nested-foo.bar2")); + Assert.assertEquals("asdf", jsonMap.get("jq-heybarx0")); + Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("jq-met-array")); + // Fields that should not be discovered Assert.assertNull(jsonMap.get("hey")); Assert.assertNull(jsonMap.get("met")); From 20fe0c222ce8406a45164ec22a77a5f51584ee3e Mon Sep 17 00:00:00 2001 From: Kenji Noguchi Date: Thu, 13 Apr 2017 19:25:04 -0700 Subject: [PATCH 02/14] more tests --- .../common/parsers/JSONPathParserTest.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java b/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java index c9ba0f64a641..466d59c405ef 100644 --- a/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java +++ b/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java @@ -169,6 +169,9 @@ public void testNestingNoDiscovery() fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "nested-foo.bar2", "$.foo.bar2")); fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "heybarx0", "$.hey[0].barx")); fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "met-array", "$.met.a")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-nested-foo.bar2", ".foo.bar2")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-heybarx0", ".hey[0].barx")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-met-array", ".met.a")); final Parser jsonParser = new JSONPathParser(fields, false, null); final Map jsonMap = jsonParser.parse(nestedJson); @@ -181,6 +184,9 @@ public void testNestingNoDiscovery() Assert.assertEquals("bbb", jsonMap.get("nested-foo.bar2")); Assert.assertEquals("asdf", jsonMap.get("heybarx0")); Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("met-array")); + Assert.assertEquals("bbb", jsonMap.get("jq-nested-foo.bar2")); + Assert.assertEquals("asdf", jsonMap.get("jq-heybarx0")); + Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("jq-met-array")); // Fields that should not be discovered Assert.assertNull(jsonMap.get("newmet")); @@ -208,6 +214,20 @@ public void testRejectDuplicates() final Map jsonMap = jsonParser.parse(nestedJson); } + @Test + public void testRejectDuplicates2() + { + List fields = new ArrayList<>(); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "met-array", "$.met.a")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "met-array", ".met.a")); + + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Cannot have duplicate field definition: met-array"); + + final Parser jsonParser = new JSONPathParser(fields, false, null); + final Map jsonMap = jsonParser.parse(nestedJson); + } + @Test public void testParseFail() { From a47efee337eae022f2730986ea3a57e5c8b6127b Mon Sep 17 00:00:00 2001 From: Kenji Noguchi Date: Thu, 25 May 2017 13:41:41 -0700 Subject: [PATCH 03/14] add benchmark --- .../druid/benchmark/FlattenJSONBenchmark.java | 18 ++++++++ .../benchmark/FlattenJSONBenchmarkUtil.java | 42 +++++++++++++++++++ .../FlattenJSONBenchmarkUtilTest.java | 3 ++ 3 files changed, 63 insertions(+) diff --git a/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmark.java index eedd9a530eaf..030e79604706 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmark.java @@ -45,12 +45,15 @@ public class FlattenJSONBenchmark List flatInputs; List nestedInputs; + List jqInputs; Parser flatParser; Parser nestedParser; + Parser jqParser; Parser fieldDiscoveryParser; Parser forcedPathParser; int flatCounter = 0; int nestedCounter = 0; + int jqCounter = 0; @Setup public void prepare() throws Exception @@ -64,9 +67,14 @@ public void prepare() throws Exception for (int i = 0; i < numEvents; i++) { nestedInputs.add(gen.generateNestedEvent()); } + jqInputs = new ArrayList(); + for (int i = 0; i < numEvents; i++) { + jqInputs.add(gen.generateNestedEvent()); // reuse the same event as "nested" + } flatParser = gen.getFlatParser(); nestedParser = gen.getNestedParser(); + jqParser = gen.getJqParser(); fieldDiscoveryParser = gen.getFieldDiscoveryParser(); forcedPathParser = gen.getForcedPathParser(); } @@ -91,6 +99,16 @@ public Map flatten() return parsed; } + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public Map jqflatten() + { + Map parsed = jqParser.parse(jqInputs.get(jqCounter)); + jqCounter = (jqCounter + 1) % numEvents; + return parsed; + } + @Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) diff --git a/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmarkUtil.java b/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmarkUtil.java index 00fbcdc97b85..b2bf55c205c3 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmarkUtil.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmarkUtil.java @@ -164,6 +164,48 @@ public Parser getForcedPathParser() return spec.makeParser(); } + public Parser getJqParser() + { + List fields = new ArrayList<>(); + fields.add(JSONPathFieldSpec.createRootField("ts")); + + fields.add(JSONPathFieldSpec.createRootField("d1")); + //fields.add(JSONPathFieldSpec.createRootField("d2")); + fields.add(JSONPathFieldSpec.createJqField("e1.d1", ".e1.d1")); + fields.add(JSONPathFieldSpec.createJqField("e1.d2", ".e1.d2")); + fields.add(JSONPathFieldSpec.createJqField("e2.d3", ".e2.d3")); + fields.add(JSONPathFieldSpec.createJqField("e2.d4", ".e2.d4")); + fields.add(JSONPathFieldSpec.createJqField("e2.d5", ".e2.d5")); + fields.add(JSONPathFieldSpec.createJqField("e2.d6", ".e2.d6")); + fields.add(JSONPathFieldSpec.createJqField("e2.ad1[0]", ".e2.ad1[0]")); + fields.add(JSONPathFieldSpec.createJqField("e2.ad1[1]", ".e2.ad1[1]")); + fields.add(JSONPathFieldSpec.createJqField("e2.ad1[2]", ".e2.ad1[2]")); + fields.add(JSONPathFieldSpec.createJqField("ae1[0].d1", ".ae1[0].d1")); + fields.add(JSONPathFieldSpec.createJqField("ae1[1].d1", ".ae1[1].d1")); + fields.add(JSONPathFieldSpec.createJqField("ae1[2].e1.d2", ".ae1[2].e1.d2")); + + fields.add(JSONPathFieldSpec.createRootField("m3")); + //fields.add(JSONPathFieldSpec.createRootField("m4")); + fields.add(JSONPathFieldSpec.createJqField("e3.m1", ".e3.m1")); + fields.add(JSONPathFieldSpec.createJqField("e3.m2", ".e3.m2")); + fields.add(JSONPathFieldSpec.createJqField("e3.m3", ".e3.m3")); + fields.add(JSONPathFieldSpec.createJqField("e3.m4", ".e3.m4")); + fields.add(JSONPathFieldSpec.createJqField("e3.am1[0]", ".e3.am1[0]")); + fields.add(JSONPathFieldSpec.createJqField("e3.am1[1]", ".e3.am1[1]")); + fields.add(JSONPathFieldSpec.createJqField("e3.am1[2]", ".e3.am1[2]")); + fields.add(JSONPathFieldSpec.createJqField("e3.am1[3]", ".e3.am1[3]")); + fields.add(JSONPathFieldSpec.createJqField("e4.e4.m4", ".e4.e4.m4")); + + JSONPathSpec flattenSpec = new JSONPathSpec(true, fields); + JSONParseSpec spec = new JSONParseSpec( + new TimestampSpec("ts", "iso", null), + new DimensionsSpec(null, null, null), + flattenSpec, + null + ); + + return spec.makeParser(); + } public String generateFlatEvent() throws Exception { diff --git a/benchmarks/src/test/java/io/druid/benchmark/FlattenJSONBenchmarkUtilTest.java b/benchmarks/src/test/java/io/druid/benchmark/FlattenJSONBenchmarkUtilTest.java index 324122ab337a..1649701840e1 100644 --- a/benchmarks/src/test/java/io/druid/benchmark/FlattenJSONBenchmarkUtilTest.java +++ b/benchmarks/src/test/java/io/druid/benchmark/FlattenJSONBenchmarkUtilTest.java @@ -37,12 +37,15 @@ public void testOne() throws Exception { Parser flatParser = eventGen.getFlatParser(); Parser nestedParser = eventGen.getNestedParser(); + Parser jqParser = eventGen.getJqParser(); Map event = flatParser.parse(newEvent); Map event2 = nestedParser.parse(newEvent2); + Map event3 = jqParser.parse(newEvent2); // reuse the same event as "nested" checkEvent1(event); checkEvent2(event2); + checkEvent2(event3); // make sure JQ parser output matches with JSONPath parser output } public void checkEvent1(Map event) { From 56e687d95ce4675f80f7309e2c804b4116c64fc8 Mon Sep 17 00:00:00 2001 From: Kenji Noguchi Date: Thu, 25 May 2017 16:29:53 -0700 Subject: [PATCH 04/14] fix style --- .../java/util/common/parsers/FlattenExpr.java | 14 ++++++++------ .../util/common/parsers/JSONPathParserTest.java | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java index 0a424949ecf2..9b11bdb5ba9e 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java @@ -34,19 +34,23 @@ public class FlattenExpr private JsonQuery jsonQueryExpr; - FlattenExpr(JsonPath jsonPathExpr) { + FlattenExpr(JsonPath jsonPathExpr) + { this.jsonPathExpr = jsonPathExpr; } - FlattenExpr(JsonQuery jsonQueryExpr) { + FlattenExpr(JsonQuery jsonQueryExpr) + { this.jsonQueryExpr = jsonQueryExpr; } - public Object read(Map document, Configuration jsonConfig) { + public Object read(Map document, Configuration jsonConfig) + { return this.jsonPathExpr.read(document, jsonConfig); } - public JsonNode read(JsonNode document) { + public JsonNode read(JsonNode document) + { try { return this.jsonQueryExpr.apply(document).get(0); } @@ -55,6 +59,4 @@ public JsonNode read(JsonNode document) { } return null; } - - } diff --git a/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java b/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java index 466d59c405ef..af9c2b703b66 100644 --- a/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java +++ b/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java @@ -123,11 +123,11 @@ public void testNestingWithFieldDiscovery() Assert.assertEquals("2999", jsonMap.get("timestamp")); Assert.assertEquals("Hello world!", jsonMap.get("foo.bar1")); - List testListConvert = (List)jsonMap.get("testListConvert"); + List testListConvert = (List) jsonMap.get("testListConvert"); Assert.assertEquals(1.23456789E21, testListConvert.get(0)); Assert.assertEquals("foo?", testListConvert.get(1)); - List testListConvert2 = (List)jsonMap.get("testListConvert2"); + List testListConvert2 = (List) jsonMap.get("testListConvert2"); Assert.assertEquals(1.23456789E21, testListConvert2.get(0)); Assert.assertEquals("foo?", testListConvert2.get(1)); Assert.assertEquals(1.23456789E21, ((List) testListConvert2.get(2)).get(0)); From 5d8a3430707ba14f66cac49a1cd5ae2320bf08b0 Mon Sep 17 00:00:00 2001 From: Kenji Noguchi Date: Sun, 28 May 2017 21:37:07 -0700 Subject: [PATCH 05/14] use JsonNode for both JSONPath and JQ --- .../java/util/common/parsers/FlattenExpr.java | 4 +- .../util/common/parsers/JSONPathParser.java | 82 +++++-------------- 2 files changed, 21 insertions(+), 65 deletions(-) diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java index 9b11bdb5ba9e..a355340c9612 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java @@ -25,8 +25,6 @@ import net.thisptr.jackson.jq.JsonQuery; import net.thisptr.jackson.jq.exception.JsonQueryException; -import java.util.Map; - public class FlattenExpr { @@ -44,7 +42,7 @@ public class FlattenExpr this.jsonQueryExpr = jsonQueryExpr; } - public Object read(Map document, Configuration jsonConfig) + public JsonNode read(JsonNode document, Configuration jsonConfig) { return this.jsonPathExpr.read(document, jsonConfig); } diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java index df993c51deb5..2a5c22d80315 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java @@ -19,21 +19,19 @@ package io.druid.java.util.common.parsers; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Charsets; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.Option; -import com.jayway.jsonpath.spi.json.JacksonJsonProvider; +import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import net.thisptr.jackson.jq.JsonQuery; import net.thisptr.jackson.jq.exception.JsonQueryException; -import java.math.BigInteger; import java.nio.charset.CharsetEncoder; import java.util.ArrayList; import java.util.EnumSet; @@ -69,7 +67,7 @@ public JSONPathParser(List fieldSpecs, boolean useFieldDiscovery, Obj // Avoid using defaultConfiguration, as this depends on json-smart which we are excluding. this.jsonPathConfig = Configuration.builder() - .jsonProvider(new JacksonJsonProvider()) + .jsonProvider(new JacksonJsonNodeJsonProvider()) .mappingProvider(new JacksonMappingProvider()) .options(EnumSet.of(Option.SUPPRESS_EXCEPTIONS)) .build(); @@ -98,13 +96,7 @@ public Map parse(String input) { try { Map map = new LinkedHashMap<>(); - Map document = mapper.readValue( - input, - new TypeReference>() - { - } - ); - JsonNode jsonNode = mapper.readValue(input, JsonNode.class); + JsonNode document = mapper.readValue(input, JsonNode.class); for (Map.Entry> entry : fieldPathMap.entrySet()) { String fieldName = entry.getKey(); @@ -116,7 +108,7 @@ public Map parse(String input) } else if (pair.lhs == FieldType.PATH) { parsedVal = valueConversionFunction(path.read(document, jsonPathConfig)); } else if (pair.lhs == FieldType.JQ) { - parsedVal = jsonNodeValueConversionFunction(path.read(jsonNode)); + parsedVal = valueConversionFunction(path.read(document)); } else { throw new ParseException("Unknown FieldType", pair.lhs); } @@ -160,69 +152,36 @@ private Map> generateFieldPaths(List map, Map document) + private void discoverFields(Map map, JsonNode document) { - for (Map.Entry e : document.entrySet()) { + for (Iterator> it = document.fields(); it.hasNext(); ) { + Map.Entry e = it.next(); String field = e.getKey(); if (!map.containsKey(field)) { - Object val = e.getValue(); - if (val == null) { + JsonNode val = e.getValue(); + if (val.isNull()) { continue; } - if (val instanceof Map) { + if (val.isObject()) { continue; } - if (val instanceof List) { - if (!isFlatList((List) val)) { + if (val.isArray()) { + if (!isFlatList(val)) { continue; } } - val = valueConversionFunction(val); - map.put(field, val); + Object val2 = valueConversionFunction(val); + map.put(field, val2); } } } - private Object valueConversionFunction(Object val) + private Object valueConversionFunction(JsonNode val) { if (val == null) { return null; } - if (val instanceof Integer) { - return Long.valueOf((Integer) val); - } - - if (val instanceof BigInteger) { - return Double.valueOf(((BigInteger) val).doubleValue()); - } - - if (val instanceof String) { - return charsetFix((String) val); - } - - if (val instanceof List) { - List newList = new ArrayList<>(); - for (Object entry : ((List) val)) { - newList.add(valueConversionFunction(entry)); - } - return newList; - } - - if (val instanceof Map) { - Map newMap = new LinkedHashMap<>(); - Map valMap = (Map) val; - for (Map.Entry entry : valMap.entrySet()) { - newMap.put(entry.getKey(), valueConversionFunction(entry.getValue())); - } - return newMap; - } - - return val; - } - - private Object jsonNodeValueConversionFunction(JsonNode val) - { if (val.isInt() || val.isLong()) { return val.asLong(); } @@ -239,17 +198,16 @@ private Object jsonNodeValueConversionFunction(JsonNode val) List newList = new ArrayList<>(); for (Iterator it = val.iterator(); it.hasNext(); ) { JsonNode entry = it.next(); - newList.add(jsonNodeValueConversionFunction(entry)); + newList.add(valueConversionFunction(entry)); } return newList; } if (val.isObject()) { Map newMap = new LinkedHashMap<>(); - Map valMap = (Map) val; for (Iterator> it = val.fields(); it.hasNext(); ) { Map.Entry entry = it.next(); - newMap.put(entry.getKey(), jsonNodeValueConversionFunction(entry.getValue())); + newMap.put(entry.getKey(), valueConversionFunction(entry.getValue())); } return newMap; } @@ -269,10 +227,10 @@ private String charsetFix(String s) } } - private boolean isFlatList(List list) + private boolean isFlatList(JsonNode list) { - for (Object obj : list) { - if ((obj instanceof Map) || (obj instanceof List)) { + for (JsonNode obj : list) { + if (obj.isObject() || obj.isArray()) { return false; } } From 70c03509e604cf09e65e8b3ebebdb8151d6f7cd5 Mon Sep 17 00:00:00 2001 From: Kenji Noguchi Date: Sun, 28 May 2017 21:51:41 -0700 Subject: [PATCH 06/14] clean up --- .../io/druid/java/util/common/parsers/FlattenExpr.java | 8 +++++--- .../druid/java/util/common/parsers/JSONPathParser.java | 9 ++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java index a355340c9612..cd26508d3f3f 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java @@ -25,7 +25,9 @@ import net.thisptr.jackson.jq.JsonQuery; import net.thisptr.jackson.jq.exception.JsonQueryException; - +/* + * Flatten expr adapter class + */ public class FlattenExpr { private JsonPath jsonPathExpr; @@ -42,12 +44,12 @@ public class FlattenExpr this.jsonQueryExpr = jsonQueryExpr; } - public JsonNode read(JsonNode document, Configuration jsonConfig) + public JsonNode readPath(JsonNode document, Configuration jsonConfig) { return this.jsonPathExpr.read(document, jsonConfig); } - public JsonNode read(JsonNode document) + public JsonNode readJq(JsonNode document) { try { return this.jsonQueryExpr.apply(document).get(0); diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java index 2a5c22d80315..031391f6b59e 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java @@ -106,9 +106,9 @@ public Map parse(String input) if (pair.lhs == FieldType.ROOT) { parsedVal = valueConversionFunction(document.get(fieldName)); } else if (pair.lhs == FieldType.PATH) { - parsedVal = valueConversionFunction(path.read(document, jsonPathConfig)); + parsedVal = valueConversionFunction(path.readPath(document, jsonPathConfig)); } else if (pair.lhs == FieldType.JQ) { - parsedVal = valueConversionFunction(path.read(document)); + parsedVal = valueConversionFunction(path.readJq(document)); } else { throw new ParseException("Unknown FieldType", pair.lhs); } @@ -143,7 +143,7 @@ private Map> generateFieldPaths(List pair = new Pair<>(fieldSpec.getType(), path); @@ -170,8 +170,7 @@ private void discoverFields(Map map, JsonNode document) continue; } } - Object val2 = valueConversionFunction(val); - map.put(field, val2); + map.put(field, valueConversionFunction(val)); } } } From 47ba363ac04f846e6b71694391d0aaec06587672 Mon Sep 17 00:00:00 2001 From: Kenji Noguchi Date: Sun, 28 May 2017 22:20:57 -0700 Subject: [PATCH 07/14] more clean up --- .../druid/java/util/common/parsers/JSONPathParser.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java index 031391f6b59e..3b1ca9db1e15 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java @@ -102,20 +102,20 @@ public Map parse(String input) String fieldName = entry.getKey(); Pair pair = entry.getValue(); FlattenExpr path = pair.rhs; - Object parsedVal; + JsonNode parsedVal; if (pair.lhs == FieldType.ROOT) { - parsedVal = valueConversionFunction(document.get(fieldName)); + parsedVal = document.get(fieldName); } else if (pair.lhs == FieldType.PATH) { - parsedVal = valueConversionFunction(path.readPath(document, jsonPathConfig)); + parsedVal = path.readPath(document, jsonPathConfig); } else if (pair.lhs == FieldType.JQ) { - parsedVal = valueConversionFunction(path.readJq(document)); + parsedVal = path.readJq(document); } else { throw new ParseException("Unknown FieldType", pair.lhs); } if (parsedVal == null) { continue; } - map.put(fieldName, parsedVal); + map.put(fieldName, valueConversionFunction(parsedVal)); } if (useFieldDiscovery) { discoverFields(map, document); From d32785886b269bfe8eeb4c0e328ca658d64698d4 Mon Sep 17 00:00:00 2001 From: Kenji Noguchi Date: Sun, 28 May 2017 22:45:28 -0700 Subject: [PATCH 08/14] add documentation --- docs/content/ingestion/flatten-json.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/content/ingestion/flatten-json.md b/docs/content/ingestion/flatten-json.md index 432823dde053..8c76974cc6aa 100644 --- a/docs/content/ingestion/flatten-json.md +++ b/docs/content/ingestion/flatten-json.md @@ -17,9 +17,9 @@ Defining the JSON Flatten Spec allows nested JSON fields to be flattened during | Field | Type | Description | Required | |-------|------|-------------|----------| -| type | String | Type of the field, "root" or "path". | yes | +| type | String | Type of the field, "root", "path" or "jq". | yes | | name | String | This string will be used as the column name when the data has been ingested. | yes | -| expr | String | Defines an expression for accessing the field within the JSON object, using [JsonPath](https://github.com/jayway/JsonPath) notation. Only used for type "path", otherwise ignored. | only for type "path" | +| expr | String | Defines an expression for accessing the field within the JSON object, using [JsonPath](https://github.com/jayway/JsonPath) notation for type "path", and [jackson-jq](https://github.com/eiiches/jackson-jq) for type "jq". This field is only used for type "path" and "jq", otherwise ignored. | only for type "path" or "jq" | Suppose the event JSON has the following form: @@ -99,6 +99,16 @@ To flatten this JSON, the parseSpec could be defined as follows: "type": "path", "name": "second-food", "expr": "$.thing.food[1]" + }, + { + "type": "jq", + "name": "first-food-by-jq", + "expr": ".thing.food[1]" + }, + { + "type": "jq", + "name": "hello-total", + "expr": ".hello | sum" } ] }, @@ -147,3 +157,4 @@ Note that: * If auto field discovery is enabled, any discovered field with the same name as one already defined in the field specs will be skipped and not added twice. * The JSON input must be a JSON object at the root, not an array. e.g., {"valid": "true"} and {"valid":[1,2,3]} are supported but [{"invalid": "true"}] and [1,2,3] are not. * [http://jsonpath.herokuapp.com/](http://jsonpath.herokuapp.com/) is useful for testing the path expressions. +* jackson-jq supports subset of [./jq](https://stedolan.github.io/jq/) syntax. Please refer jackson-jq document. \ No newline at end of file From a1bc0eebe12089821ebdb8b13e1c762830cecff5 Mon Sep 17 00:00:00 2001 From: Kenji Noguchi Date: Sun, 28 May 2017 23:04:04 -0700 Subject: [PATCH 09/14] fix style --- .../java/io/druid/java/util/common/parsers/FlattenExpr.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java index cd26508d3f3f..1a1ee11c8b50 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java @@ -25,9 +25,7 @@ import net.thisptr.jackson.jq.JsonQuery; import net.thisptr.jackson.jq.exception.JsonQueryException; -/* - * Flatten expr adapter class - */ + public class FlattenExpr { private JsonPath jsonPathExpr; From 0fe3d1daa6e35686996a9c2eea4ee1809ea2de38 Mon Sep 17 00:00:00 2001 From: Kenji Noguchi Date: Wed, 6 Sep 2017 14:21:59 -0700 Subject: [PATCH 10/14] move jackson-jq version to dependencyManagement section. remove commented code --- api/pom.xml | 1 - .../java/io/druid/benchmark/FlattenJSONBenchmarkUtil.java | 2 -- java-util/pom.xml | 1 - .../io/druid/java/util/common/parsers/JSONPathParser.java | 2 +- pom.xml | 5 +++++ 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index bda9be4658b2..2b06a20d346d 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -90,7 +90,6 @@ net.thisptr jackson-jq - 0.0.7 diff --git a/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmarkUtil.java b/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmarkUtil.java index b2bf55c205c3..81c613be764e 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmarkUtil.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmarkUtil.java @@ -170,7 +170,6 @@ public Parser getJqParser() fields.add(JSONPathFieldSpec.createRootField("ts")); fields.add(JSONPathFieldSpec.createRootField("d1")); - //fields.add(JSONPathFieldSpec.createRootField("d2")); fields.add(JSONPathFieldSpec.createJqField("e1.d1", ".e1.d1")); fields.add(JSONPathFieldSpec.createJqField("e1.d2", ".e1.d2")); fields.add(JSONPathFieldSpec.createJqField("e2.d3", ".e2.d3")); @@ -185,7 +184,6 @@ public Parser getJqParser() fields.add(JSONPathFieldSpec.createJqField("ae1[2].e1.d2", ".ae1[2].e1.d2")); fields.add(JSONPathFieldSpec.createRootField("m3")); - //fields.add(JSONPathFieldSpec.createRootField("m4")); fields.add(JSONPathFieldSpec.createJqField("e3.m1", ".e3.m1")); fields.add(JSONPathFieldSpec.createJqField("e3.m2", ".e3.m2")); fields.add(JSONPathFieldSpec.createJqField("e3.m3", ".e3.m3")); diff --git a/java-util/pom.xml b/java-util/pom.xml index aed51349be62..0c67c71545df 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -110,7 +110,6 @@ net.thisptr jackson-jq - RELEASE diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java index 3b1ca9db1e15..8e6b9f8851d0 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java @@ -110,7 +110,7 @@ public Map parse(String input) } else if (pair.lhs == FieldType.JQ) { parsedVal = path.readJq(document); } else { - throw new ParseException("Unknown FieldType", pair.lhs); + throw new IllegalArgumentException("Unknown FieldType: " + pair.lhs); } if (parsedVal == null) { continue; diff --git a/pom.xml b/pom.xml index 0c197cac5b7c..92bdca38a584 100644 --- a/pom.xml +++ b/pom.xml @@ -629,6 +629,11 @@ json-path 2.1.0 + + net.thisptr + jackson-jq + 0.0.7 + org.slf4j slf4j-api From c140788c6f92a610175c1fdbdae8ad0759b93d75 Mon Sep 17 00:00:00 2001 From: Kenji Noguchi Date: Wed, 6 Sep 2017 14:26:13 -0700 Subject: [PATCH 11/14] oops. revert wrong fix --- .../java/io/druid/java/util/common/parsers/JSONPathParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java index 8e6b9f8851d0..3b1ca9db1e15 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java @@ -110,7 +110,7 @@ public Map parse(String input) } else if (pair.lhs == FieldType.JQ) { parsedVal = path.readJq(document); } else { - throw new IllegalArgumentException("Unknown FieldType: " + pair.lhs); + throw new ParseException("Unknown FieldType", pair.lhs); } if (parsedVal == null) { continue; From 24eeefa74382e6f5d4765d79dcfe79743bcd3f2a Mon Sep 17 00:00:00 2001 From: Kenji Noguchi Date: Wed, 6 Sep 2017 14:26:51 -0700 Subject: [PATCH 12/14] throw IllegalArgumentException for JQ syntax error --- .../java/io/druid/java/util/common/parsers/JSONPathParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java index 3b1ca9db1e15..d7ec988ae8fa 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java @@ -143,7 +143,7 @@ private Map> generateFieldPaths(List pair = new Pair<>(fieldSpec.getType(), path); From 5e4954d6cb7e2b1d5ded1380567ced79136f97ea Mon Sep 17 00:00:00 2001 From: Kenji Noguchi Date: Wed, 6 Sep 2017 14:40:43 -0700 Subject: [PATCH 13/14] remove e.printStackTrace() that is forbidden --- .../java/io/druid/java/util/common/parsers/FlattenExpr.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java index 1a1ee11c8b50..46ddb05b5383 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java @@ -53,7 +53,7 @@ public JsonNode readJq(JsonNode document) return this.jsonQueryExpr.apply(document).get(0); } catch (JsonQueryException e) { - e.printStackTrace(); + // ignore errors } return null; } From 79c7baaf82c38979aefd26122832771cb5c74380 Mon Sep 17 00:00:00 2001 From: Kenji Noguchi Date: Wed, 6 Sep 2017 16:06:40 -0700 Subject: [PATCH 14/14] touch --- .../java/io/druid/java/util/common/parsers/FlattenExpr.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java index 46ddb05b5383..38b56813b8df 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java @@ -53,7 +53,7 @@ public JsonNode readJq(JsonNode document) return this.jsonQueryExpr.apply(document).get(0); } catch (JsonQueryException e) { - // ignore errors + // ignore errors. } return null; }