diff --git a/api/pom.xml b/api/pom.xml index d494af174e4c..2b06a20d346d 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -87,7 +87,10 @@ com.google.code.findbugs jsr305 - + + net.thisptr + jackson-jq + junit diff --git a/api/src/main/java/io/druid/data/input/impl/JSONParseSpec.java b/api/src/main/java/io/druid/data/input/impl/JSONParseSpec.java index 81ce73b94a44..53fe138c59f4 100644 --- a/api/src/main/java/io/druid/data/input/impl/JSONParseSpec.java +++ b/api/src/main/java/io/druid/data/input/impl/JSONParseSpec.java @@ -115,6 +115,9 @@ private List convertFieldSpecs(List case PATH: type = JSONPathParser.FieldType.PATH; break; + case JQ: + type = JSONPathParser.FieldType.JQ; + break; default: throw new IllegalArgumentException("Invalid type for field " + druidSpec.getName()); } diff --git a/api/src/main/java/io/druid/data/input/impl/JSONPathFieldSpec.java b/api/src/main/java/io/druid/data/input/impl/JSONPathFieldSpec.java index 2825d92652f4..af436c941c83 100644 --- a/api/src/main/java/io/druid/data/input/impl/JSONPathFieldSpec.java +++ b/api/src/main/java/io/druid/data/input/impl/JSONPathFieldSpec.java @@ -69,6 +69,11 @@ public static JSONPathFieldSpec createNestedField(String name, String expr) return new JSONPathFieldSpec(JSONPathFieldType.PATH, name, expr); } + public static JSONPathFieldSpec createJqField(String name, String expr) + { + return new JSONPathFieldSpec(JSONPathFieldType.JQ, name, expr); + } + public static JSONPathFieldSpec createRootField(String name) { return new JSONPathFieldSpec(JSONPathFieldType.ROOT, name, null); diff --git a/api/src/main/java/io/druid/data/input/impl/JSONPathFieldType.java b/api/src/main/java/io/druid/data/input/impl/JSONPathFieldType.java index d99ad77c44e6..a69165837fcd 100644 --- a/api/src/main/java/io/druid/data/input/impl/JSONPathFieldType.java +++ b/api/src/main/java/io/druid/data/input/impl/JSONPathFieldType.java @@ -25,7 +25,8 @@ public enum JSONPathFieldType { ROOT, - PATH; + PATH, + JQ; @JsonValue @Override diff --git a/api/src/test/java/io/druid/data/input/impl/JSONPathSpecTest.java b/api/src/test/java/io/druid/data/input/impl/JSONPathSpecTest.java index 5f405409d86c..ec7dd41da08b 100644 --- a/api/src/test/java/io/druid/data/input/impl/JSONPathSpecTest.java +++ b/api/src/test/java/io/druid/data/input/impl/JSONPathSpecTest.java @@ -41,6 +41,9 @@ public void testSerde() throws IOException fields.add(JSONPathFieldSpec.createNestedField("hey0barx", "$.hey[0].barx")); fields.add(JSONPathFieldSpec.createRootField("timestamp")); fields.add(JSONPathFieldSpec.createRootField("foo.bar1")); + fields.add(JSONPathFieldSpec.createJqField("foobar1", ".foo.bar1")); + fields.add(JSONPathFieldSpec.createJqField("baz0", ".baz[0]")); + fields.add(JSONPathFieldSpec.createJqField("hey0barx", ".hey[0].barx")); JSONPathSpec flattenSpec = new JSONPathSpec(true, fields); @@ -55,6 +58,9 @@ public void testSerde() throws IOException JSONPathFieldSpec hey0barx = serdeFields.get(2); JSONPathFieldSpec timestamp = serdeFields.get(3); JSONPathFieldSpec foodotbar1 = serdeFields.get(4); + JSONPathFieldSpec jqFoobar1 = serdeFields.get(5); + JSONPathFieldSpec jqBaz0 = serdeFields.get(6); + JSONPathFieldSpec jqHey0barx = serdeFields.get(7); Assert.assertEquals(JSONPathFieldType.PATH, foobar1.getType()); Assert.assertEquals("foobar1", foobar1.getName()); @@ -68,6 +74,18 @@ public void testSerde() throws IOException Assert.assertEquals("hey0barx", hey0barx.getName()); Assert.assertEquals("$.hey[0].barx", hey0barx.getExpr()); + Assert.assertEquals(JSONPathFieldType.JQ, jqFoobar1.getType()); + Assert.assertEquals("foobar1", jqFoobar1.getName()); + Assert.assertEquals(".foo.bar1", jqFoobar1.getExpr()); + + Assert.assertEquals(JSONPathFieldType.JQ, jqBaz0.getType()); + Assert.assertEquals("baz0", jqBaz0.getName()); + Assert.assertEquals(".baz[0]", jqBaz0.getExpr()); + + Assert.assertEquals(JSONPathFieldType.JQ, jqHey0barx.getType()); + Assert.assertEquals("hey0barx", jqHey0barx.getName()); + Assert.assertEquals(".hey[0].barx", jqHey0barx.getExpr()); + Assert.assertEquals(JSONPathFieldType.ROOT, timestamp.getType()); Assert.assertEquals("timestamp", timestamp.getName()); Assert.assertEquals(null, timestamp.getExpr()); diff --git a/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmark.java index eedd9a530eaf..030e79604706 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmark.java @@ -45,12 +45,15 @@ public class FlattenJSONBenchmark List flatInputs; List nestedInputs; + List jqInputs; Parser flatParser; Parser nestedParser; + Parser jqParser; Parser fieldDiscoveryParser; Parser forcedPathParser; int flatCounter = 0; int nestedCounter = 0; + int jqCounter = 0; @Setup public void prepare() throws Exception @@ -64,9 +67,14 @@ public void prepare() throws Exception for (int i = 0; i < numEvents; i++) { nestedInputs.add(gen.generateNestedEvent()); } + jqInputs = new ArrayList(); + for (int i = 0; i < numEvents; i++) { + jqInputs.add(gen.generateNestedEvent()); // reuse the same event as "nested" + } flatParser = gen.getFlatParser(); nestedParser = gen.getNestedParser(); + jqParser = gen.getJqParser(); fieldDiscoveryParser = gen.getFieldDiscoveryParser(); forcedPathParser = gen.getForcedPathParser(); } @@ -91,6 +99,16 @@ public Map flatten() return parsed; } + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public Map jqflatten() + { + Map parsed = jqParser.parse(jqInputs.get(jqCounter)); + jqCounter = (jqCounter + 1) % numEvents; + return parsed; + } + @Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) diff --git a/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmarkUtil.java b/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmarkUtil.java index 00fbcdc97b85..81c613be764e 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmarkUtil.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FlattenJSONBenchmarkUtil.java @@ -164,6 +164,46 @@ public Parser getForcedPathParser() return spec.makeParser(); } + public Parser getJqParser() + { + List fields = new ArrayList<>(); + fields.add(JSONPathFieldSpec.createRootField("ts")); + + fields.add(JSONPathFieldSpec.createRootField("d1")); + fields.add(JSONPathFieldSpec.createJqField("e1.d1", ".e1.d1")); + fields.add(JSONPathFieldSpec.createJqField("e1.d2", ".e1.d2")); + fields.add(JSONPathFieldSpec.createJqField("e2.d3", ".e2.d3")); + fields.add(JSONPathFieldSpec.createJqField("e2.d4", ".e2.d4")); + fields.add(JSONPathFieldSpec.createJqField("e2.d5", ".e2.d5")); + fields.add(JSONPathFieldSpec.createJqField("e2.d6", ".e2.d6")); + fields.add(JSONPathFieldSpec.createJqField("e2.ad1[0]", ".e2.ad1[0]")); + fields.add(JSONPathFieldSpec.createJqField("e2.ad1[1]", ".e2.ad1[1]")); + fields.add(JSONPathFieldSpec.createJqField("e2.ad1[2]", ".e2.ad1[2]")); + fields.add(JSONPathFieldSpec.createJqField("ae1[0].d1", ".ae1[0].d1")); + fields.add(JSONPathFieldSpec.createJqField("ae1[1].d1", ".ae1[1].d1")); + fields.add(JSONPathFieldSpec.createJqField("ae1[2].e1.d2", ".ae1[2].e1.d2")); + + fields.add(JSONPathFieldSpec.createRootField("m3")); + fields.add(JSONPathFieldSpec.createJqField("e3.m1", ".e3.m1")); + fields.add(JSONPathFieldSpec.createJqField("e3.m2", ".e3.m2")); + fields.add(JSONPathFieldSpec.createJqField("e3.m3", ".e3.m3")); + fields.add(JSONPathFieldSpec.createJqField("e3.m4", ".e3.m4")); + fields.add(JSONPathFieldSpec.createJqField("e3.am1[0]", ".e3.am1[0]")); + fields.add(JSONPathFieldSpec.createJqField("e3.am1[1]", ".e3.am1[1]")); + fields.add(JSONPathFieldSpec.createJqField("e3.am1[2]", ".e3.am1[2]")); + fields.add(JSONPathFieldSpec.createJqField("e3.am1[3]", ".e3.am1[3]")); + fields.add(JSONPathFieldSpec.createJqField("e4.e4.m4", ".e4.e4.m4")); + + JSONPathSpec flattenSpec = new JSONPathSpec(true, fields); + JSONParseSpec spec = new JSONParseSpec( + new TimestampSpec("ts", "iso", null), + new DimensionsSpec(null, null, null), + flattenSpec, + null + ); + + return spec.makeParser(); + } public String generateFlatEvent() throws Exception { diff --git a/benchmarks/src/test/java/io/druid/benchmark/FlattenJSONBenchmarkUtilTest.java b/benchmarks/src/test/java/io/druid/benchmark/FlattenJSONBenchmarkUtilTest.java index 324122ab337a..1649701840e1 100644 --- a/benchmarks/src/test/java/io/druid/benchmark/FlattenJSONBenchmarkUtilTest.java +++ b/benchmarks/src/test/java/io/druid/benchmark/FlattenJSONBenchmarkUtilTest.java @@ -37,12 +37,15 @@ public void testOne() throws Exception { Parser flatParser = eventGen.getFlatParser(); Parser nestedParser = eventGen.getNestedParser(); + Parser jqParser = eventGen.getJqParser(); Map event = flatParser.parse(newEvent); Map event2 = nestedParser.parse(newEvent2); + Map event3 = jqParser.parse(newEvent2); // reuse the same event as "nested" checkEvent1(event); checkEvent2(event2); + checkEvent2(event3); // make sure JQ parser output matches with JSONPath parser output } public void checkEvent1(Map event) { diff --git a/docs/content/ingestion/flatten-json.md b/docs/content/ingestion/flatten-json.md index 432823dde053..8c76974cc6aa 100644 --- a/docs/content/ingestion/flatten-json.md +++ b/docs/content/ingestion/flatten-json.md @@ -17,9 +17,9 @@ Defining the JSON Flatten Spec allows nested JSON fields to be flattened during | Field | Type | Description | Required | |-------|------|-------------|----------| -| type | String | Type of the field, "root" or "path". | yes | +| type | String | Type of the field, "root", "path" or "jq". | yes | | name | String | This string will be used as the column name when the data has been ingested. | yes | -| expr | String | Defines an expression for accessing the field within the JSON object, using [JsonPath](https://github.com/jayway/JsonPath) notation. Only used for type "path", otherwise ignored. | only for type "path" | +| expr | String | Defines an expression for accessing the field within the JSON object, using [JsonPath](https://github.com/jayway/JsonPath) notation for type "path", and [jackson-jq](https://github.com/eiiches/jackson-jq) for type "jq". This field is only used for type "path" and "jq", otherwise ignored. | only for type "path" or "jq" | Suppose the event JSON has the following form: @@ -99,6 +99,16 @@ To flatten this JSON, the parseSpec could be defined as follows: "type": "path", "name": "second-food", "expr": "$.thing.food[1]" + }, + { + "type": "jq", + "name": "first-food-by-jq", + "expr": ".thing.food[1]" + }, + { + "type": "jq", + "name": "hello-total", + "expr": ".hello | sum" } ] }, @@ -147,3 +157,4 @@ Note that: * If auto field discovery is enabled, any discovered field with the same name as one already defined in the field specs will be skipped and not added twice. * The JSON input must be a JSON object at the root, not an array. e.g., {"valid": "true"} and {"valid":[1,2,3]} are supported but [{"invalid": "true"}] and [1,2,3] are not. * [http://jsonpath.herokuapp.com/](http://jsonpath.herokuapp.com/) is useful for testing the path expressions. +* jackson-jq supports subset of [./jq](https://stedolan.github.io/jq/) syntax. Please refer jackson-jq document. \ No newline at end of file diff --git a/java-util/pom.xml b/java-util/pom.xml index 91225da71152..0c67c71545df 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -107,6 +107,10 @@ test true + + net.thisptr + jackson-jq + diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java new file mode 100644 index 000000000000..38b56813b8df --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/FlattenExpr.java @@ -0,0 +1,60 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.common.parsers; + +import com.fasterxml.jackson.databind.JsonNode; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.JsonPath; +import net.thisptr.jackson.jq.JsonQuery; +import net.thisptr.jackson.jq.exception.JsonQueryException; + + +public class FlattenExpr +{ + private JsonPath jsonPathExpr; + private JsonQuery jsonQueryExpr; + + + FlattenExpr(JsonPath jsonPathExpr) + { + this.jsonPathExpr = jsonPathExpr; + } + + FlattenExpr(JsonQuery jsonQueryExpr) + { + this.jsonQueryExpr = jsonQueryExpr; + } + + public JsonNode readPath(JsonNode document, Configuration jsonConfig) + { + return this.jsonPathExpr.read(document, jsonConfig); + } + + public JsonNode readJq(JsonNode document) + { + try { + return this.jsonQueryExpr.apply(document).get(0); + } + catch (JsonQueryException e) { + // ignore errors. + } + return null; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java index 2717d2db915a..d7ec988ae8fa 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/JSONPathParser.java @@ -19,21 +19,23 @@ package io.druid.java.util.common.parsers; -import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Charsets; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.Option; -import com.jayway.jsonpath.spi.json.JacksonJsonProvider; +import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; +import net.thisptr.jackson.jq.JsonQuery; +import net.thisptr.jackson.jq.exception.JsonQueryException; -import java.math.BigInteger; import java.nio.charset.CharsetEncoder; import java.util.ArrayList; import java.util.EnumSet; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -43,7 +45,7 @@ */ public class JSONPathParser implements Parser { - private final Map> fieldPathMap; + private final Map> fieldPathMap; private final boolean useFieldDiscovery; private final ObjectMapper mapper; private final CharsetEncoder enc = Charsets.UTF_8.newEncoder(); @@ -65,7 +67,7 @@ public JSONPathParser(List fieldSpecs, boolean useFieldDiscovery, Obj // Avoid using defaultConfiguration, as this depends on json-smart which we are excluding. this.jsonPathConfig = Configuration.builder() - .jsonProvider(new JacksonJsonProvider()) + .jsonProvider(new JacksonJsonNodeJsonProvider()) .mappingProvider(new JacksonMappingProvider()) .options(EnumSet.of(Option.SUPPRESS_EXCEPTIONS)) .build(); @@ -94,27 +96,26 @@ public Map parse(String input) { try { Map map = new LinkedHashMap<>(); - Map document = mapper.readValue( - input, - new TypeReference>() - { - } - ); - for (Map.Entry> entry : fieldPathMap.entrySet()) { + JsonNode document = mapper.readValue(input, JsonNode.class); + + for (Map.Entry> entry : fieldPathMap.entrySet()) { String fieldName = entry.getKey(); - Pair pair = entry.getValue(); - JsonPath path = pair.rhs; - Object parsedVal; + Pair pair = entry.getValue(); + FlattenExpr path = pair.rhs; + JsonNode parsedVal; if (pair.lhs == FieldType.ROOT) { parsedVal = document.get(fieldName); + } else if (pair.lhs == FieldType.PATH) { + parsedVal = path.readPath(document, jsonPathConfig); + } else if (pair.lhs == FieldType.JQ) { + parsedVal = path.readJq(document); } else { - parsedVal = path.read(document, jsonPathConfig); + throw new ParseException("Unknown FieldType", pair.lhs); } if (parsedVal == null) { continue; } - parsedVal = valueConversionFunction(parsedVal); - map.put(fieldName, parsedVal); + map.put(fieldName, valueConversionFunction(parsedVal)); } if (useFieldDiscovery) { discoverFields(map, document); @@ -126,70 +127,85 @@ public Map parse(String input) } } - private Map> generateFieldPaths(List fieldSpecs) + private Map> generateFieldPaths(List fieldSpecs) { - Map> map = new LinkedHashMap<>(); + Map> map = new LinkedHashMap<>(); for (FieldSpec fieldSpec : fieldSpecs) { String fieldName = fieldSpec.getName(); if (map.get(fieldName) != null) { throw new IllegalArgumentException("Cannot have duplicate field definition: " + fieldName); } - JsonPath path = fieldSpec.getType() == FieldType.PATH ? JsonPath.compile(fieldSpec.getExpr()) : null; - Pair pair = new Pair<>(fieldSpec.getType(), path); + FlattenExpr path = null; + if (fieldSpec.getType() == FieldType.PATH) { + path = new FlattenExpr(JsonPath.compile(fieldSpec.getExpr())); + } else if (fieldSpec.getType() == FieldType.JQ) { + try { + path = new FlattenExpr(JsonQuery.compile(fieldSpec.getExpr())); + } + catch (JsonQueryException e) { + throw new IllegalArgumentException("Unable to compile JQ expression: " + fieldSpec.getExpr()); + } + } + Pair pair = new Pair<>(fieldSpec.getType(), path); map.put(fieldName, pair); } return map; } - private void discoverFields(Map map, Map document) + private void discoverFields(Map map, JsonNode document) { - for (Map.Entry e : document.entrySet()) { + for (Iterator> it = document.fields(); it.hasNext(); ) { + Map.Entry e = it.next(); String field = e.getKey(); if (!map.containsKey(field)) { - Object val = e.getValue(); - if (val == null) { + JsonNode val = e.getValue(); + if (val.isNull()) { continue; } - if (val instanceof Map) { + if (val.isObject()) { continue; } - if (val instanceof List) { - if (!isFlatList((List) val)) { + if (val.isArray()) { + if (!isFlatList(val)) { continue; } } - val = valueConversionFunction(val); - map.put(field, val); + map.put(field, valueConversionFunction(val)); } } } - private Object valueConversionFunction(Object val) + private Object valueConversionFunction(JsonNode val) { - if (val instanceof Integer) { - return Long.valueOf((Integer) val); + if (val == null) { + return null; } - if (val instanceof BigInteger) { - return Double.valueOf(((BigInteger) val).doubleValue()); + if (val.isInt() || val.isLong()) { + return val.asLong(); } - if (val instanceof String) { - return charsetFix((String) val); + if (val.isNumber()) { + return val.asDouble(); } - if (val instanceof List) { + if (val.isTextual()) { + return charsetFix(val.asText()); + } + + if (val.isArray()) { List newList = new ArrayList<>(); - for (Object entry : ((List) val)) { + for (Iterator it = val.iterator(); it.hasNext(); ) { + JsonNode entry = it.next(); newList.add(valueConversionFunction(entry)); } return newList; } - if (val instanceof Map) { + if (val.isObject()) { Map newMap = new LinkedHashMap<>(); - Map valMap = (Map) val; - for (Map.Entry entry : valMap.entrySet()) { + for (Iterator> it = val.fields(); it.hasNext(); ) { + Map.Entry entry = it.next(); newMap.put(entry.getKey(), valueConversionFunction(entry.getValue())); } return newMap; @@ -210,10 +226,10 @@ private String charsetFix(String s) } } - private boolean isFlatList(List list) + private boolean isFlatList(JsonNode list) { - for (Object obj : list) { - if ((obj instanceof Map) || (obj instanceof List)) { + for (JsonNode obj : list) { + if (obj.isObject() || obj.isArray()) { return false; } } @@ -233,7 +249,12 @@ public enum FieldType /** * A PATH field uses a JsonPath expression to retrieve the field value */ - PATH; + PATH, + + /** + * A JQ field uses a JsonQuery expression to retrieve the field value + */ + JQ; } /** diff --git a/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java b/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java index 671cf83acc3d..af9c2b703b66 100644 --- a/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java +++ b/java-util/src/test/java/io/druid/java/util/common/parsers/JSONPathParserTest.java @@ -106,6 +106,11 @@ public void testNestingWithFieldDiscovery() fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.ROOT, "INVALID_ROOT", "INVALID_ROOT_EXPR")); fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "INVALID_PATH", "INVALID_PATH_EXPR")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-nested-foo.bar1", ".foo.bar1")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-nested-foo.bar2", ".foo.bar2")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-heybarx0", ".hey[0].barx")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-met-array", ".met.a")); + final Parser jsonParser = new JSONPathParser(fields, true, null); final Map jsonMap = jsonParser.parse(nestedJson); @@ -118,11 +123,11 @@ public void testNestingWithFieldDiscovery() Assert.assertEquals("2999", jsonMap.get("timestamp")); Assert.assertEquals("Hello world!", jsonMap.get("foo.bar1")); - List testListConvert = (List)jsonMap.get("testListConvert"); + List testListConvert = (List) jsonMap.get("testListConvert"); Assert.assertEquals(1.23456789E21, testListConvert.get(0)); Assert.assertEquals("foo?", testListConvert.get(1)); - List testListConvert2 = (List)jsonMap.get("testListConvert2"); + List testListConvert2 = (List) jsonMap.get("testListConvert2"); Assert.assertEquals(1.23456789E21, testListConvert2.get(0)); Assert.assertEquals("foo?", testListConvert2.get(1)); Assert.assertEquals(1.23456789E21, ((List) testListConvert2.get(2)).get(0)); @@ -139,6 +144,11 @@ public void testNestingWithFieldDiscovery() Assert.assertEquals("asdf", jsonMap.get("heybarx0")); Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("met-array")); + Assert.assertEquals("aaa", jsonMap.get("jq-nested-foo.bar1")); + Assert.assertEquals("bbb", jsonMap.get("jq-nested-foo.bar2")); + Assert.assertEquals("asdf", jsonMap.get("jq-heybarx0")); + Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("jq-met-array")); + // Fields that should not be discovered Assert.assertNull(jsonMap.get("hey")); Assert.assertNull(jsonMap.get("met")); @@ -159,6 +169,9 @@ public void testNestingNoDiscovery() fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "nested-foo.bar2", "$.foo.bar2")); fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "heybarx0", "$.hey[0].barx")); fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "met-array", "$.met.a")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-nested-foo.bar2", ".foo.bar2")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-heybarx0", ".hey[0].barx")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "jq-met-array", ".met.a")); final Parser jsonParser = new JSONPathParser(fields, false, null); final Map jsonMap = jsonParser.parse(nestedJson); @@ -171,6 +184,9 @@ public void testNestingNoDiscovery() Assert.assertEquals("bbb", jsonMap.get("nested-foo.bar2")); Assert.assertEquals("asdf", jsonMap.get("heybarx0")); Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("met-array")); + Assert.assertEquals("bbb", jsonMap.get("jq-nested-foo.bar2")); + Assert.assertEquals("asdf", jsonMap.get("jq-heybarx0")); + Assert.assertEquals(ImmutableList.of(7L, 8L, 9L), jsonMap.get("jq-met-array")); // Fields that should not be discovered Assert.assertNull(jsonMap.get("newmet")); @@ -198,6 +214,20 @@ public void testRejectDuplicates() final Map jsonMap = jsonParser.parse(nestedJson); } + @Test + public void testRejectDuplicates2() + { + List fields = new ArrayList<>(); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.PATH, "met-array", "$.met.a")); + fields.add(new JSONPathParser.FieldSpec(JSONPathParser.FieldType.JQ, "met-array", ".met.a")); + + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Cannot have duplicate field definition: met-array"); + + final Parser jsonParser = new JSONPathParser(fields, false, null); + final Map jsonMap = jsonParser.parse(nestedJson); + } + @Test public void testParseFail() { diff --git a/pom.xml b/pom.xml index 0c197cac5b7c..92bdca38a584 100644 --- a/pom.xml +++ b/pom.xml @@ -629,6 +629,11 @@ json-path 2.1.0 + + net.thisptr + jackson-jq + 0.0.7 + org.slf4j slf4j-api