From 70152eb653c2ea2e09428d602af68e55e63af290 Mon Sep 17 00:00:00 2001 From: Kenrick Yap <14yapkc1@gmail.com> Date: Fri, 3 Jan 2025 10:43:05 -0800 Subject: [PATCH 01/87] added implementation Signed-off-by: Kenrick Yap <14yapkc1@gmail.com> --- .../org/opensearch/sql/expression/DSL.java | 4 ++ .../function/BuiltinFunctionName.java | 3 ++ .../function/BuiltinFunctionRepository.java | 2 + .../sql/expression/json/JsonFunctions.java | 46 +++++++++++++++++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 3 ++ ppl/src/main/antlr/OpenSearchPPLParser.g4 | 1 + 6 files changed, 59 insertions(+) create mode 100644 core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 44ecc2bc867..fc97f35d117 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -683,6 +683,10 @@ public static FunctionExpression notLike(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.NOT_LIKE, expressions); } + public static FunctionExpression jsonValid(Expression... expressions){ + return compile(FunctionProperties.None, BuiltinFunctionName.JSON_VALID, expressions); + } + public static Aggregator avg(Expression... expressions) { return aggregate(BuiltinFunctionName.AVG, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index f8e9cf7c5f7..43fdbf2eb75 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -204,6 +204,9 @@ public enum BuiltinFunctionName { TRIM(FunctionName.of("trim")), UPPER(FunctionName.of("upper")), + /** Json Functions. */ + JSON_VALID(FunctionName.of("json_valid")), + /** NULL Test. */ IS_NULL(FunctionName.of("is null")), IS_NOT_NULL(FunctionName.of("is not null")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java index 79ea58b8608..72d637fd2ba 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java @@ -28,6 +28,7 @@ import org.opensearch.sql.expression.datetime.DateTimeFunctions; import org.opensearch.sql.expression.datetime.IntervalClause; import org.opensearch.sql.expression.ip.IPFunctions; +import org.opensearch.sql.expression.json.JsonFunctions; import org.opensearch.sql.expression.operator.arthmetic.ArithmeticFunctions; import org.opensearch.sql.expression.operator.arthmetic.MathematicalFunctions; import org.opensearch.sql.expression.operator.convert.TypeCastOperators; @@ -83,6 +84,7 @@ public static synchronized BuiltinFunctionRepository getInstance() { SystemFunctions.register(instance); OpenSearchFunctions.register(instance); IPFunctions.register(instance); + JsonFunctions.register(instance); } return instance; } diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java new file mode 100644 index 00000000000..e4ba7af4998 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -0,0 +1,46 @@ +package org.opensearch.sql.expression.json; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.expression.function.DefaultFunctionResolver; + +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.expression.function.FunctionDSL.define; +import static org.opensearch.sql.expression.function.FunctionDSL.impl; +import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; + +@UtilityClass +public class JsonFunctions { + public void register(BuiltinFunctionRepository repository) { + + repository.register(jsonValid()); + } + + private DefaultFunctionResolver jsonValid() { + return define( + BuiltinFunctionName.JSON_VALID.getName(), + impl(nullMissingHandling(JsonFunctions::isValidJson), BOOLEAN, STRING)); + } + + /** + * Checks if given JSON string can be parsed as valid JSON. + * + * @param jsonExprValue JSON string (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329"). + * @return true if the string can be parsed as valid JSON, else false. + */ + private ExprValue isValidJson(ExprValue jsonExprValue) { + ObjectMapper objectMapper = new ObjectMapper(); + try { + objectMapper.readTree(jsonExprValue.stringValue()); + return ExprValueUtils.LITERAL_TRUE; + } catch (Exception e) { + return ExprValueUtils.LITERAL_FALSE; + } + } +} diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 053ec530db4..5b6b9e41b89 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -332,6 +332,9 @@ ISNULL: 'ISNULL'; ISNOTNULL: 'ISNOTNULL'; CIDRMATCH: 'CIDRMATCH'; +// JSON FUNCTIONS +JSON_VALID: 'JSON_VALID'; + // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; NULLIF: 'NULLIF'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 27f7e4014ba..999c5d9c87c 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -662,6 +662,7 @@ conditionFunctionName | ISNULL | ISNOTNULL | CIDRMATCH + | JSON_VALID ; // flow control function return non-boolean value From 76c3995ddd8765ce26672dd4948abf3f0270ba4b Mon Sep 17 00:00:00 2001 From: Kenrick Yap <14yapkc1@gmail.com> Date: Mon, 6 Jan 2025 15:14:25 -0800 Subject: [PATCH 02/87] added doctest, integ-tests, and unit tests Signed-off-by: Kenrick Yap <14yapkc1@gmail.com> --- .../expression/json/JsonFunctionsTest.java | 59 +++++++++++++++++ docs/category.json | 1 + docs/user/dql/metadata.rst | 3 +- docs/user/ppl/functions/json.rst | 34 ++++++++++ doctest/test_data/json_test.json | 5 ++ doctest/test_docs.py | 4 +- .../sql/legacy/SQLIntegTestCase.java | 8 ++- .../org/opensearch/sql/legacy/TestUtils.java | 5 ++ .../opensearch/sql/legacy/TestsConstants.java | 1 + .../opensearch/sql/ppl/JsonFunctionIT.java | 65 +++++++++++++++++++ .../json_test_index_mappping.json | 12 ++++ integ-test/src/test/resources/json_test.json | 10 +++ 12 files changed, 204 insertions(+), 3 deletions(-) create mode 100644 core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java create mode 100644 docs/user/ppl/functions/json.rst create mode 100644 doctest/test_data/json_test.json create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java create mode 100644 integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json create mode 100644 integ-test/src/test/resources/json_test.json diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java new file mode 100644 index 00000000000..ee817dc71a8 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.json; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.env.Environment; + +@ExtendWith(MockitoExtension.class) +public class JsonFunctionsTest { + + private static final ExprValue JsonObject = ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}"); + private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]"); + private static final ExprValue JsonScalarString = ExprValueUtils.stringValue("\"abc\""); + private static final ExprValue JsonEmptyString = ExprValueUtils.stringValue(""); + private static final ExprValue JsonInvalidObject = ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"); + private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc"); + + @Mock private Environment env; + + @Test + public void json_valid_invalid_json_string() { + assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); + assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); + } + + @Test + public void json_valid_valid_json_string() { + assertEquals(LITERAL_TRUE, JsonObject); + assertEquals(LITERAL_TRUE, JsonArray); + assertEquals(LITERAL_TRUE, JsonScalarString); + assertEquals(LITERAL_TRUE, JsonEmptyString); + } + + private ExprValue execute(ExprValue jsonString) { + final String fieldName = "json_string"; + FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); + + when(DSL.ref(fieldName, STRING).valueOf(env)).thenReturn(jsonString); + + return exp.valueOf(env); + } +} diff --git a/docs/category.json b/docs/category.json index 32f56cfb467..efbb57d6e60 100644 --- a/docs/category.json +++ b/docs/category.json @@ -34,6 +34,7 @@ "user/ppl/functions/datetime.rst", "user/ppl/functions/expressions.rst", "user/ppl/functions/ip.rst", + "user/ppl/functions/json.rst", "user/ppl/functions/math.rst", "user/ppl/functions/relevance.rst", "user/ppl/functions/string.rst" diff --git a/docs/user/dql/metadata.rst b/docs/user/dql/metadata.rst index aba4eb0c75f..b059c0cded9 100644 --- a/docs/user/dql/metadata.rst +++ b/docs/user/dql/metadata.rst @@ -35,7 +35,7 @@ Example 1: Show All Indices Information SQL query:: os> SHOW TABLES LIKE '%' - fetched rows / total rows = 10/10 + fetched rows / total rows = 11/11 +----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | |----------------+-------------+-----------------+------------+---------+----------+------------+-----------+---------------------------+----------------| @@ -44,6 +44,7 @@ SQL query:: | docTestCluster | null | accounts | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | apache | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | books | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | json_test | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | nested | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | nyc_taxi | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | people | BASE TABLE | null | null | null | null | null | null | diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst new file mode 100644 index 00000000000..8f986ace6b9 --- /dev/null +++ b/docs/user/ppl/functions/json.rst @@ -0,0 +1,34 @@ +==================== +IP Address Functions +==================== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 1 + +JSON_VALID +---------- + +Description +>>>>>>>>>>> + +Usage: `json_valid(json_string)` checks if `json_string` is a valid STRING string. + +Argument type: STRING + +Return type: BOOLEAN + +Example:: + + > source=json_test | where json_valid(json_string) | fields test_name, json_string + fetched rows / total rows = 4/4 + +--------------------+--------------------+ + | test_name | json_string | + |--------------------|--------------------| + | json object | {"a":"1","b":"2"} | + | json array | [1, 2, 3, 4] | + | json scalar string | [1, 2, 3, 4] | + | json empty string | [1, 2, 3, 4] | + +--------------------+--------------------+ diff --git a/doctest/test_data/json_test.json b/doctest/test_data/json_test.json new file mode 100644 index 00000000000..2da491675e5 --- /dev/null +++ b/doctest/test_data/json_test.json @@ -0,0 +1,5 @@ +{"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} +{"test_name":"json array", "json_string":"[1, 2, 3, 4]"} +{"test_name":"json scalar string", "json_string":"\"abc\""} +{"test_name":"json empty string","json_string":""} +{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} diff --git a/doctest/test_docs.py b/doctest/test_docs.py index 1d46766c6d6..906bbd65b54 100644 --- a/doctest/test_docs.py +++ b/doctest/test_docs.py @@ -30,6 +30,7 @@ NESTED = "nested" DATASOURCES = ".ql-datasources" WEBLOGS = "weblogs" +JSON_TEST = "json_test" class DocTestConnection(OpenSearchConnection): @@ -123,6 +124,7 @@ def set_up_test_indices(test): load_file("nested_objects.json", index_name=NESTED) load_file("datasources.json", index_name=DATASOURCES) load_file("weblogs.json", index_name=WEBLOGS) + load_file("json_test.json", index_name=JSON_TEST) def load_file(filename, index_name): @@ -151,7 +153,7 @@ def set_up(test): def tear_down(test): # drop leftover tables after each test - test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI, BOOKS, APACHE, WILDCARD, NESTED, WEBLOGS], ignore_unavailable=True) + test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI, BOOKS, APACHE, WILDCARD, NESTED, WEBLOGS, JSON_TEST], ignore_unavailable=True) docsuite = partial(doctest.DocFileSuite, diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 1728be74e6c..d4f72137364 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -22,6 +22,7 @@ import static org.opensearch.sql.legacy.TestUtils.getGameOfThronesIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getGeopointIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getJoinTypeIndexMapping; +import static org.opensearch.sql.legacy.TestUtils.getJsonTestIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getLocationIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getMappingFile; import static org.opensearch.sql.legacy.TestUtils.getNestedSimpleIndexMapping; @@ -745,7 +746,12 @@ public enum Index { TestsConstants.TEST_INDEX_GEOPOINT, "dates", getGeopointIndexMapping(), - "src/test/resources/geopoints.json"); + "src/test/resources/geopoints.json"), + JSON_TEST( + TestsConstants.TEST_INDEX_JSON_TEST, + "json", + getJsonTestIndexMapping(), + "src/test/resources/json_test.json"); private final String name; private final String type; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 195dda0cbdd..610ad1366a9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -250,6 +250,11 @@ public static String getGeopointIndexMapping() { return getMappingFile(mappingFile); } + public static String getJsonTestIndexMapping() { + String mappingFile = "json_test_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static void loadBulk(Client client, String jsonPath, String defaultIndex) throws Exception { System.out.println(String.format("Loading file %s into opensearch cluster", jsonPath)); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 1e336f544e9..387054ac7e5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -58,6 +58,7 @@ public class TestsConstants { public static final String TEST_INDEX_MULTI_NESTED_TYPE = TEST_INDEX + "_multi_nested"; public static final String TEST_INDEX_NESTED_WITH_NULLS = TEST_INDEX + "_nested_with_nulls"; public static final String TEST_INDEX_GEOPOINT = TEST_INDEX + "_geopoint"; + public static final String TEST_INDEX_JSON_TEST = TEST_INDEX + "_json_test"; public static final String DATASOURCES = ".ql-datasources"; public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java new file mode 100644 index 00000000000..62e7868b41e --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import org.json.JSONObject; + +import javax.json.Json; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +public class JsonFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.JSON_TEST); + } + + @Test + public void test_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST + ) + ); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string") + ); + } + + @Test + public void test_not_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where not json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST + ) + ); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json invalid object") + ); + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json new file mode 100644 index 00000000000..b825254b111 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json @@ -0,0 +1,12 @@ +{ + "mappings": { + "properties": { + "test_name": { + "type": "text" + }, + "json_string": { + "type": "text" + } + } + } +} diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json new file mode 100644 index 00000000000..e198eb7c430 --- /dev/null +++ b/integ-test/src/test/resources/json_test.json @@ -0,0 +1,10 @@ +{"index":{"_id":"1"}} +{"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} +{"index":{"_id":"2"}} +{"test_name":"json array", "json_string":"[1, 2, 3, 4]"} +{"index":{"_id":"3"}} +{"test_name":"json scalar string", "json_string":"\"abc\""} +{"index":{"_id":"4"}} +{"test_name":"json empty string","json_string":""} +{"index":{"_id":"5"}} +{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} From ce2c551351eaf58f3055cd280f54a7bb3f0ac95a Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Mon, 6 Jan 2025 15:27:23 -0800 Subject: [PATCH 03/87] addressed PR comments Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/expression/json/JsonFunctions.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index e4ba7af4998..f0745af4b7d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.expression.json; import com.fasterxml.jackson.databind.ObjectMapper; From ad1bde30c80a9eaf5305af1bb22c8d776a2cefbe Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Mon, 6 Jan 2025 17:03:25 -0800 Subject: [PATCH 04/87] fixed unit tests Signed-off-by: Kenrick Yap --- .../sql/expression/json/JsonFunctions.java | 1 - .../expression/json/JsonFunctionsTest.java | 20 +++++-------------- docs/user/ppl/functions/json.rst | 19 +++++++++--------- 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index f0745af4b7d..7ef90cf73c7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -23,7 +23,6 @@ @UtilityClass public class JsonFunctions { public void register(BuiltinFunctionRepository repository) { - repository.register(jsonValid()); } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index ee817dc71a8..56b63631677 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -13,18 +13,14 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.expression.DSL; -import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.env.Environment; @ExtendWith(MockitoExtension.class) public class JsonFunctionsTest { - private static final ExprValue JsonObject = ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}"); private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]"); private static final ExprValue JsonScalarString = ExprValueUtils.stringValue("\"abc\""); @@ -32,8 +28,6 @@ public class JsonFunctionsTest { private static final ExprValue JsonInvalidObject = ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"); private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc"); - @Mock private Environment env; - @Test public void json_valid_invalid_json_string() { assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); @@ -42,18 +36,14 @@ public void json_valid_invalid_json_string() { @Test public void json_valid_valid_json_string() { - assertEquals(LITERAL_TRUE, JsonObject); - assertEquals(LITERAL_TRUE, JsonArray); - assertEquals(LITERAL_TRUE, JsonScalarString); - assertEquals(LITERAL_TRUE, JsonEmptyString); + assertEquals(LITERAL_TRUE, execute(JsonObject)); + assertEquals(LITERAL_TRUE, execute(JsonArray)); + assertEquals(LITERAL_TRUE, execute(JsonScalarString)); + assertEquals(LITERAL_TRUE, execute(JsonEmptyString)); } private ExprValue execute(ExprValue jsonString) { - final String fieldName = "json_string"; FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); - - when(DSL.ref(fieldName, STRING).valueOf(env)).thenReturn(jsonString); - - return exp.valueOf(env); + return exp.valueOf(); } } diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 8f986ace6b9..bf5bd46b7a3 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -22,13 +22,14 @@ Return type: BOOLEAN Example:: - > source=json_test | where json_valid(json_string) | fields test_name, json_string + > source=json_test | eval is_valid = json_valid(json_string) | fields test_name, json_string, is_valid fetched rows / total rows = 4/4 - +--------------------+--------------------+ - | test_name | json_string | - |--------------------|--------------------| - | json object | {"a":"1","b":"2"} | - | json array | [1, 2, 3, 4] | - | json scalar string | [1, 2, 3, 4] | - | json empty string | [1, 2, 3, 4] | - +--------------------+--------------------+ + +---------------------+------------------------------+----------+ + | test_name | json_string | is_valid | + |---------------------|------------------------------|----------| + | json object | {"a":"1","b":"2"} | True | + | json array | [1, 2, 3, 4] | True | + | json scalar string | "abc" | True | + | json empty string | | True | + | json invalid object | {"invalid":"json", "string"} | True | + +---------------------+------------------------------+----------+ From ccf47a277b420ccf2ec9ab94d23c05c33b0408d1 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 09:20:59 -0800 Subject: [PATCH 05/87] addressed pr comments Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/expression/json/JsonFunctions.java | 3 ++- .../org/opensearch/sql/expression/json/JsonFunctionsTest.java | 4 ++-- .../resources/indexDefinitions/json_test_index_mappping.json | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 7ef90cf73c7..6cdc14807ed 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,6 +5,7 @@ package org.opensearch.sql.expression.json; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.experimental.UtilityClass; import org.opensearch.sql.data.model.ExprValue; @@ -43,7 +44,7 @@ private ExprValue isValidJson(ExprValue jsonExprValue) { try { objectMapper.readTree(jsonExprValue.stringValue()); return ExprValueUtils.LITERAL_TRUE; - } catch (Exception e) { + } catch (JsonProcessingException e) { return ExprValueUtils.LITERAL_FALSE; } } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 56b63631677..5f2f51bcb9f 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -29,13 +29,13 @@ public class JsonFunctionsTest { private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc"); @Test - public void json_valid_invalid_json_string() { + public void json_valid_returns_false() { assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); } @Test - public void json_valid_valid_json_string() { + public void json_valid_returns_true() { assertEquals(LITERAL_TRUE, execute(JsonObject)); assertEquals(LITERAL_TRUE, execute(JsonArray)); assertEquals(LITERAL_TRUE, execute(JsonScalarString)); diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json index b825254b111..fb97836d5ec 100644 --- a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json +++ b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json @@ -2,7 +2,7 @@ "mappings": { "properties": { "test_name": { - "type": "text" + "type": "keyword" }, "json_string": { "type": "text" From acc76a027792e7f66d6a6dd2c3c8771080fb01ff Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 13:40:28 -0800 Subject: [PATCH 06/87] addressed PR comments Signed-off-by: Kenrick Yap --- .../sql/expression/json/JsonFunctions.java | 19 ++------------ .../org/opensearch/sql/utils/JsonUtils.java | 26 +++++++++++++++++++ .../datetime/DateTimeFunctionTest.java | 4 +-- .../sql/expression/datetime/ExtractTest.java | 3 +++ .../sql/expression/datetime/YearweekTest.java | 5 ++-- 5 files changed, 36 insertions(+), 21 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/utils/JsonUtils.java diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 6cdc14807ed..4e8a3bba695 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -14,6 +14,7 @@ import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; +import org.opensearch.sql.utils.JsonUtils; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; @@ -30,22 +31,6 @@ public void register(BuiltinFunctionRepository repository) { private DefaultFunctionResolver jsonValid() { return define( BuiltinFunctionName.JSON_VALID.getName(), - impl(nullMissingHandling(JsonFunctions::isValidJson), BOOLEAN, STRING)); - } - - /** - * Checks if given JSON string can be parsed as valid JSON. - * - * @param jsonExprValue JSON string (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329"). - * @return true if the string can be parsed as valid JSON, else false. - */ - private ExprValue isValidJson(ExprValue jsonExprValue) { - ObjectMapper objectMapper = new ObjectMapper(); - try { - objectMapper.readTree(jsonExprValue.stringValue()); - return ExprValueUtils.LITERAL_TRUE; - } catch (JsonProcessingException e) { - return ExprValueUtils.LITERAL_FALSE; - } + impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); } } diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java new file mode 100644 index 00000000000..393f83256a5 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -0,0 +1,26 @@ +package org.opensearch.sql.utils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; + +@UtilityClass +public class JsonUtils { + /** + * Checks if given JSON string can be parsed as valid JSON. + * + * @param jsonExprValue JSON string (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329"). + * @return true if the string can be parsed as valid JSON, else false. + */ + public static ExprValue isValidJson(ExprValue jsonExprValue) { + ObjectMapper objectMapper = new ObjectMapper(); + try { + objectMapper.readTree(jsonExprValue.stringValue()); + return ExprValueUtils.LITERAL_TRUE; + } catch (JsonProcessingException e) { + return ExprValueUtils.LITERAL_FALSE; + } + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index c820c97196f..dbc75c45ae9 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.stream.Stream; import lombok.AllArgsConstructor; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -1228,9 +1229,8 @@ public void testWeekFormats( expectedInteger); } - // subtracting 1 as a temporary fix for year 2024. - // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { assertAll( () -> diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 02d50d0b597..fd87c144daa 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -11,6 +11,8 @@ import java.time.LocalDate; import java.util.stream.Stream; + +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -91,6 +93,7 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { } @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { datePartWithTimeArgQuery( "DAY", timeInput, LocalDate.now(functionProperties.getQueryStartClock()).getDayOfMonth()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index 47225ac601e..87e2f05d85b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -14,6 +14,8 @@ import java.time.LocalDate; import java.util.stream.Stream; + +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -97,9 +99,8 @@ public void testYearweekWithoutMode() { assertEquals(eval(expression), eval(expressionWithoutMode)); } - // subtracting 1 as a temporary fix for year 2024. - // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int week = LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR) - 1; int year = LocalDate.now(functionProperties.getQueryStartClock()).getYear(); From 519c6f21aaa8c45dd22d525c414f220db64a704d Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 13:40:47 -0800 Subject: [PATCH 07/87] removed unused dependencies Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/expression/json/JsonFunctions.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 4e8a3bba695..dc7bde32c23 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,12 +5,7 @@ package org.opensearch.sql.expression.json; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import lombok.experimental.UtilityClass; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; From 2e319fea1e26bcb1b324413cc2ebc9983eb11453 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 14:28:45 -0800 Subject: [PATCH 08/87] linting Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/expression/DSL.java | 2 +- .../sql/expression/json/JsonFunctions.java | 28 +++--- .../org/opensearch/sql/utils/JsonUtils.java | 28 +++--- .../datetime/DateTimeFunctionTest.java | 3 +- .../sql/expression/datetime/ExtractTest.java | 4 +- .../sql/expression/datetime/YearweekTest.java | 4 +- .../expression/json/JsonFunctionsTest.java | 48 +++++----- .../opensearch/sql/ppl/JsonFunctionIT.java | 89 ++++++++----------- 8 files changed, 98 insertions(+), 108 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index fc97f35d117..dc819c8163d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -683,7 +683,7 @@ public static FunctionExpression notLike(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.NOT_LIKE, expressions); } - public static FunctionExpression jsonValid(Expression... expressions){ + public static FunctionExpression jsonValid(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.JSON_VALID, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index dc7bde32c23..49541e5d591 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,27 +5,27 @@ package org.opensearch.sql.expression.json; -import lombok.experimental.UtilityClass; -import org.opensearch.sql.expression.function.BuiltinFunctionName; -import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.expression.function.DefaultFunctionResolver; -import org.opensearch.sql.utils.JsonUtils; - import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; +import lombok.experimental.UtilityClass; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.expression.function.DefaultFunctionResolver; +import org.opensearch.sql.utils.JsonUtils; + @UtilityClass public class JsonFunctions { - public void register(BuiltinFunctionRepository repository) { - repository.register(jsonValid()); - } + public void register(BuiltinFunctionRepository repository) { + repository.register(jsonValid()); + } - private DefaultFunctionResolver jsonValid() { - return define( - BuiltinFunctionName.JSON_VALID.getName(), - impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); - } + private DefaultFunctionResolver jsonValid() { + return define( + BuiltinFunctionName.JSON_VALID.getName(), + impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); + } } diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 393f83256a5..af02d26ef75 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -8,19 +8,19 @@ @UtilityClass public class JsonUtils { - /** - * Checks if given JSON string can be parsed as valid JSON. - * - * @param jsonExprValue JSON string (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329"). - * @return true if the string can be parsed as valid JSON, else false. - */ - public static ExprValue isValidJson(ExprValue jsonExprValue) { - ObjectMapper objectMapper = new ObjectMapper(); - try { - objectMapper.readTree(jsonExprValue.stringValue()); - return ExprValueUtils.LITERAL_TRUE; - } catch (JsonProcessingException e) { - return ExprValueUtils.LITERAL_FALSE; - } + /** + * Checks if given JSON string can be parsed as valid JSON. + * + * @param jsonExprValue JSON string (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329"). + * @return true if the string can be parsed as valid JSON, else false. + */ + public static ExprValue isValidJson(ExprValue jsonExprValue) { + ObjectMapper objectMapper = new ObjectMapper(); + try { + objectMapper.readTree(jsonExprValue.stringValue()); + return ExprValueUtils.LITERAL_TRUE; + } catch (JsonProcessingException e) { + return ExprValueUtils.LITERAL_FALSE; } + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index dbc75c45ae9..042491b2fc5 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1230,7 +1230,8 @@ public void testWeekFormats( } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { assertAll( () -> diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index fd87c144daa..feb809af57f 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -11,7 +11,6 @@ import java.time.LocalDate; import java.util.stream.Stream; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -93,7 +92,8 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { datePartWithTimeArgQuery( "DAY", timeInput, LocalDate.now(functionProperties.getQueryStartClock()).getDayOfMonth()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index 87e2f05d85b..9e7b8c04c15 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -14,7 +14,6 @@ import java.time.LocalDate; import java.util.stream.Stream; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -100,7 +99,8 @@ public void testYearweekWithoutMode() { } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int week = LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR) - 1; int year = LocalDate.now(functionProperties.getQueryStartClock()).getYear(); diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 5f2f51bcb9f..2e8ece2817c 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -6,10 +6,8 @@ package org.opensearch.sql.expression.json; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.when; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -21,29 +19,31 @@ @ExtendWith(MockitoExtension.class) public class JsonFunctionsTest { - private static final ExprValue JsonObject = ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}"); - private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]"); - private static final ExprValue JsonScalarString = ExprValueUtils.stringValue("\"abc\""); - private static final ExprValue JsonEmptyString = ExprValueUtils.stringValue(""); - private static final ExprValue JsonInvalidObject = ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"); - private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc"); + private static final ExprValue JsonObject = + ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}"); + private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]"); + private static final ExprValue JsonScalarString = ExprValueUtils.stringValue("\"abc\""); + private static final ExprValue JsonEmptyString = ExprValueUtils.stringValue(""); + private static final ExprValue JsonInvalidObject = + ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"); + private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc"); - @Test - public void json_valid_returns_false() { - assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); - assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); - } + @Test + public void json_valid_returns_false() { + assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); + assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); + } - @Test - public void json_valid_returns_true() { - assertEquals(LITERAL_TRUE, execute(JsonObject)); - assertEquals(LITERAL_TRUE, execute(JsonArray)); - assertEquals(LITERAL_TRUE, execute(JsonScalarString)); - assertEquals(LITERAL_TRUE, execute(JsonEmptyString)); - } + @Test + public void json_valid_returns_true() { + assertEquals(LITERAL_TRUE, execute(JsonObject)); + assertEquals(LITERAL_TRUE, execute(JsonArray)); + assertEquals(LITERAL_TRUE, execute(JsonScalarString)); + assertEquals(LITERAL_TRUE, execute(JsonEmptyString)); + } - private ExprValue execute(ExprValue jsonString) { - FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); - return exp.valueOf(); - } + private ExprValue execute(ExprValue jsonString) { + FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); + return exp.valueOf(); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java index 62e7868b41e..f02750147d6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -5,61 +5,50 @@ package org.opensearch.sql.ppl; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import org.json.JSONObject; - -import javax.json.Json; - import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; -public class JsonFunctionIT extends PPLIntegTestCase { - @Override - public void init() throws IOException { - loadIndex(Index.JSON_TEST); - } - - @Test - public void test_json_valid() throws IOException { - JSONObject result; - - result = - executeQuery( - String.format( - "source=%s | where json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST - ) - ); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json object"), - rows("json array"), - rows("json scalar string"), - rows("json empty string") - ); - } - - @Test - public void test_not_json_valid() throws IOException { - JSONObject result; +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; - result = - executeQuery( - String.format( - "source=%s | where not json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST - ) - ); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json invalid object") - ); - } +public class JsonFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.JSON_TEST); + } + + @Test + public void test_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string")); + } + + @Test + public void test_not_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where not json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows(result, rows("json invalid object")); + } } From ee0820df0b2edb2e44753822831305e493e8c43a Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Wed, 8 Jan 2025 14:16:31 -0800 Subject: [PATCH 09/87] addressed pr comment and rolling back disabled test case Signed-off-by: Kenrick Yap --- core/src/main/java/org/opensearch/sql/utils/JsonUtils.java | 2 +- .../sql/expression/datetime/DateTimeFunctionTest.java | 4 ++-- .../org/opensearch/sql/expression/datetime/ExtractTest.java | 2 -- .../org/opensearch/sql/expression/datetime/YearweekTest.java | 4 ++-- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index af02d26ef75..d7f37b41970 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -11,7 +11,7 @@ public class JsonUtils { /** * Checks if given JSON string can be parsed as valid JSON. * - * @param jsonExprValue JSON string (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329"). + * @param jsonExprValue JSON string (e.g. "{\"hello\": \"world\"}"). * @return true if the string can be parsed as valid JSON, else false. */ public static ExprValue isValidJson(ExprValue jsonExprValue) { diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 042491b2fc5..4b287319ba5 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1229,9 +1229,9 @@ public void testWeekFormats( expectedInteger); } + // subtracting 1 as a temporary fix for year 2024. + // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { assertAll( () -> diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index feb809af57f..a9ae6274b1a 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -92,8 +92,6 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { } @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { datePartWithTimeArgQuery( "DAY", timeInput, LocalDate.now(functionProperties.getQueryStartClock()).getDayOfMonth()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index 9e7b8c04c15..e273ee9ed40 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -98,9 +98,9 @@ public void testYearweekWithoutMode() { assertEquals(eval(expression), eval(expressionWithoutMode)); } + // subtracting 1 as a temporary fix for year 2024. + // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int week = LocalDate.now(functionProperties.getQueryStartClock()).get(ALIGNED_WEEK_OF_YEAR) - 1; int year = LocalDate.now(functionProperties.getQueryStartClock()).getYear(); From 3407d4a4d92473b6c2be0154c325b49b6f26751a Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 10:58:21 -0800 Subject: [PATCH 10/87] removed disabled import Signed-off-by: Kenrick Yap --- .../opensearch/sql/expression/datetime/DateTimeFunctionTest.java | 1 - .../java/org/opensearch/sql/expression/datetime/ExtractTest.java | 1 - .../org/opensearch/sql/expression/datetime/YearweekTest.java | 1 - 3 files changed, 3 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 4b287319ba5..c820c97196f 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -27,7 +27,6 @@ import java.util.List; import java.util.stream.Stream; import lombok.AllArgsConstructor; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 1dfa3908e6d..d7635de6102 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -11,7 +11,6 @@ import java.time.LocalDate; import java.util.stream.Stream; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index e273ee9ed40..47225ac601e 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -14,7 +14,6 @@ import java.time.LocalDate; import java.util.stream.Stream; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; From 7ef6cc95f9dec0515a1316efafb3b48181433358 Mon Sep 17 00:00:00 2001 From: kenrickyap <121634635+kenrickyap@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:01:48 -0800 Subject: [PATCH 11/87] Update docs/user/ppl/functions/json.rst Co-authored-by: Andrew Carbonetto Signed-off-by: kenrickyap <121634635+kenrickyap@users.noreply.github.com> --- docs/user/ppl/functions/json.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index bf5bd46b7a3..a69101300bb 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -31,5 +31,5 @@ Example:: | json array | [1, 2, 3, 4] | True | | json scalar string | "abc" | True | | json empty string | | True | - | json invalid object | {"invalid":"json", "string"} | True | + | json invalid object | {"invalid":"json", "string"} | False | +---------------------+------------------------------+----------+ From e5e90acc98d2b71209fc3d450eb91899d00cb058 Mon Sep 17 00:00:00 2001 From: kenrickyap <121634635+kenrickyap@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:02:10 -0800 Subject: [PATCH 12/87] Update integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java Co-authored-by: Andrew Carbonetto Signed-off-by: kenrickyap <121634635+kenrickyap@users.noreply.github.com> --- .../src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java index f02750147d6..501ef9448e6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -15,7 +15,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; -public class JsonFunctionIT extends PPLIntegTestCase { +public class JsonFunctionsIT extends PPLIntegTestCase { @Override public void init() throws IOException { loadIndex(Index.JSON_TEST); From 2187a5ac0f053b6815c1900459e4ba295ca7d060 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 13:05:12 -0800 Subject: [PATCH 13/87] nit Signed-off-by: Kenrick Yap --- ...json_test_index_mappping.json => json_test_index_mapping.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename integ-test/src/test/resources/indexDefinitions/{json_test_index_mappping.json => json_test_index_mapping.json} (100%) diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mapping.json similarity index 100% rename from integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json rename to integ-test/src/test/resources/indexDefinitions/json_test_index_mapping.json From 3512b335980f8f8fcbc6cfa10accb1b6d622f877 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 13:16:09 -0800 Subject: [PATCH 14/87] fixed integ test Signed-off-by: Kenrick Yap --- .../sql/ppl/{JsonFunctionIT.java => JsonFunctionsIT.java} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename integ-test/src/test/java/org/opensearch/sql/ppl/{JsonFunctionIT.java => JsonFunctionsIT.java} (100%) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java similarity index 100% rename from integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java rename to integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java From 9fea6060ba12f95f174041439422787b59cdc3d4 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 13:29:03 -0800 Subject: [PATCH 15/87] change text type to keyword Signed-off-by: Kenrick Yap --- .../resources/indexDefinitions/json_test_index_mapping.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mapping.json index fb97836d5ec..86bd0c6e948 100644 --- a/integ-test/src/test/resources/indexDefinitions/json_test_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/json_test_index_mapping.json @@ -5,7 +5,7 @@ "type": "keyword" }, "json_string": { - "type": "text" + "type": "keyword" } } } From fbc54bc59f1e54b89d14dff96b2578772f2647c5 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Fri, 10 Jan 2025 10:37:08 -0800 Subject: [PATCH 16/87] addressed PR comments Signed-off-by: Kenrick Yap --- .../expression/json/JsonFunctionsTest.java | 3 +++ docs/user/ppl/functions/json.rst | 21 ++++++++++--------- doctest/test_data/json_test.json | 1 + .../opensearch/sql/ppl/JsonFunctionsIT.java | 1 + integ-test/src/test/resources/json_test.json | 2 ++ 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 2e8ece2817c..e374841e7f6 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -19,6 +19,8 @@ @ExtendWith(MockitoExtension.class) public class JsonFunctionsTest { + private static final ExprValue JsonNestedObject = + ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"); private static final ExprValue JsonObject = ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}"); private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]"); @@ -36,6 +38,7 @@ public void json_valid_returns_false() { @Test public void json_valid_returns_true() { + assertEquals(LITERAL_TRUE, execute(JsonNestedObject)); assertEquals(LITERAL_TRUE, execute(JsonObject)); assertEquals(LITERAL_TRUE, execute(JsonArray)); assertEquals(LITERAL_TRUE, execute(JsonScalarString)); diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index a69101300bb..ce3d1a4c76d 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -1,5 +1,5 @@ ==================== -IP Address Functions +JSON Functions ==================== .. rubric:: Table of contents @@ -24,12 +24,13 @@ Example:: > source=json_test | eval is_valid = json_valid(json_string) | fields test_name, json_string, is_valid fetched rows / total rows = 4/4 - +---------------------+------------------------------+----------+ - | test_name | json_string | is_valid | - |---------------------|------------------------------|----------| - | json object | {"a":"1","b":"2"} | True | - | json array | [1, 2, 3, 4] | True | - | json scalar string | "abc" | True | - | json empty string | | True | - | json invalid object | {"invalid":"json", "string"} | False | - +---------------------+------------------------------+----------+ + +---------------------+---------------------------------+----------+ + | test_name | json_string | is_valid | + |---------------------|---------------------------------|----------| + | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | True | + | json object | {"a":"1","b":"2"} | True | + | json array | [1, 2, 3, 4] | True | + | json scalar string | "abc" | True | + | json empty string | | True | + | json invalid object | {"invalid":"json", "string"} | False | + +---------------------+---------------------------------+----------+ diff --git a/doctest/test_data/json_test.json b/doctest/test_data/json_test.json index 2da491675e5..7494fc4aa91 100644 --- a/doctest/test_data/json_test.json +++ b/doctest/test_data/json_test.json @@ -1,3 +1,4 @@ +{"test_name":"json nested object", "json_string":"{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"} {"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} {"test_name":"json array", "json_string":"[1, 2, 3, 4]"} {"test_name":"json scalar string", "json_string":"\"abc\""} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 501ef9448e6..f852a97d48c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -33,6 +33,7 @@ public void test_json_valid() throws IOException { verifySchema(result, schema("test_name", null, "string")); verifyDataRows( result, + rows("json nested object"), rows("json object"), rows("json array"), rows("json scalar string"), diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json index e198eb7c430..badb4f4f6e8 100644 --- a/integ-test/src/test/resources/json_test.json +++ b/integ-test/src/test/resources/json_test.json @@ -1,3 +1,5 @@ +{"index":{"_id":"0"}} +{"test_name":"json nested object", "json_string":"{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"} {"index":{"_id":"1"}} {"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} {"index":{"_id":"2"}} From 31ad2a4286c7a1d4876eb9e42eb4a41777b9c826 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Fri, 10 Jan 2025 16:07:52 -0800 Subject: [PATCH 17/87] fix doc-test Signed-off-by: Kenrick Yap --- docs/user/ppl/functions/json.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index ce3d1a4c76d..74c173be139 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -23,7 +23,7 @@ Return type: BOOLEAN Example:: > source=json_test | eval is_valid = json_valid(json_string) | fields test_name, json_string, is_valid - fetched rows / total rows = 4/4 + fetched rows / total rows = 6/6 +---------------------+---------------------------------+----------+ | test_name | json_string | is_valid | |---------------------|---------------------------------|----------| From 2b2a8f3a4143ead02a96373b58e0482388066c21 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 14 Jan 2025 09:10:06 -0800 Subject: [PATCH 18/87] added null test Signed-off-by: Kenrick Yap --- core/src/main/java/org/opensearch/sql/utils/JsonUtils.java | 5 +++++ .../test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java | 2 +- integ-test/src/test/resources/json_test.json | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index d7f37b41970..37c374286e2 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -16,6 +16,11 @@ public class JsonUtils { */ public static ExprValue isValidJson(ExprValue jsonExprValue) { ObjectMapper objectMapper = new ObjectMapper(); + + if (jsonExprValue.isNull() || jsonExprValue.isMissing()) { + return ExprValueUtils.LITERAL_FALSE; + } + try { objectMapper.readTree(jsonExprValue.stringValue()); return ExprValueUtils.LITERAL_TRUE; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index f852a97d48c..9e5ac041fb6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -50,6 +50,6 @@ public void test_not_json_valid() throws IOException { "source=%s | where not json_valid(json_string) | fields test_name", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string")); - verifyDataRows(result, rows("json invalid object")); + verifyDataRows(result, rows("json invalid object"), rows("json null")); } } diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json index badb4f4f6e8..e393bfeb8e4 100644 --- a/integ-test/src/test/resources/json_test.json +++ b/integ-test/src/test/resources/json_test.json @@ -10,3 +10,5 @@ {"test_name":"json empty string","json_string":""} {"index":{"_id":"5"}} {"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} +{"index":{"_id":"6"}} +{"test_name":"json null", "json_string":null} From 1913bfe755abf3bfe3607bcb8f3e421efd1d9a17 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Tue, 14 Jan 2025 22:24:54 -0800 Subject: [PATCH 19/87] SQL: adding error case unit tests for json_valid Signed-off-by: Andrew Carbonetto --- .../sql/expression/json/JsonFunctionsTest.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index e374841e7f6..c889a237d41 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -6,7 +6,9 @@ package org.opensearch.sql.expression.json; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import org.junit.jupiter.api.Test; @@ -14,6 +16,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.FunctionExpression; @@ -34,6 +37,15 @@ public class JsonFunctionsTest { public void json_valid_returns_false() { assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); + + // caught by nullMissingHandling and returns null + assertEquals(LITERAL_NULL, execute(LITERAL_NULL)); + } + + @Test + public void json_valid_throws_ExpressionEvaluationException() { + assertThrows( + ExpressionEvaluationException.class, () -> execute(ExprValueUtils.booleanValue(true))); } @Test From 67d979d2f61f0e6b94d394014b82950bceb00f57 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 09:12:53 -0800 Subject: [PATCH 20/87] json_valid: null and missing should return false Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/json/JsonFunctions.java | 4 +--- .../opensearch/sql/expression/json/JsonFunctionsTest.java | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 49541e5d591..acc0c4c064a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -9,7 +9,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; -import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import lombok.experimental.UtilityClass; import org.opensearch.sql.expression.function.BuiltinFunctionName; @@ -25,7 +24,6 @@ public void register(BuiltinFunctionRepository repository) { private DefaultFunctionResolver jsonValid() { return define( - BuiltinFunctionName.JSON_VALID.getName(), - impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); + BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index c889a237d41..3228a565c2e 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; @@ -37,9 +38,8 @@ public class JsonFunctionsTest { public void json_valid_returns_false() { assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); - - // caught by nullMissingHandling and returns null - assertEquals(LITERAL_NULL, execute(LITERAL_NULL)); + assertEquals(LITERAL_FALSE, execute(LITERAL_NULL)); + assertEquals(LITERAL_FALSE, execute(LITERAL_MISSING)); } @Test From aa6b72354d45fd9a423cd0837b7dee213e7df8f8 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 10:49:03 -0800 Subject: [PATCH 21/87] PPL: Add json and cast to json functions Signed-off-by: Andrew Carbonetto --- .../opensearch/sql/ast/expression/Cast.java | 2 + .../org/opensearch/sql/expression/DSL.java | 8 ++ .../function/BuiltinFunctionName.java | 6 +- .../operator/convert/TypeCastOperators.java | 9 ++ .../org/opensearch/sql/utils/JsonUtils.java | 68 ++++++++- .../expression/json/JsonFunctionsTest.java | 132 ++++++++++++++++++ .../convert/TypeCastOperatorTest.java | 103 ++++++++++++++ docs/user/ppl/functions/json.rst | 25 ++++ .../opensearch/sql/ppl/JsonFunctionsIT.java | 28 ++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 8 ++ 11 files changed, 386 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java index 541dbedeadd..854ba0ed696 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java @@ -13,6 +13,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_FLOAT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_INT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_IP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_JSON; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_LONG; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_SHORT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_STRING; @@ -56,6 +57,7 @@ public class Cast extends UnresolvedExpression { .put("timestamp", CAST_TO_TIMESTAMP.getName()) .put("datetime", CAST_TO_DATETIME.getName()) .put("ip", CAST_TO_IP.getName()) + .put("json", CAST_TO_JSON.getName()) .build(); /** The source expression cast from. */ diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index dc819c8163d..a50e3d14706 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -843,6 +843,10 @@ public static FunctionExpression castIp(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_IP, value); } + public static FunctionExpression castJson(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_JSON, value); + } + public static FunctionExpression typeof(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.TYPEOF, value); } @@ -973,6 +977,10 @@ public static FunctionExpression utc_timestamp( return compile(functionProperties, BuiltinFunctionName.UTC_TIMESTAMP, args); } + public static FunctionExpression json_function(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); + } + @SuppressWarnings("unchecked") private static T compile( FunctionProperties functionProperties, BuiltinFunctionName bfn, Expression... args) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 43fdbf2eb75..ef9a872974d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -235,6 +235,7 @@ public enum BuiltinFunctionName { CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")), CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")), CAST_TO_IP(FunctionName.of("cast_to_ip")), + CAST_TO_JSON(FunctionName.of("cast_to_json")), TYPEOF(FunctionName.of("typeof")), /** Relevance Function. */ @@ -259,7 +260,10 @@ public enum BuiltinFunctionName { MULTIMATCH(FunctionName.of("multimatch")), MULTIMATCHQUERY(FunctionName.of("multimatchquery")), WILDCARDQUERY(FunctionName.of("wildcardquery")), - WILDCARD_QUERY(FunctionName.of("wildcard_query")); + WILDCARD_QUERY(FunctionName.of("wildcard_query")), + + /* Json Functions. */ + JSON(FunctionName.of("json")); private final FunctionName name; diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index b388f7d89ab..d6518e47d91 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -17,10 +17,12 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.impl; import static org.opensearch.sql.expression.function.FunctionDSL.implWithProperties; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandlingWithProperties; +import static org.opensearch.sql.utils.JsonUtils.castJson; import java.util.Arrays; import java.util.stream.Collectors; @@ -57,6 +59,7 @@ public static void register(BuiltinFunctionRepository repository) { repository.register(castToDouble()); repository.register(castToBoolean()); repository.register(castToIp()); + repository.register(castToJson()); repository.register(castToDate()); repository.register(castToTime()); repository.register(castToTimestamp()); @@ -183,6 +186,12 @@ private static DefaultFunctionResolver castToIp() { impl(nullMissingHandling((v) -> v), IP, IP)); } + private static DefaultFunctionResolver castToJson() { + return FunctionDSL.define( + BuiltinFunctionName.CAST_TO_JSON.getName(), + impl(nullMissingHandling((v) -> castJson(v.stringValue())), UNDEFINED, STRING)); + } + private static DefaultFunctionResolver castToDate() { return FunctionDSL.define( BuiltinFunctionName.CAST_TO_DATE.getName(), diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 37c374286e2..9835db91931 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -1,10 +1,24 @@ package org.opensearch.sql.utils; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; + import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.SemanticCheckException; @UtilityClass public class JsonUtils { @@ -23,9 +37,57 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { try { objectMapper.readTree(jsonExprValue.stringValue()); - return ExprValueUtils.LITERAL_TRUE; + return LITERAL_TRUE; } catch (JsonProcessingException e) { - return ExprValueUtils.LITERAL_FALSE; + return LITERAL_FALSE; + } + } + + /** Converts a JSON encoded string to an Expression object. */ + public static ExprValue castJson(String json) { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode; + try { + jsonNode = objectMapper.readTree(json); + } catch (JsonProcessingException e) { + final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; + throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); + } + + return processJsonNode(jsonNode); + } + + private static ExprValue processJsonNode(JsonNode jsonNode) { + if (jsonNode.isFloatingPointNumber()) { + return new ExprDoubleValue(jsonNode.asDouble()); + } + if (jsonNode.isIntegralNumber()) { + return new ExprIntegerValue(jsonNode.asLong()); + } + if (jsonNode.isBoolean()) { + return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE; } + if (jsonNode.isTextual()) { + return new ExprStringValue(jsonNode.asText()); + } + if (jsonNode.isArray()) { + List elements = new LinkedList<>(); + for (var iter = jsonNode.iterator(); iter.hasNext(); ) { + jsonNode = iter.next(); + elements.add(processJsonNode(jsonNode)); + } + return new ExprCollectionValue(elements); + } + if (jsonNode.isObject()) { + Map values = new LinkedHashMap<>(); + for (var iter = jsonNode.fields(); iter.hasNext(); ) { + Map.Entry entry = iter.next(); + values.put(entry.getKey(), processJsonNode(entry.getValue())); + } + return ExprTupleValue.fromExprValueMap(values); + } + + // in all other cases, return null + return LITERAL_NULL; } } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 3228a565c2e..e0fcb9ab81a 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -7,17 +7,60 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.FunctionExpression; @@ -61,4 +104,93 @@ private ExprValue execute(ExprValue jsonString) { FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); return exp.valueOf(); } + + @Test + void json_returnsJsonObject() { + FunctionExpression exp; + + // Setup + final String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.json_function(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void json_returnsJsonArray() { + FunctionExpression exp; + + // Setup + final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.json_function(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void json_returnsScalar() { + assertEquals( + new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); + + assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + } + + @Test + void json_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index fd579dfb470..27bd806c110 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -8,6 +8,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.BYTE; import static org.opensearch.sql.data.type.ExprCoreType.DATE; @@ -21,12 +23,16 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprByteValue; +import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprFloatValue; @@ -39,6 +45,7 @@ import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; @@ -389,4 +396,100 @@ void castToIp() { assertEquals(IP, exp.type()); assertTrue(exp.valueOf().isMissing()); } + + @Test + void castJson_returnsJsonObject() { + FunctionExpression exp; + + // Setup + String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.castJson(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void castJson_returnsJsonArray() { + FunctionExpression exp; + + // Setup + String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.castJson(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void castJson_returnsScalar() { + String scalarStringJson = "\"foobar\""; + assertEquals( + new ExprStringValue("foobar"), DSL.castJson(DSL.literal(scalarStringJson)).valueOf()); + + String scalarNumberJson = "1234"; + assertEquals(new ExprIntegerValue(1234), DSL.castJson(DSL.literal(scalarNumberJson)).valueOf()); + + String scalarBooleanJson = "true"; + assertEquals(LITERAL_TRUE, DSL.castJson(DSL.literal(scalarBooleanJson)).valueOf()); + + String scalarNullJson = "null"; + assertEquals(LITERAL_NULL, DSL.castJson(DSL.literal(scalarNullJson)).valueOf()); + + String empty = ""; + assertEquals(LITERAL_NULL, DSL.castJson(DSL.literal(empty)).valueOf()); + + String emptyObject = "{}"; + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), + DSL.castJson(DSL.literal(emptyObject)).valueOf()); + } + + @Test + void castJson_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 74c173be139..82c0da23a67 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -34,3 +34,28 @@ Example:: | json empty string | | True | | json invalid object | {"invalid":"json", "string"} | False | +---------------------+---------------------------------+----------+ + +JSON +---------- + +Description +>>>>>>>>>>> + +Usage: `json(value)` Evaluates whether a string can be parsed as a json-encoded string and casted as an expression. Returns the JSON value if valid, null otherwise. + +Argument type: STRING + +Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY + +Example:: + + > source=json_test | where json_valid(json_string) | eval json=json(json_string) | fields test_name, json_string, json + fetched rows / total rows = 4/4 + +---------------------+------------------------------+---------------+ + | test_name | json_string | json | + |---------------------|------------------------------|---------------| + | json object | {"a":"1","b":"2"} | {a:"1",b:"2"} | + | json array | [1, 2, 3, 4] | [1,2,3,4] | + | json scalar string | "abc" | "abc" | + | json empty string | | null | + +---------------------+------------------------------+---------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 9e5ac041fb6..33c54ff60bf 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -52,4 +52,32 @@ public void test_not_json_valid() throws IOException { verifySchema(result, schema("test_name", null, "string")); verifyDataRows(result, rows("json invalid object"), rows("json null")); } + + @Test + public void test_cast_json() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval json=cast(json_string to json) | fields json", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string")); + } + + @Test + public void test_json() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | eval json=json(json_string) | fields json", TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows(result, rows("json invalid object")); + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index ea7e060d9cd..9d6707a8722 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -143,6 +143,7 @@ FLOAT: 'FLOAT'; STRING: 'STRING'; BOOLEAN: 'BOOLEAN'; IP: 'IP'; +JSON: 'JSON'; // SPECIAL CHARACTERS AND OPERATORS PIPE: '|'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 999c5d9c87c..74b05dc28b6 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -409,6 +409,7 @@ convertedDataType | typeName = STRING | typeName = BOOLEAN | typeName = IP + | typeName = JSON ; evalFunctionName @@ -419,6 +420,7 @@ evalFunctionName | flowControlFunctionName | systemFunctionName | positionFunctionName + | jsonFunctionName ; functionArgs @@ -700,6 +702,10 @@ positionFunctionName : POSITION ; +jsonFunctionName + : JSON + ; + // operators comparisonOperator : EQUAL @@ -963,4 +969,6 @@ keywordsCanBeId | SPARKLINE | C | DC + // JSON + | JSON ; From 4c992359992b557c481ac8ef4f6df3880525e53c Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 10:56:50 -0800 Subject: [PATCH 22/87] PPL: Update json cast for review Signed-off-by: Andrew Carbonetto --- .../sql/expression/function/BuiltinFunctionName.java | 6 ++---- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 3 +-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index ef9a872974d..cd309a712d4 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -206,6 +206,7 @@ public enum BuiltinFunctionName { /** Json Functions. */ JSON_VALID(FunctionName.of("json_valid")), + JSON(FunctionName.of("json")), /** NULL Test. */ IS_NULL(FunctionName.of("is null")), @@ -260,10 +261,7 @@ public enum BuiltinFunctionName { MULTIMATCH(FunctionName.of("multimatch")), MULTIMATCHQUERY(FunctionName.of("multimatchquery")), WILDCARDQUERY(FunctionName.of("wildcardquery")), - WILDCARD_QUERY(FunctionName.of("wildcard_query")), - - /* Json Functions. */ - JSON(FunctionName.of("json")); + WILDCARD_QUERY(FunctionName.of("wildcard_query")); private final FunctionName name; diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 9d6707a8722..31a668be00d 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -143,7 +143,6 @@ FLOAT: 'FLOAT'; STRING: 'STRING'; BOOLEAN: 'BOOLEAN'; IP: 'IP'; -JSON: 'JSON'; // SPECIAL CHARACTERS AND OPERATORS PIPE: '|'; @@ -335,6 +334,7 @@ CIDRMATCH: 'CIDRMATCH'; // JSON FUNCTIONS JSON_VALID: 'JSON_VALID'; +JSON: 'JSON'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 74b05dc28b6..5d1f29614de 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -870,6 +870,7 @@ keywordsCanBeId | mathematicalFunctionName | positionFunctionName | conditionFunctionName + | jsonFunctionName // commands | SEARCH | DESCRIBE @@ -969,6 +970,4 @@ keywordsCanBeId | SPARKLINE | C | DC - // JSON - | JSON ; From 9ccde7f3da431f223cf9d596bb6f61b606b1df44 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 17:17:51 -0800 Subject: [PATCH 23/87] Fix testes Signed-off-by: Andrew Carbonetto --- .../sql/expression/json/JsonFunctions.java | 8 ++++++ .../operator/convert/TypeCastOperators.java | 4 +-- .../org/opensearch/sql/utils/JsonUtils.java | 4 +-- .../opensearch/sql/ppl/JsonFunctionsIT.java | 25 ++++++++++++------- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index acc0c4c064a..f5bcc505a6e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; @@ -20,10 +21,17 @@ public class JsonFunctions { public void register(BuiltinFunctionRepository repository) { repository.register(jsonValid()); + repository.register(jsonFunction()); } private DefaultFunctionResolver jsonValid() { return define( BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); } + + private DefaultFunctionResolver jsonFunction() { + return define( + BuiltinFunctionName.JSON.getName(), + impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index d6518e47d91..cfd570ab89a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -22,7 +22,6 @@ import static org.opensearch.sql.expression.function.FunctionDSL.implWithProperties; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandlingWithProperties; -import static org.opensearch.sql.utils.JsonUtils.castJson; import java.util.Arrays; import java.util.stream.Collectors; @@ -44,6 +43,7 @@ import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; import org.opensearch.sql.expression.function.FunctionDSL; +import org.opensearch.sql.utils.JsonUtils; @UtilityClass public class TypeCastOperators { @@ -189,7 +189,7 @@ private static DefaultFunctionResolver castToIp() { private static DefaultFunctionResolver castToJson() { return FunctionDSL.define( BuiltinFunctionName.CAST_TO_JSON.getName(), - impl(nullMissingHandling((v) -> castJson(v.stringValue())), UNDEFINED, STRING)); + impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); } private static DefaultFunctionResolver castToDate() { diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 9835db91931..0579eda933e 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -44,11 +44,11 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { } /** Converts a JSON encoded string to an Expression object. */ - public static ExprValue castJson(String json) { + public static ExprValue castJson(ExprValue json) { ObjectMapper objectMapper = new ObjectMapper(); JsonNode jsonNode; try { - jsonNode = objectMapper.readTree(json); + jsonNode = objectMapper.readTree(json.stringValue()); } catch (JsonProcessingException e) { final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 33c54ff60bf..465443c096b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,6 +12,8 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.util.List; +import java.util.Map; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -58,15 +60,15 @@ public void test_cast_json() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | eval json=cast(json_string to json) | fields json", + "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json) | fields test_name, casted", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, - rows("json object"), - rows("json array"), - rows("json scalar string"), - rows("json empty string")); + rows("json object", Map.of("a", "1", "b", "2")), + rows("json array", List.of(1,2,3,4)), + rows("json scalar string", "abc"), + rows("json empty string", null)); } @Test @@ -76,8 +78,13 @@ public void test_json() throws IOException { result = executeQuery( String.format( - "source=%s | eval json=json(json_string) | fields json", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows(result, rows("json invalid object")); + "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields test_name, casted", TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + verifyDataRows( + result, + rows("json object", Map.of("a", "1", "b", "2")), + rows("json array", List.of(1,2,3,4)), + rows("json scalar string", "abc"), + rows("json empty string", null)); } } From 4306bf374bce01404ca8067ec6966dd9c0ac2b89 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Thu, 9 Jan 2025 09:33:34 -0800 Subject: [PATCH 24/87] spotless Signed-off-by: Andrew Carbonetto --- .../java/org/opensearch/sql/ppl/JsonFunctionsIT.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 465443c096b..015333cdeae 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -60,13 +60,14 @@ public void test_cast_json() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json) | fields test_name, casted", + "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json)" + + " | fields test_name, casted", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1,2,3,4)), + rows("json array", List.of(1, 2, 3, 4)), rows("json scalar string", "abc"), rows("json empty string", null)); } @@ -78,12 +79,14 @@ public void test_json() throws IOException { result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields test_name, casted", TEST_INDEX_JSON_TEST)); + "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields" + + " test_name, casted", + TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1,2,3,4)), + rows("json array", List.of(1, 2, 3, 4)), rows("json scalar string", "abc"), rows("json empty string", null)); } From 613137bdc229169641f9dd483990f91f9709b48d Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 13 Jan 2025 16:13:01 -0800 Subject: [PATCH 25/87] Fix tests Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/ppl/JsonFunctionsIT.java | 13 +++++++++---- integ-test/src/test/resources/json_test.json | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 015333cdeae..a636273107b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,8 +12,10 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -66,8 +68,9 @@ public void test_cast_json() throws IOException { verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, - rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1, 2, 3, 4)), + rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), + rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json empty string", null)); } @@ -83,10 +86,12 @@ public void test_json() throws IOException { + " test_name, casted", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + JSONObject firstRow = new JSONObject(Map.of("c", 2)); verifyDataRows( result, - rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1, 2, 3, 4)), + rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), + rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json empty string", null)); } diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json index e393bfeb8e4..dae01ea4cef 100644 --- a/integ-test/src/test/resources/json_test.json +++ b/integ-test/src/test/resources/json_test.json @@ -1,5 +1,5 @@ {"index":{"_id":"0"}} -{"test_name":"json nested object", "json_string":"{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"} +{"test_name":"json nested object", "json_string":"{\"a\":\"1\", \"b\": {\"c\": \"3\"}, \"d\": [1, 2, 3]}"} {"index":{"_id":"1"}} {"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} {"index":{"_id":"2"}} From ab288728fa9263ce1a79d1baff7ea25ac85966c3 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 13 Jan 2025 16:20:17 -0800 Subject: [PATCH 26/87] SPOTLESS Signed-off-by: Andrew Carbonetto --- .../expression/json/JsonFunctionsTest.java | 176 +++++++++--------- .../opensearch/sql/ppl/JsonFunctionsIT.java | 9 +- 2 files changed, 94 insertions(+), 91 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index e0fcb9ab81a..a1f18f62da7 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -105,92 +105,92 @@ private ExprValue execute(ExprValue jsonString) { return exp.valueOf(); } - @Test - void json_returnsJsonObject() { - FunctionExpression exp; - - // Setup - final String objectJson = - "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " - + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; - - LinkedHashMap objectMap = new LinkedHashMap<>(); - objectMap.put("foo", new ExprStringValue("foo")); - objectMap.put("fuzz", ExprBooleanValue.of(true)); - objectMap.put("bar", new ExprLongValue(1234)); - objectMap.put("bar2", new ExprDoubleValue(12.34)); - objectMap.put("baz", ExprNullValue.of()); - objectMap.put( - "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); - objectMap.put( - "arr", - new ExprCollectionValue( - List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); - ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); - - // exercise - exp = DSL.json_function(DSL.literal(objectJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprTupleValue); - assertEquals(expectedTupleExpr, value); - } - - @Test - void json_returnsJsonArray() { - FunctionExpression exp; - - // Setup - final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; - ExprValue expectedArrayExpr = - new ExprCollectionValue( - List.of( - new ExprStringValue("foo"), - new ExprStringValue("fuzz"), - LITERAL_TRUE, - new ExprStringValue("bar"), - new ExprIntegerValue(1234), - new ExprDoubleValue(12.34), - LITERAL_NULL)); - - // exercise - exp = DSL.json_function(DSL.literal(arrayJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprCollectionValue); - assertEquals(expectedArrayExpr, value); - } - - @Test - void json_returnsScalar() { - assertEquals( - new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); - - assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); - - assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); - - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); - } - - @Test - void json_returnsSemanticCheckException() { - // invalid type - assertThrows( - SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); - - // missing bracket - assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); - - // mnissing quote - assertThrows( - SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); - } + @Test + void json_returnsJsonObject() { + FunctionExpression exp; + + // Setup + final String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.json_function(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void json_returnsJsonArray() { + FunctionExpression exp; + + // Setup + final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.json_function(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void json_returnsScalar() { + assertEquals( + new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); + + assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + } + + @Test + void json_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index a636273107b..eecf7fb338a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,7 +12,6 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.json.JSONArray; @@ -68,7 +67,9 @@ public void test_cast_json() throws IOException { verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, - rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows( + "json nested object", + new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), @@ -89,7 +90,9 @@ public void test_json() throws IOException { JSONObject firstRow = new JSONObject(Map.of("c", 2)); verifyDataRows( result, - rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows( + "json nested object", + new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), From 3ec16e0a4fe7c856b5990ce7d4d32c74e9010609 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 10:33:11 -0800 Subject: [PATCH 27/87] Clean up for merge Signed-off-by: Andrew Carbonetto --- .../sql/expression/json/JsonFunctions.java | 1 + .../org/opensearch/sql/utils/JsonUtils.java | 1 + .../expression/json/JsonFunctionsTest.java | 30 ------------------- 3 files changed, 2 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index f5bcc505a6e..75f134aa4e9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -10,6 +10,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; +import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import lombok.experimental.UtilityClass; import org.opensearch.sql.expression.function.BuiltinFunctionName; diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 0579eda933e..c5f7031b13e 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -18,6 +18,7 @@ import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.SemanticCheckException; @UtilityClass diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index a1f18f62da7..89ea57c2e4b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -7,24 +7,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -39,28 +27,10 @@ import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprBooleanValue; -import org.opensearch.sql.data.model.ExprCollectionValue; -import org.opensearch.sql.data.model.ExprDoubleValue; -import org.opensearch.sql.data.model.ExprIntegerValue; -import org.opensearch.sql.data.model.ExprLongValue; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprStringValue; -import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprBooleanValue; -import org.opensearch.sql.data.model.ExprCollectionValue; -import org.opensearch.sql.data.model.ExprDoubleValue; -import org.opensearch.sql.data.model.ExprIntegerValue; -import org.opensearch.sql.data.model.ExprLongValue; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprStringValue; -import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.FunctionExpression; From 6dbf37bb19fd3e49dc94a6c237ca940bfbe068b7 Mon Sep 17 00:00:00 2001 From: Kenrick Yap <14yapkc1@gmail.com> Date: Fri, 3 Jan 2025 10:43:05 -0800 Subject: [PATCH 28/87] added implementation Signed-off-by: Kenrick Yap <14yapkc1@gmail.com> --- .../opensearch/sql/expression/json/JsonFunctions.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index acc0c4c064a..56ea5f10c3f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,10 +5,20 @@ package org.opensearch.sql.expression.json; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.expression.function.DefaultFunctionResolver; + import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; +import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import lombok.experimental.UtilityClass; import org.opensearch.sql.expression.function.BuiltinFunctionName; From b8c6d68a4f340464c84c7c5afa2cb98b9b1c7537 Mon Sep 17 00:00:00 2001 From: Kenrick Yap <14yapkc1@gmail.com> Date: Mon, 6 Jan 2025 15:14:25 -0800 Subject: [PATCH 29/87] added doctest, integ-tests, and unit tests Signed-off-by: Kenrick Yap <14yapkc1@gmail.com> --- .../opensearch/sql/ppl/JsonFunctionIT.java | 65 +++++++++++++++++++ .../json_test_index_mappping.json | 12 ++++ 2 files changed, 77 insertions(+) create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java create mode 100644 integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java new file mode 100644 index 00000000000..62e7868b41e --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import org.json.JSONObject; + +import javax.json.Json; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +public class JsonFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.JSON_TEST); + } + + @Test + public void test_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST + ) + ); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string") + ); + } + + @Test + public void test_not_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where not json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST + ) + ); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json invalid object") + ); + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json new file mode 100644 index 00000000000..b825254b111 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json @@ -0,0 +1,12 @@ +{ + "mappings": { + "properties": { + "test_name": { + "type": "text" + }, + "json_string": { + "type": "text" + } + } + } +} From afb668c74bd1a2f140ec16c18ee2b38cf8fbf1ef Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 09:20:59 -0800 Subject: [PATCH 30/87] addressed pr comments Signed-off-by: Kenrick Yap --- .../resources/indexDefinitions/json_test_index_mappping.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json index b825254b111..fb97836d5ec 100644 --- a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json +++ b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json @@ -2,7 +2,7 @@ "mappings": { "properties": { "test_name": { - "type": "text" + "type": "keyword" }, "json_string": { "type": "text" From 54ef1835c6d9cc4766aa57c9bb05a017ae5fca6d Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 13:40:28 -0800 Subject: [PATCH 31/87] addressed PR comments Signed-off-by: Kenrick Yap --- .../sql/expression/datetime/DateTimeFunctionTest.java | 2 ++ .../org/opensearch/sql/expression/datetime/ExtractTest.java | 3 +++ .../org/opensearch/sql/expression/datetime/YearweekTest.java | 3 +++ 3 files changed, 8 insertions(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index ad15dadfb73..e8a4fc81fbc 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.stream.Stream; import lombok.AllArgsConstructor; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -1231,6 +1232,7 @@ public void testWeekFormats( } @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index d7635de6102..02645eca06e 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -11,6 +11,8 @@ import java.time.LocalDate; import java.util.stream.Stream; + +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -91,6 +93,7 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { } @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { LocalDate now = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index d944f7c85c3..4de5d3a341a 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -16,6 +16,8 @@ import java.time.LocalDate; import java.time.temporal.ChronoUnit; import java.util.stream.Stream; + +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -100,6 +102,7 @@ public void testYearweekWithoutMode() { } @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); From d84139489e107edb6c1a12cafc8872b5eb90af76 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 13:40:47 -0800 Subject: [PATCH 32/87] removed unused dependencies Signed-off-by: Kenrick Yap --- .../java/org/opensearch/sql/expression/json/JsonFunctions.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 56ea5f10c3f..05d5187a14f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -7,9 +7,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.experimental.UtilityClass; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; From 25fb527882104b9083e0cf0c614b24a1586336f4 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 14:28:45 -0800 Subject: [PATCH 33/87] linting Signed-off-by: Kenrick Yap --- .../sql/expression/json/JsonFunctions.java | 5 ++ .../datetime/DateTimeFunctionTest.java | 3 +- .../sql/expression/datetime/ExtractTest.java | 4 +- .../sql/expression/datetime/YearweekTest.java | 4 +- .../opensearch/sql/ppl/JsonFunctionIT.java | 89 ++++++++----------- 5 files changed, 50 insertions(+), 55 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 05d5187a14f..2f832629229 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -33,4 +33,9 @@ private DefaultFunctionResolver jsonValid() { return define( BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); } + private DefaultFunctionResolver jsonValid() { + return define( + BuiltinFunctionName.JSON_VALID.getName(), + impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index e8a4fc81fbc..eb5074f4f7c 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1232,7 +1232,8 @@ public void testWeekFormats( } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 02645eca06e..925c1f1b7c7 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -11,7 +11,6 @@ import java.time.LocalDate; import java.util.stream.Stream; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -93,7 +92,8 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { LocalDate now = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index 4de5d3a341a..daae8b1ff54 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -16,7 +16,6 @@ import java.time.LocalDate; import java.time.temporal.ChronoUnit; import java.util.stream.Stream; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -102,7 +101,8 @@ public void testYearweekWithoutMode() { } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java index 62e7868b41e..f02750147d6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -5,61 +5,50 @@ package org.opensearch.sql.ppl; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import org.json.JSONObject; - -import javax.json.Json; - import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; -public class JsonFunctionIT extends PPLIntegTestCase { - @Override - public void init() throws IOException { - loadIndex(Index.JSON_TEST); - } - - @Test - public void test_json_valid() throws IOException { - JSONObject result; - - result = - executeQuery( - String.format( - "source=%s | where json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST - ) - ); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json object"), - rows("json array"), - rows("json scalar string"), - rows("json empty string") - ); - } - - @Test - public void test_not_json_valid() throws IOException { - JSONObject result; +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; - result = - executeQuery( - String.format( - "source=%s | where not json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST - ) - ); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json invalid object") - ); - } +public class JsonFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.JSON_TEST); + } + + @Test + public void test_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string")); + } + + @Test + public void test_not_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where not json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows(result, rows("json invalid object")); + } } From 4a20d0870a58c8273f43f837a5382aaeff80fabc Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Wed, 8 Jan 2025 14:16:31 -0800 Subject: [PATCH 34/87] addressed pr comment and rolling back disabled test case Signed-off-by: Kenrick Yap --- .../sql/expression/datetime/DateTimeFunctionTest.java | 4 ++-- .../org/opensearch/sql/expression/datetime/ExtractTest.java | 2 -- .../org/opensearch/sql/expression/datetime/YearweekTest.java | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index eb5074f4f7c..ab391c6834c 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1231,9 +1231,9 @@ public void testWeekFormats( expectedInteger); } + // subtracting 1 as a temporary fix for year 2024. + // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 925c1f1b7c7..1dfa3908e6d 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -92,8 +92,6 @@ private void datePartWithTimeArgQuery(String part, String time, long expected) { } @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { LocalDate now = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index daae8b1ff54..266994c0461 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -100,9 +100,9 @@ public void testYearweekWithoutMode() { assertEquals(eval(expression), eval(expressionWithoutMode)); } + // subtracting 1 as a temporary fix for year 2024. + // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); From fdc4729de3b1c8310245cec8b469b41b96ec95d6 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 10:58:21 -0800 Subject: [PATCH 35/87] removed disabled import Signed-off-by: Kenrick Yap --- .../opensearch/sql/expression/datetime/DateTimeFunctionTest.java | 1 - .../java/org/opensearch/sql/expression/datetime/ExtractTest.java | 1 - .../org/opensearch/sql/expression/datetime/YearweekTest.java | 1 - 3 files changed, 3 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index ab391c6834c..115898e349e 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -29,7 +29,6 @@ import java.util.List; import java.util.stream.Stream; import lombok.AllArgsConstructor; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 1dfa3908e6d..d7635de6102 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -11,7 +11,6 @@ import java.time.LocalDate; import java.util.stream.Stream; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index 266994c0461..ee4df23be4e 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -16,7 +16,6 @@ import java.time.LocalDate; import java.time.temporal.ChronoUnit; import java.util.stream.Stream; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; From 707a0b9693b6293cf0261311b852ecb9f91f689f Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 13:05:12 -0800 Subject: [PATCH 36/87] nit Signed-off-by: Kenrick Yap --- .../indexDefinitions/json_test_index_mappping.json | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json deleted file mode 100644 index fb97836d5ec..00000000000 --- a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "mappings": { - "properties": { - "test_name": { - "type": "keyword" - }, - "json_string": { - "type": "text" - } - } - } -} From 4f28211701c7ceb7a974abc0dd377b0acc66f500 Mon Sep 17 00:00:00 2001 From: kenrickyap <121634635+kenrickyap@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:02:10 -0800 Subject: [PATCH 37/87] Update integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java Co-authored-by: Andrew Carbonetto Signed-off-by: kenrickyap <121634635+kenrickyap@users.noreply.github.com> --- .../src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java index f02750147d6..501ef9448e6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -15,7 +15,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; -public class JsonFunctionIT extends PPLIntegTestCase { +public class JsonFunctionsIT extends PPLIntegTestCase { @Override public void init() throws IOException { loadIndex(Index.JSON_TEST); From 9ec633599d14814284b368d9294263b3760213c9 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 13:16:09 -0800 Subject: [PATCH 38/87] fixed integ test Signed-off-by: Kenrick Yap --- .../opensearch/sql/ppl/JsonFunctionIT.java | 54 ------------------- 1 file changed, 54 deletions(-) delete mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java deleted file mode 100644 index 501ef9448e6..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.ppl; - -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; -import static org.opensearch.sql.util.MatcherUtils.rows; -import static org.opensearch.sql.util.MatcherUtils.schema; -import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; -import static org.opensearch.sql.util.MatcherUtils.verifySchema; - -import java.io.IOException; -import org.json.JSONObject; -import org.junit.jupiter.api.Test; - -public class JsonFunctionsIT extends PPLIntegTestCase { - @Override - public void init() throws IOException { - loadIndex(Index.JSON_TEST); - } - - @Test - public void test_json_valid() throws IOException { - JSONObject result; - - result = - executeQuery( - String.format( - "source=%s | where json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json object"), - rows("json array"), - rows("json scalar string"), - rows("json empty string")); - } - - @Test - public void test_not_json_valid() throws IOException { - JSONObject result; - - result = - executeQuery( - String.format( - "source=%s | where not json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows(result, rows("json invalid object")); - } -} From 3324e661953a855fc46dce474001796d4fdf33ac Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Tue, 14 Jan 2025 22:24:54 -0800 Subject: [PATCH 39/87] SQL: adding error case unit tests for json_valid Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/json/JsonFunctionsTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 3228a565c2e..557a2ab9e1e 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -46,6 +46,9 @@ public void json_valid_returns_false() { public void json_valid_throws_ExpressionEvaluationException() { assertThrows( ExpressionEvaluationException.class, () -> execute(ExprValueUtils.booleanValue(true))); + + // caught by nullMissingHandling and returns null + assertEquals(LITERAL_NULL, execute(LITERAL_NULL)); } @Test From 7123c350f0164a0dda45f3daa8aade84a9b28abf Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 09:12:53 -0800 Subject: [PATCH 40/87] json_valid: null and missing should return false Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/json/JsonFunctions.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 2f832629229..b0554fd2105 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -15,7 +15,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; -import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import lombok.experimental.UtilityClass; import org.opensearch.sql.expression.function.BuiltinFunctionName; @@ -33,9 +32,4 @@ private DefaultFunctionResolver jsonValid() { return define( BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); } - private DefaultFunctionResolver jsonValid() { - return define( - BuiltinFunctionName.JSON_VALID.getName(), - impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); - } } From dbca9915557246f4f4728f8b92fd5046efecec0d Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 10:49:03 -0800 Subject: [PATCH 41/87] PPL: Add json and cast to json functions Signed-off-by: Andrew Carbonetto --- .../opensearch/sql/ast/expression/Cast.java | 2 + .../org/opensearch/sql/expression/DSL.java | 8 ++ .../function/BuiltinFunctionName.java | 6 +- .../operator/convert/TypeCastOperators.java | 9 ++ .../org/opensearch/sql/utils/JsonUtils.java | 68 ++++++++- .../expression/json/JsonFunctionsTest.java | 132 ++++++++++++++++++ .../convert/TypeCastOperatorTest.java | 103 ++++++++++++++ docs/user/ppl/functions/json.rst | 25 ++++ .../opensearch/sql/ppl/JsonFunctionsIT.java | 28 ++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 8 ++ 11 files changed, 386 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java index 541dbedeadd..854ba0ed696 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/Cast.java @@ -13,6 +13,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_FLOAT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_INT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_IP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_JSON; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_LONG; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_SHORT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_STRING; @@ -56,6 +57,7 @@ public class Cast extends UnresolvedExpression { .put("timestamp", CAST_TO_TIMESTAMP.getName()) .put("datetime", CAST_TO_DATETIME.getName()) .put("ip", CAST_TO_IP.getName()) + .put("json", CAST_TO_JSON.getName()) .build(); /** The source expression cast from. */ diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index dc819c8163d..a50e3d14706 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -843,6 +843,10 @@ public static FunctionExpression castIp(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_IP, value); } + public static FunctionExpression castJson(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_JSON, value); + } + public static FunctionExpression typeof(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.TYPEOF, value); } @@ -973,6 +977,10 @@ public static FunctionExpression utc_timestamp( return compile(functionProperties, BuiltinFunctionName.UTC_TIMESTAMP, args); } + public static FunctionExpression json_function(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); + } + @SuppressWarnings("unchecked") private static T compile( FunctionProperties functionProperties, BuiltinFunctionName bfn, Expression... args) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 43fdbf2eb75..ef9a872974d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -235,6 +235,7 @@ public enum BuiltinFunctionName { CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")), CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")), CAST_TO_IP(FunctionName.of("cast_to_ip")), + CAST_TO_JSON(FunctionName.of("cast_to_json")), TYPEOF(FunctionName.of("typeof")), /** Relevance Function. */ @@ -259,7 +260,10 @@ public enum BuiltinFunctionName { MULTIMATCH(FunctionName.of("multimatch")), MULTIMATCHQUERY(FunctionName.of("multimatchquery")), WILDCARDQUERY(FunctionName.of("wildcardquery")), - WILDCARD_QUERY(FunctionName.of("wildcard_query")); + WILDCARD_QUERY(FunctionName.of("wildcard_query")), + + /* Json Functions. */ + JSON(FunctionName.of("json")); private final FunctionName name; diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index b388f7d89ab..d6518e47d91 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -17,10 +17,12 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.impl; import static org.opensearch.sql.expression.function.FunctionDSL.implWithProperties; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandlingWithProperties; +import static org.opensearch.sql.utils.JsonUtils.castJson; import java.util.Arrays; import java.util.stream.Collectors; @@ -57,6 +59,7 @@ public static void register(BuiltinFunctionRepository repository) { repository.register(castToDouble()); repository.register(castToBoolean()); repository.register(castToIp()); + repository.register(castToJson()); repository.register(castToDate()); repository.register(castToTime()); repository.register(castToTimestamp()); @@ -183,6 +186,12 @@ private static DefaultFunctionResolver castToIp() { impl(nullMissingHandling((v) -> v), IP, IP)); } + private static DefaultFunctionResolver castToJson() { + return FunctionDSL.define( + BuiltinFunctionName.CAST_TO_JSON.getName(), + impl(nullMissingHandling((v) -> castJson(v.stringValue())), UNDEFINED, STRING)); + } + private static DefaultFunctionResolver castToDate() { return FunctionDSL.define( BuiltinFunctionName.CAST_TO_DATE.getName(), diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 37c374286e2..9835db91931 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -1,10 +1,24 @@ package org.opensearch.sql.utils; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; + import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.SemanticCheckException; @UtilityClass public class JsonUtils { @@ -23,9 +37,57 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { try { objectMapper.readTree(jsonExprValue.stringValue()); - return ExprValueUtils.LITERAL_TRUE; + return LITERAL_TRUE; } catch (JsonProcessingException e) { - return ExprValueUtils.LITERAL_FALSE; + return LITERAL_FALSE; + } + } + + /** Converts a JSON encoded string to an Expression object. */ + public static ExprValue castJson(String json) { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode; + try { + jsonNode = objectMapper.readTree(json); + } catch (JsonProcessingException e) { + final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; + throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); + } + + return processJsonNode(jsonNode); + } + + private static ExprValue processJsonNode(JsonNode jsonNode) { + if (jsonNode.isFloatingPointNumber()) { + return new ExprDoubleValue(jsonNode.asDouble()); + } + if (jsonNode.isIntegralNumber()) { + return new ExprIntegerValue(jsonNode.asLong()); + } + if (jsonNode.isBoolean()) { + return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE; } + if (jsonNode.isTextual()) { + return new ExprStringValue(jsonNode.asText()); + } + if (jsonNode.isArray()) { + List elements = new LinkedList<>(); + for (var iter = jsonNode.iterator(); iter.hasNext(); ) { + jsonNode = iter.next(); + elements.add(processJsonNode(jsonNode)); + } + return new ExprCollectionValue(elements); + } + if (jsonNode.isObject()) { + Map values = new LinkedHashMap<>(); + for (var iter = jsonNode.fields(); iter.hasNext(); ) { + Map.Entry entry = iter.next(); + values.put(entry.getKey(), processJsonNode(entry.getValue())); + } + return ExprTupleValue.fromExprValueMap(values); + } + + // in all other cases, return null + return LITERAL_NULL; } } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 557a2ab9e1e..809d82941b2 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -7,17 +7,60 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprBooleanValue; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.FunctionExpression; @@ -64,4 +107,93 @@ private ExprValue execute(ExprValue jsonString) { FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); return exp.valueOf(); } + + @Test + void json_returnsJsonObject() { + FunctionExpression exp; + + // Setup + final String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.json_function(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void json_returnsJsonArray() { + FunctionExpression exp; + + // Setup + final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.json_function(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void json_returnsScalar() { + assertEquals( + new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); + + assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + } + + @Test + void json_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index fd579dfb470..27bd806c110 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -8,6 +8,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.BYTE; import static org.opensearch.sql.data.type.ExprCoreType.DATE; @@ -21,12 +23,16 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprByteValue; +import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprFloatValue; @@ -39,6 +45,7 @@ import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; +import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; @@ -389,4 +396,100 @@ void castToIp() { assertEquals(IP, exp.type()); assertTrue(exp.valueOf().isMissing()); } + + @Test + void castJson_returnsJsonObject() { + FunctionExpression exp; + + // Setup + String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.castJson(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void castJson_returnsJsonArray() { + FunctionExpression exp; + + // Setup + String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.castJson(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void castJson_returnsScalar() { + String scalarStringJson = "\"foobar\""; + assertEquals( + new ExprStringValue("foobar"), DSL.castJson(DSL.literal(scalarStringJson)).valueOf()); + + String scalarNumberJson = "1234"; + assertEquals(new ExprIntegerValue(1234), DSL.castJson(DSL.literal(scalarNumberJson)).valueOf()); + + String scalarBooleanJson = "true"; + assertEquals(LITERAL_TRUE, DSL.castJson(DSL.literal(scalarBooleanJson)).valueOf()); + + String scalarNullJson = "null"; + assertEquals(LITERAL_NULL, DSL.castJson(DSL.literal(scalarNullJson)).valueOf()); + + String empty = ""; + assertEquals(LITERAL_NULL, DSL.castJson(DSL.literal(empty)).valueOf()); + + String emptyObject = "{}"; + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), + DSL.castJson(DSL.literal(emptyObject)).valueOf()); + } + + @Test + void castJson_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index fa704b6c65f..bbc972ba738 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -34,3 +34,28 @@ Example:: | json empty string | | True | | json invalid object | {"invalid":"json", "string"} | False | +---------------------+---------------------------------+----------+ + +JSON +---------- + +Description +>>>>>>>>>>> + +Usage: `json(value)` Evaluates whether a string can be parsed as a json-encoded string and casted as an expression. Returns the JSON value if valid, null otherwise. + +Argument type: STRING + +Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY + +Example:: + + > source=json_test | where json_valid(json_string) | eval json=json(json_string) | fields test_name, json_string, json + fetched rows / total rows = 4/4 + +---------------------+------------------------------+---------------+ + | test_name | json_string | json | + |---------------------|------------------------------|---------------| + | json object | {"a":"1","b":"2"} | {a:"1",b:"2"} | + | json array | [1, 2, 3, 4] | [1,2,3,4] | + | json scalar string | "abc" | "abc" | + | json empty string | | null | + +---------------------+------------------------------+---------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 9e5ac041fb6..33c54ff60bf 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -52,4 +52,32 @@ public void test_not_json_valid() throws IOException { verifySchema(result, schema("test_name", null, "string")); verifyDataRows(result, rows("json invalid object"), rows("json null")); } + + @Test + public void test_cast_json() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval json=cast(json_string to json) | fields json", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string")); + } + + @Test + public void test_json() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | eval json=json(json_string) | fields json", TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows(result, rows("json invalid object")); + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index ea7e060d9cd..9d6707a8722 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -143,6 +143,7 @@ FLOAT: 'FLOAT'; STRING: 'STRING'; BOOLEAN: 'BOOLEAN'; IP: 'IP'; +JSON: 'JSON'; // SPECIAL CHARACTERS AND OPERATORS PIPE: '|'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 999c5d9c87c..74b05dc28b6 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -409,6 +409,7 @@ convertedDataType | typeName = STRING | typeName = BOOLEAN | typeName = IP + | typeName = JSON ; evalFunctionName @@ -419,6 +420,7 @@ evalFunctionName | flowControlFunctionName | systemFunctionName | positionFunctionName + | jsonFunctionName ; functionArgs @@ -700,6 +702,10 @@ positionFunctionName : POSITION ; +jsonFunctionName + : JSON + ; + // operators comparisonOperator : EQUAL @@ -963,4 +969,6 @@ keywordsCanBeId | SPARKLINE | C | DC + // JSON + | JSON ; From 7df87cbfbd1f94986bcdaacd022309f22287ffbf Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 10:56:50 -0800 Subject: [PATCH 42/87] PPL: Update json cast for review Signed-off-by: Andrew Carbonetto --- .../sql/expression/function/BuiltinFunctionName.java | 6 ++---- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 3 +-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index ef9a872974d..cd309a712d4 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -206,6 +206,7 @@ public enum BuiltinFunctionName { /** Json Functions. */ JSON_VALID(FunctionName.of("json_valid")), + JSON(FunctionName.of("json")), /** NULL Test. */ IS_NULL(FunctionName.of("is null")), @@ -260,10 +261,7 @@ public enum BuiltinFunctionName { MULTIMATCH(FunctionName.of("multimatch")), MULTIMATCHQUERY(FunctionName.of("multimatchquery")), WILDCARDQUERY(FunctionName.of("wildcardquery")), - WILDCARD_QUERY(FunctionName.of("wildcard_query")), - - /* Json Functions. */ - JSON(FunctionName.of("json")); + WILDCARD_QUERY(FunctionName.of("wildcard_query")); private final FunctionName name; diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 9d6707a8722..31a668be00d 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -143,7 +143,6 @@ FLOAT: 'FLOAT'; STRING: 'STRING'; BOOLEAN: 'BOOLEAN'; IP: 'IP'; -JSON: 'JSON'; // SPECIAL CHARACTERS AND OPERATORS PIPE: '|'; @@ -335,6 +334,7 @@ CIDRMATCH: 'CIDRMATCH'; // JSON FUNCTIONS JSON_VALID: 'JSON_VALID'; +JSON: 'JSON'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 74b05dc28b6..5d1f29614de 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -870,6 +870,7 @@ keywordsCanBeId | mathematicalFunctionName | positionFunctionName | conditionFunctionName + | jsonFunctionName // commands | SEARCH | DESCRIBE @@ -969,6 +970,4 @@ keywordsCanBeId | SPARKLINE | C | DC - // JSON - | JSON ; From cd45fcc628f6ad57a1de1c7b479dde6fbe73cfc8 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 17:17:51 -0800 Subject: [PATCH 43/87] Fix testes Signed-off-by: Andrew Carbonetto --- .../sql/expression/json/JsonFunctions.java | 8 ++++++ .../operator/convert/TypeCastOperators.java | 4 +-- .../org/opensearch/sql/utils/JsonUtils.java | 4 +-- .../opensearch/sql/ppl/JsonFunctionsIT.java | 25 ++++++++++++------- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index b0554fd2105..b1380eb0172 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -13,6 +13,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; @@ -26,10 +27,17 @@ public class JsonFunctions { public void register(BuiltinFunctionRepository repository) { repository.register(jsonValid()); + repository.register(jsonFunction()); } private DefaultFunctionResolver jsonValid() { return define( BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); } + + private DefaultFunctionResolver jsonFunction() { + return define( + BuiltinFunctionName.JSON.getName(), + impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index d6518e47d91..cfd570ab89a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -22,7 +22,6 @@ import static org.opensearch.sql.expression.function.FunctionDSL.implWithProperties; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandlingWithProperties; -import static org.opensearch.sql.utils.JsonUtils.castJson; import java.util.Arrays; import java.util.stream.Collectors; @@ -44,6 +43,7 @@ import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; import org.opensearch.sql.expression.function.FunctionDSL; +import org.opensearch.sql.utils.JsonUtils; @UtilityClass public class TypeCastOperators { @@ -189,7 +189,7 @@ private static DefaultFunctionResolver castToIp() { private static DefaultFunctionResolver castToJson() { return FunctionDSL.define( BuiltinFunctionName.CAST_TO_JSON.getName(), - impl(nullMissingHandling((v) -> castJson(v.stringValue())), UNDEFINED, STRING)); + impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); } private static DefaultFunctionResolver castToDate() { diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 9835db91931..0579eda933e 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -44,11 +44,11 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { } /** Converts a JSON encoded string to an Expression object. */ - public static ExprValue castJson(String json) { + public static ExprValue castJson(ExprValue json) { ObjectMapper objectMapper = new ObjectMapper(); JsonNode jsonNode; try { - jsonNode = objectMapper.readTree(json); + jsonNode = objectMapper.readTree(json.stringValue()); } catch (JsonProcessingException e) { final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 33c54ff60bf..465443c096b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,6 +12,8 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.util.List; +import java.util.Map; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -58,15 +60,15 @@ public void test_cast_json() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | eval json=cast(json_string to json) | fields json", + "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json) | fields test_name, casted", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, - rows("json object"), - rows("json array"), - rows("json scalar string"), - rows("json empty string")); + rows("json object", Map.of("a", "1", "b", "2")), + rows("json array", List.of(1,2,3,4)), + rows("json scalar string", "abc"), + rows("json empty string", null)); } @Test @@ -76,8 +78,13 @@ public void test_json() throws IOException { result = executeQuery( String.format( - "source=%s | eval json=json(json_string) | fields json", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows(result, rows("json invalid object")); + "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields test_name, casted", TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + verifyDataRows( + result, + rows("json object", Map.of("a", "1", "b", "2")), + rows("json array", List.of(1,2,3,4)), + rows("json scalar string", "abc"), + rows("json empty string", null)); } } From 6f5dc07f749975bb60b6016baa25357633179e54 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Thu, 9 Jan 2025 09:33:34 -0800 Subject: [PATCH 44/87] spotless Signed-off-by: Andrew Carbonetto --- .../java/org/opensearch/sql/ppl/JsonFunctionsIT.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 465443c096b..015333cdeae 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -60,13 +60,14 @@ public void test_cast_json() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json) | fields test_name, casted", + "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json)" + + " | fields test_name, casted", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1,2,3,4)), + rows("json array", List.of(1, 2, 3, 4)), rows("json scalar string", "abc"), rows("json empty string", null)); } @@ -78,12 +79,14 @@ public void test_json() throws IOException { result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields test_name, casted", TEST_INDEX_JSON_TEST)); + "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields" + + " test_name, casted", + TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1,2,3,4)), + rows("json array", List.of(1, 2, 3, 4)), rows("json scalar string", "abc"), rows("json empty string", null)); } From 0aae36e94f765094ea66122197885124e4462153 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 13 Jan 2025 16:13:01 -0800 Subject: [PATCH 45/87] Fix tests Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/ppl/JsonFunctionsIT.java | 13 +++++++++---- integ-test/src/test/resources/json_test.json | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 015333cdeae..a636273107b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,8 +12,10 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -66,8 +68,9 @@ public void test_cast_json() throws IOException { verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, - rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1, 2, 3, 4)), + rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), + rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json empty string", null)); } @@ -83,10 +86,12 @@ public void test_json() throws IOException { + " test_name, casted", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + JSONObject firstRow = new JSONObject(Map.of("c", 2)); verifyDataRows( result, - rows("json object", Map.of("a", "1", "b", "2")), - rows("json array", List.of(1, 2, 3, 4)), + rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), + rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json empty string", null)); } diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json index e393bfeb8e4..dae01ea4cef 100644 --- a/integ-test/src/test/resources/json_test.json +++ b/integ-test/src/test/resources/json_test.json @@ -1,5 +1,5 @@ {"index":{"_id":"0"}} -{"test_name":"json nested object", "json_string":"{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"} +{"test_name":"json nested object", "json_string":"{\"a\":\"1\", \"b\": {\"c\": \"3\"}, \"d\": [1, 2, 3]}"} {"index":{"_id":"1"}} {"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} {"index":{"_id":"2"}} From b225f2886513637cd8316471a56e800a303fa2b8 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 13 Jan 2025 16:20:17 -0800 Subject: [PATCH 46/87] SPOTLESS Signed-off-by: Andrew Carbonetto --- .../expression/json/JsonFunctionsTest.java | 176 +++++++++--------- .../opensearch/sql/ppl/JsonFunctionsIT.java | 9 +- 2 files changed, 94 insertions(+), 91 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 809d82941b2..1d06e1b2bd7 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -108,92 +108,92 @@ private ExprValue execute(ExprValue jsonString) { return exp.valueOf(); } - @Test - void json_returnsJsonObject() { - FunctionExpression exp; - - // Setup - final String objectJson = - "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " - + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; - - LinkedHashMap objectMap = new LinkedHashMap<>(); - objectMap.put("foo", new ExprStringValue("foo")); - objectMap.put("fuzz", ExprBooleanValue.of(true)); - objectMap.put("bar", new ExprLongValue(1234)); - objectMap.put("bar2", new ExprDoubleValue(12.34)); - objectMap.put("baz", ExprNullValue.of()); - objectMap.put( - "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); - objectMap.put( - "arr", - new ExprCollectionValue( - List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); - ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); - - // exercise - exp = DSL.json_function(DSL.literal(objectJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprTupleValue); - assertEquals(expectedTupleExpr, value); - } - - @Test - void json_returnsJsonArray() { - FunctionExpression exp; - - // Setup - final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; - ExprValue expectedArrayExpr = - new ExprCollectionValue( - List.of( - new ExprStringValue("foo"), - new ExprStringValue("fuzz"), - LITERAL_TRUE, - new ExprStringValue("bar"), - new ExprIntegerValue(1234), - new ExprDoubleValue(12.34), - LITERAL_NULL)); - - // exercise - exp = DSL.json_function(DSL.literal(arrayJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprCollectionValue); - assertEquals(expectedArrayExpr, value); - } - - @Test - void json_returnsScalar() { - assertEquals( - new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); - - assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); - - assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); - - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); - } - - @Test - void json_returnsSemanticCheckException() { - // invalid type - assertThrows( - SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); - - // missing bracket - assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); - - // mnissing quote - assertThrows( - SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); - } + @Test + void json_returnsJsonObject() { + FunctionExpression exp; + + // Setup + final String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.json_function(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void json_returnsJsonArray() { + FunctionExpression exp; + + // Setup + final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.json_function(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void json_returnsScalar() { + assertEquals( + new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); + + assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + } + + @Test + void json_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index a636273107b..eecf7fb338a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -12,7 +12,6 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.json.JSONArray; @@ -68,7 +67,9 @@ public void test_cast_json() throws IOException { verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); verifyDataRows( result, - rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows( + "json nested object", + new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), @@ -89,7 +90,9 @@ public void test_json() throws IOException { JSONObject firstRow = new JSONObject(Map.of("c", 2)); verifyDataRows( result, - rows("json nested object", new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), + rows( + "json nested object", + new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3)))), rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), From 78af4f8f636d26ba05f1a32044fd942076ef94ba Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 10:33:11 -0800 Subject: [PATCH 47/87] Clean up for merge Signed-off-by: Andrew Carbonetto --- .../sql/expression/json/JsonFunctions.java | 1 + .../org/opensearch/sql/utils/JsonUtils.java | 1 + .../expression/json/JsonFunctionsTest.java | 30 ------------------- 3 files changed, 2 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index b1380eb0172..64f84c44f75 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -16,6 +16,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; +import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import lombok.experimental.UtilityClass; import org.opensearch.sql.expression.function.BuiltinFunctionName; diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 0579eda933e..c5f7031b13e 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -18,6 +18,7 @@ import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.SemanticCheckException; @UtilityClass diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 1d06e1b2bd7..f3b17241fbc 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -7,24 +7,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -39,28 +27,10 @@ import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprBooleanValue; -import org.opensearch.sql.data.model.ExprCollectionValue; -import org.opensearch.sql.data.model.ExprDoubleValue; -import org.opensearch.sql.data.model.ExprIntegerValue; -import org.opensearch.sql.data.model.ExprLongValue; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprStringValue; -import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprBooleanValue; -import org.opensearch.sql.data.model.ExprCollectionValue; -import org.opensearch.sql.data.model.ExprDoubleValue; -import org.opensearch.sql.data.model.ExprIntegerValue; -import org.opensearch.sql.data.model.ExprLongValue; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprStringValue; -import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.FunctionExpression; From b84282aa2fbf0459a13591ea85041a7ee282ef4d Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 10:52:02 -0800 Subject: [PATCH 48/87] clean up unit tests Signed-off-by: Andrew Carbonetto --- .../expression/json/JsonFunctionsTest.java | 60 +++++++++++-------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index f3b17241fbc..4ba8b69e1cf 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -36,23 +36,16 @@ @ExtendWith(MockitoExtension.class) public class JsonFunctionsTest { - private static final ExprValue JsonNestedObject = - ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"); - private static final ExprValue JsonObject = - ExprValueUtils.stringValue("{\"a\":\"1\",\"b\":\"2\"}"); - private static final ExprValue JsonArray = ExprValueUtils.stringValue("[1, 2, 3, 4]"); - private static final ExprValue JsonScalarString = ExprValueUtils.stringValue("\"abc\""); - private static final ExprValue JsonEmptyString = ExprValueUtils.stringValue(""); - private static final ExprValue JsonInvalidObject = - ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"); - private static final ExprValue JsonInvalidScalar = ExprValueUtils.stringValue("abc"); - @Test public void json_valid_returns_false() { - assertEquals(LITERAL_FALSE, execute(JsonInvalidObject)); - assertEquals(LITERAL_FALSE, execute(JsonInvalidScalar)); - assertEquals(LITERAL_FALSE, execute(LITERAL_NULL)); - assertEquals(LITERAL_FALSE, execute(LITERAL_MISSING)); + assertEquals( + LITERAL_FALSE, + DSL.jsonValid(DSL.literal(ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"))) + .valueOf()); + assertEquals( + LITERAL_FALSE, DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue("abc")))).valueOf()); + assertEquals(LITERAL_FALSE, DSL.jsonValid(DSL.literal((LITERAL_NULL))).valueOf()); + assertEquals(LITERAL_FALSE, DSL.jsonValid(DSL.literal((LITERAL_MISSING))).valueOf()); } @Test @@ -66,16 +59,35 @@ public void json_valid_throws_ExpressionEvaluationException() { @Test public void json_valid_returns_true() { - assertEquals(LITERAL_TRUE, execute(JsonNestedObject)); - assertEquals(LITERAL_TRUE, execute(JsonObject)); - assertEquals(LITERAL_TRUE, execute(JsonArray)); - assertEquals(LITERAL_TRUE, execute(JsonScalarString)); - assertEquals(LITERAL_TRUE, execute(JsonEmptyString)); - } - private ExprValue execute(ExprValue jsonString) { - FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString)); - return exp.valueOf(); + List validJsonStrings = + List.of( + // test json objects are valid + "{\"a\":\"1\",\"b\":\"2\"}", + "{\"a\":1,\"b\":{\"c\":2,\"d\":3}}", + "{\"arr1\": [1,2,3], \"arr2\": [4,5,6]}", + + // test json arrays are valid + "[1, 2, 3, 4]", + "[{\"a\":1,\"b\":2}, {\"c\":3,\"d\":2}]", + + // test json scalars are valid + "\"abc\"", + "1234", + "true", + "false", + "null", + + // test empty string is valid + ""); + + validJsonStrings.stream() + .forEach( + str -> + assertEquals( + LITERAL_TRUE, + DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue(str)))).valueOf(), + String.format("String %s must be valid json", str))); } @Test From 1e2328606c375b166ef2a7335d7a89e0c37e80b7 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 11:07:48 -0800 Subject: [PATCH 49/87] Add casting from undefined Signed-off-by: Andrew Carbonetto --- .../operator/convert/TypeCastOperators.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index cfd570ab89a..960f6743c31 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -122,7 +122,9 @@ private static DefaultFunctionResolver castToInt() { impl( nullMissingHandling((v) -> new ExprIntegerValue(v.booleanValue() ? 1 : 0)), INTEGER, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), INTEGER, UNDEFINED) + ); } private static DefaultFunctionResolver castToLong() { @@ -136,7 +138,9 @@ private static DefaultFunctionResolver castToLong() { impl( nullMissingHandling((v) -> new ExprLongValue(v.booleanValue() ? 1L : 0L)), LONG, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), LONG, UNDEFINED) + ); } private static DefaultFunctionResolver castToFloat() { @@ -150,7 +154,9 @@ private static DefaultFunctionResolver castToFloat() { impl( nullMissingHandling((v) -> new ExprFloatValue(v.booleanValue() ? 1f : 0f)), FLOAT, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), FLOAT, UNDEFINED) + ); } private static DefaultFunctionResolver castToDouble() { @@ -164,7 +170,9 @@ private static DefaultFunctionResolver castToDouble() { impl( nullMissingHandling((v) -> new ExprDoubleValue(v.booleanValue() ? 1D : 0D)), DOUBLE, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), DOUBLE, UNDEFINED) + ); } private static DefaultFunctionResolver castToBoolean() { @@ -176,7 +184,9 @@ private static DefaultFunctionResolver castToBoolean() { STRING), impl( nullMissingHandling((v) -> ExprBooleanValue.of(v.doubleValue() != 0)), BOOLEAN, DOUBLE), - impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN)); + impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN), + impl(nullMissingHandling((v) -> v), BOOLEAN, UNDEFINED) + ); } private static DefaultFunctionResolver castToIp() { From 343f5a2970948393caafd1b6b0b243fd7bf5640a Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Thu, 16 Jan 2025 14:48:51 -0800 Subject: [PATCH 50/87] Add cast to scalar from undefined expression Signed-off-by: Andrew Carbonetto --- .../operator/convert/TypeCastOperators.java | 23 +++--- .../convert/TypeCastOperatorTest.java | 53 ++++++++++++++ .../opensearch/sql/ppl/JsonFunctionsIT.java | 71 +++++++++++++++++++ integ-test/src/test/resources/json_test.json | 14 +++- 4 files changed, 145 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index 960f6743c31..1a43ef14e1f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -92,9 +92,8 @@ private static DefaultFunctionResolver castToByte() { STRING), impl(nullMissingHandling((v) -> new ExprByteValue(v.byteValue())), BYTE, DOUBLE), impl( - nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), - BYTE, - BOOLEAN)); + nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), BYTE, BOOLEAN), + impl(nullMissingHandling((v) -> v), BYTE, UNDEFINED)); } private static DefaultFunctionResolver castToShort() { @@ -108,7 +107,8 @@ private static DefaultFunctionResolver castToShort() { impl( nullMissingHandling((v) -> new ExprShortValue(v.booleanValue() ? 1 : 0)), SHORT, - BOOLEAN)); + BOOLEAN), + impl(nullMissingHandling((v) -> v), SHORT, UNDEFINED)); } private static DefaultFunctionResolver castToInt() { @@ -123,8 +123,7 @@ private static DefaultFunctionResolver castToInt() { nullMissingHandling((v) -> new ExprIntegerValue(v.booleanValue() ? 1 : 0)), INTEGER, BOOLEAN), - impl(nullMissingHandling((v) -> v), INTEGER, UNDEFINED) - ); + impl(nullMissingHandling((v) -> v), INTEGER, UNDEFINED)); } private static DefaultFunctionResolver castToLong() { @@ -139,8 +138,7 @@ private static DefaultFunctionResolver castToLong() { nullMissingHandling((v) -> new ExprLongValue(v.booleanValue() ? 1L : 0L)), LONG, BOOLEAN), - impl(nullMissingHandling((v) -> v), LONG, UNDEFINED) - ); + impl(nullMissingHandling((v) -> v), LONG, UNDEFINED)); } private static DefaultFunctionResolver castToFloat() { @@ -155,8 +153,7 @@ private static DefaultFunctionResolver castToFloat() { nullMissingHandling((v) -> new ExprFloatValue(v.booleanValue() ? 1f : 0f)), FLOAT, BOOLEAN), - impl(nullMissingHandling((v) -> v), FLOAT, UNDEFINED) - ); + impl(nullMissingHandling((v) -> v), FLOAT, UNDEFINED)); } private static DefaultFunctionResolver castToDouble() { @@ -171,8 +168,7 @@ private static DefaultFunctionResolver castToDouble() { nullMissingHandling((v) -> new ExprDoubleValue(v.booleanValue() ? 1D : 0D)), DOUBLE, BOOLEAN), - impl(nullMissingHandling((v) -> v), DOUBLE, UNDEFINED) - ); + impl(nullMissingHandling((v) -> v), DOUBLE, UNDEFINED)); } private static DefaultFunctionResolver castToBoolean() { @@ -185,8 +181,7 @@ private static DefaultFunctionResolver castToBoolean() { impl( nullMissingHandling((v) -> ExprBooleanValue.of(v.doubleValue() != 0)), BOOLEAN, DOUBLE), impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN), - impl(nullMissingHandling((v) -> v), BOOLEAN, UNDEFINED) - ); + impl(nullMissingHandling((v) -> v), BOOLEAN, UNDEFINED)); } private static DefaultFunctionResolver castToIp() { diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index 27bd806c110..4ad28d76d04 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -165,6 +165,15 @@ void castBooleanToShort() { assertEquals(new ExprShortValue(0), expression.valueOf()); } + @Test + void castUndefinedToShort() { + Short value = 42; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castShort(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(SHORT, expression.type()); + assertEquals(new ExprShortValue(value), expression.valueOf()); + } + @Test void castBooleanToInt() { FunctionExpression expression = DSL.castInt(DSL.literal(true)); @@ -176,6 +185,15 @@ void castBooleanToInt() { assertEquals(new ExprIntegerValue(0), expression.valueOf()); } + @Test + void castUndefinedToInt() { + Integer value = 42; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castInt(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(INTEGER, expression.type()); + assertEquals(new ExprIntegerValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToLong({0})") @MethodSource({"numberData"}) void castToLong(ExprValue value) { @@ -208,6 +226,15 @@ void castBooleanToLong() { assertEquals(new ExprLongValue(0), expression.valueOf()); } + @Test + void castUndefinedToLong() { + Long value = 42l; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castLong(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(LONG, expression.type()); + assertEquals(new ExprLongValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToFloat({0})") @MethodSource({"numberData"}) void castToFloat(ExprValue value) { @@ -240,6 +267,15 @@ void castBooleanToFloat() { assertEquals(new ExprFloatValue(0), expression.valueOf()); } + @Test + void castUndefinedToFloat() { + Float value = 23.45f; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castFloat(DSL.castJson(DSL.literal(value.toString()))); + assertEquals(FLOAT, expression.type()); + assertEquals(new ExprFloatValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToDouble({0})") @MethodSource({"numberData"}) void castToDouble(ExprValue value) { @@ -272,6 +308,15 @@ void castBooleanToDouble() { assertEquals(new ExprDoubleValue(0), expression.valueOf()); } + @Test + void castUndefinedToDouble() { + Double value = 23.45e5; + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castDouble(DSL.castJson(DSL.literal(value))); + assertEquals(DOUBLE, expression.type()); + assertEquals(new ExprDoubleValue(value), expression.valueOf()); + } + @ParameterizedTest(name = "castToBoolean({0})") @MethodSource({"numberData"}) void castToBoolean(ExprValue value) { @@ -301,6 +346,14 @@ void castBooleanToBoolean() { assertEquals(ExprBooleanValue.of(true), expression.valueOf()); } + @Test + void castUndefinedToBoolean() { + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castBoolean(DSL.castJson(DSL.literal("true"))); + assertEquals(BOOLEAN, expression.type()); + assertEquals(ExprBooleanValue.of(true), expression.valueOf()); + } + @Test void castToDate() { FunctionExpression expression = DSL.castDate(DSL.literal("2012-08-07")); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index eecf7fb338a..e837057b682 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -73,6 +73,11 @@ public void test_cast_json() throws IOException { rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), + rows("json scalar int", 1234), + rows("json scalar float", 12.34f), + rows("json scalar double", 2.99792458e8), + rows("json scalar boolean true", true), + rows("json scalar boolean false", false), rows("json empty string", null)); } @@ -96,6 +101,72 @@ public void test_json() throws IOException { rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), + rows("json scalar int", 1234), + rows("json scalar long", 42), + rows("json scalar double", 2.99792458e8), + rows("json scalar boolean true", true), + rows("json scalar boolean false", false), rows("json empty string", null)); } + + @Test + public void test_cast_json_scalar_to_type() throws IOException { + // cast to integer + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar int' | " + + "eval casted=cast(json(json_string) as int) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "integer")); + verifyDataRows(result, rows("json scalar int", 1234)); + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar int' | " + + "eval casted=cast(json(json_string) as long) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "long")); + verifyDataRows(result, rows("json scalar int", 1234l)); + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar float' | " + + "eval casted=cast(json(json_string) as float) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "float")); + verifyDataRows(result, rows("json scalar float", 12.34f)); + + result = + executeQuery( + String.format( + "source=%s | " + + "where test_name='json scalar double' | " + + "eval casted=cast(json(json_string) as double) | " + + "fields test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "double")); + verifyDataRows(result, rows("json scalar double", 2.99792458e8)); + + result = + executeQuery( + String.format( + "source=%s | where test_name='json scalar boolean true' OR test_name='json scalar" + + " boolean false' | eval casted=cast(json(json_string) as boolean) | fields" + + " test_name, casted", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "boolean")); + verifyDataRows( + result, rows("json scalar boolean true", true), rows("json scalar boolean false", false)); + } } diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json index dae01ea4cef..acab339c016 100644 --- a/integ-test/src/test/resources/json_test.json +++ b/integ-test/src/test/resources/json_test.json @@ -7,8 +7,18 @@ {"index":{"_id":"3"}} {"test_name":"json scalar string", "json_string":"\"abc\""} {"index":{"_id":"4"}} -{"test_name":"json empty string","json_string":""} +{"test_name":"json scalar int", "json_string":"1234"} {"index":{"_id":"5"}} -{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} +{"test_name":"json scalar float", "json_string":"12.34"} {"index":{"_id":"6"}} +{"test_name":"json scalar double", "json_string":"2.99792458e8"} +{"index":{"_id":"7"}} +{"test_name":"json scalar boolean true", "json_string":"true"} +{"index":{"_id":"8"}} +{"test_name":"json scalar boolean false", "json_string":"false"} +{"index":{"_id":"9"}} +{"test_name":"json empty string", "json_string":""} +{"index":{"_id":"10"}} +{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} +{"index":{"_id":"11"}} {"test_name":"json null", "json_string":null} From e8b6df3e6ab0ce616b0ba1c9868dca19044cdfad Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Thu, 16 Jan 2025 15:34:17 -0800 Subject: [PATCH 51/87] Add test for missing/null Signed-off-by: Andrew Carbonetto --- .../opensearch/sql/expression/json/JsonFunctionsTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 4ba8b69e1cf..1cbd71b806f 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -74,6 +74,7 @@ public void json_valid_returns_true() { // test json scalars are valid "\"abc\"", "1234", + "12.34", "true", "false", "null", @@ -155,10 +156,16 @@ void json_returnsScalar() { assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + assertEquals(new ExprDoubleValue(12.34), DSL.json_function(DSL.literal("12.34")).valueOf()); + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal(LITERAL_NULL)).valueOf()); + + assertEquals(LITERAL_MISSING, DSL.json_function(DSL.literal(LITERAL_MISSING)).valueOf()); + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); assertEquals( From ab9be7533c3d497b5e13c6cc36337312264247c4 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Fri, 17 Jan 2025 09:58:42 -0800 Subject: [PATCH 52/87] Clean up merge conflicts Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/DSL.java | 8 ++-- .../sql/expression/json/JsonFunctions.java | 6 --- .../operator/convert/TypeCastOperators.java | 5 ++- .../org/opensearch/sql/utils/JsonUtils.java | 20 +++++++++- .../expression/json/JsonFunctionsTest.java | 38 ++++++++++--------- .../convert/TypeCastOperatorTest.java | 14 ++++--- .../opensearch/sql/ppl/JsonFunctionsIT.java | 7 +++- 7 files changed, 59 insertions(+), 39 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index a50e3d14706..966077c7906 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -687,6 +687,10 @@ public static FunctionExpression jsonValid(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.JSON_VALID, expressions); } + public static FunctionExpression stringToJson(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); + } + public static Aggregator avg(Expression... expressions) { return aggregate(BuiltinFunctionName.AVG, expressions); } @@ -977,10 +981,6 @@ public static FunctionExpression utc_timestamp( return compile(functionProperties, BuiltinFunctionName.UTC_TIMESTAMP, args); } - public static FunctionExpression json_function(Expression value) { - return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); - } - @SuppressWarnings("unchecked") private static T compile( FunctionProperties functionProperties, BuiltinFunctionName bfn, Expression... args) { diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 64f84c44f75..75f134aa4e9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,12 +5,6 @@ package org.opensearch.sql.expression.json; -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.experimental.UtilityClass; -import org.opensearch.sql.expression.function.BuiltinFunctionName; -import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.expression.function.DefaultFunctionResolver; - import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index 1a43ef14e1f..c1391ac9abe 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -92,8 +92,9 @@ private static DefaultFunctionResolver castToByte() { STRING), impl(nullMissingHandling((v) -> new ExprByteValue(v.byteValue())), BYTE, DOUBLE), impl( - nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), BYTE, BOOLEAN), - impl(nullMissingHandling((v) -> v), BYTE, UNDEFINED)); + nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), + BYTE, + BOOLEAN)); } private static DefaultFunctionResolver castToShort() { diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index c5f7031b13e..63cbaf4a998 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -12,9 +12,11 @@ import java.util.List; import java.util.Map; import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; @@ -27,7 +29,7 @@ public class JsonUtils { * Checks if given JSON string can be parsed as valid JSON. * * @param jsonExprValue JSON string (e.g. "{\"hello\": \"world\"}"). - * @return true if the string can be parsed as valid JSON, else false. + * @return true if the string can be parsed as valid JSON, else false (including null or missing). */ public static ExprValue isValidJson(ExprValue jsonExprValue) { ObjectMapper objectMapper = new ObjectMapper(); @@ -44,7 +46,21 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { } } - /** Converts a JSON encoded string to an Expression object. */ + /** + * Converts a JSON encoded string to a {@link ExprValue}. Expression type will be UNDEFINED. + * + * @param json JSON string (e.g. "{\"hello\": \"world\"}"). + * @return ExprValue returns an expression that best represents the provided JSON-encoded string. + *
    + *
  1. {@link ExprTupleValue} if the JSON is an object + *
  2. {@link ExprCollectionValue} if the JSON is an array + *
  3. {@link ExprDoubleValue} if the JSON is a floating-point number scalar + *
  4. {@link ExprIntegerValue} if the JSON is an integral number scalar + *
  5. {@link ExprStringValue} if the JSON is a string scalar + *
  6. {@link ExprBooleanValue} if the JSON is a boolean scalar + *
  7. {@link ExprNullValue} if the JSON is null, empty, or invalid + *
+ */ public static ExprValue castJson(ExprValue json) { ObjectMapper objectMapper = new ObjectMapper(); JsonNode jsonNode; diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 1cbd71b806f..26b8ed89fd4 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -51,10 +51,8 @@ public void json_valid_returns_false() { @Test public void json_valid_throws_ExpressionEvaluationException() { assertThrows( - ExpressionEvaluationException.class, () -> execute(ExprValueUtils.booleanValue(true))); - - // caught by nullMissingHandling and returns null - assertEquals(LITERAL_NULL, execute(LITERAL_NULL)); + ExpressionEvaluationException.class, + () -> DSL.jsonValid(DSL.literal((ExprValueUtils.booleanValue(true)))).valueOf()); } @Test @@ -115,12 +113,16 @@ void json_returnsJsonObject() { ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); // exercise - exp = DSL.json_function(DSL.literal(objectJson)); + exp = DSL.stringToJson(DSL.literal(objectJson)); // Verify var value = exp.valueOf(); assertTrue(value instanceof ExprTupleValue); assertEquals(expectedTupleExpr, value); + + // also test the empty object case + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.stringToJson(DSL.literal("{}")).valueOf()); } @Test @@ -141,35 +143,35 @@ void json_returnsJsonArray() { LITERAL_NULL)); // exercise - exp = DSL.json_function(DSL.literal(arrayJson)); + exp = DSL.stringToJson(DSL.literal(arrayJson)); // Verify var value = exp.valueOf(); assertTrue(value instanceof ExprCollectionValue); assertEquals(expectedArrayExpr, value); + + // also test the empty-array case + assertEquals(new ExprCollectionValue(List.of()), DSL.stringToJson(DSL.literal("[]")).valueOf()); } @Test void json_returnsScalar() { assertEquals( - new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); - - assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + new ExprStringValue("foobar"), DSL.stringToJson(DSL.literal("\"foobar\"")).valueOf()); - assertEquals(new ExprDoubleValue(12.34), DSL.json_function(DSL.literal("12.34")).valueOf()); + assertEquals(new ExprIntegerValue(1234), DSL.stringToJson(DSL.literal("1234")).valueOf()); - assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + assertEquals(new ExprDoubleValue(12.34), DSL.stringToJson(DSL.literal("12.34")).valueOf()); - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + assertEquals(LITERAL_TRUE, DSL.stringToJson(DSL.literal("true")).valueOf()); - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal(LITERAL_NULL)).valueOf()); + assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("null")).valueOf()); - assertEquals(LITERAL_MISSING, DSL.json_function(DSL.literal(LITERAL_MISSING)).valueOf()); + assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal(LITERAL_NULL)).valueOf()); - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + assertEquals(LITERAL_MISSING, DSL.stringToJson(DSL.literal(LITERAL_MISSING)).valueOf()); - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("")).valueOf()); } @Test @@ -181,7 +183,7 @@ void json_returnsSemanticCheckException() { // missing bracket assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); - // mnissing quote + // missing quote assertThrows( SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); } diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index 4ad28d76d04..ff0c8bcc019 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -312,7 +312,7 @@ void castBooleanToDouble() { void castUndefinedToDouble() { Double value = 23.45e5; // json cast is an UNDEFINED type expression - FunctionExpression expression = DSL.castDouble(DSL.castJson(DSL.literal(value))); + FunctionExpression expression = DSL.castDouble(DSL.castJson(DSL.literal(value.toString()))); assertEquals(DOUBLE, expression.type()); assertEquals(new ExprDoubleValue(value), expression.valueOf()); } @@ -480,6 +480,10 @@ void castJson_returnsJsonObject() { var value = exp.valueOf(); assertTrue(value instanceof ExprTupleValue); assertEquals(expectedTupleExpr, value); + + // also test the empty-object case + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.castJson(DSL.literal("{}")).valueOf()); } @Test @@ -506,6 +510,9 @@ void castJson_returnsJsonArray() { var value = exp.valueOf(); assertTrue(value instanceof ExprCollectionValue); assertEquals(expectedArrayExpr, value); + + // also test the empty-array case + assertEquals(new ExprCollectionValue(List.of()), DSL.castJson(DSL.literal("[]")).valueOf()); } @Test @@ -525,11 +532,6 @@ void castJson_returnsScalar() { String empty = ""; assertEquals(LITERAL_NULL, DSL.castJson(DSL.literal(empty)).valueOf()); - - String emptyObject = "{}"; - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), - DSL.castJson(DSL.literal(emptyObject)).valueOf()); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index e837057b682..9ed61c33a48 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -40,6 +40,11 @@ public void test_json_valid() throws IOException { rows("json object"), rows("json array"), rows("json scalar string"), + rows("json scalar int"), + rows("json scalar float"), + rows("json scalar double"), + rows("json scalar boolean true"), + rows("json scalar boolean false"), rows("json empty string")); } @@ -102,7 +107,7 @@ public void test_json() throws IOException { rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json scalar int", 1234), - rows("json scalar long", 42), + rows("json scalar float", 12.34), rows("json scalar double", 2.99792458e8), rows("json scalar boolean true", true), rows("json scalar boolean false", false), From 788be9d6e91e4c647cc14859e3b66fab40c5324d Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Fri, 17 Jan 2025 15:24:03 -0800 Subject: [PATCH 53/87] Fix jacoco coverage Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/json/JsonFunctionsTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 26b8ed89fd4..1f39b5be9b2 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -164,6 +164,7 @@ void json_returnsScalar() { assertEquals(new ExprDoubleValue(12.34), DSL.stringToJson(DSL.literal("12.34")).valueOf()); assertEquals(LITERAL_TRUE, DSL.stringToJson(DSL.literal("true")).valueOf()); + assertEquals(LITERAL_FALSE, DSL.stringToJson(DSL.literal("false")).valueOf()); assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("null")).valueOf()); From a9721bfb2269524e83ba381eb32ed3e468c8fc5f Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Fri, 17 Jan 2025 15:44:23 -0800 Subject: [PATCH 54/87] Move to Switch by json type Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/utils/JsonUtils.java | 57 +++++++++---------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 63cbaf4a998..6120d11bb52 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -75,36 +75,33 @@ public static ExprValue castJson(ExprValue json) { } private static ExprValue processJsonNode(JsonNode jsonNode) { - if (jsonNode.isFloatingPointNumber()) { - return new ExprDoubleValue(jsonNode.asDouble()); + switch (jsonNode.getNodeType()) { + case ARRAY: + List elements = new LinkedList<>(); + for (var iter = jsonNode.iterator(); iter.hasNext(); ) { + jsonNode = iter.next(); + elements.add(processJsonNode(jsonNode)); + } + return new ExprCollectionValue(elements); + case OBJECT: + Map values = new LinkedHashMap<>(); + for (var iter = jsonNode.fields(); iter.hasNext(); ) { + Map.Entry entry = iter.next(); + values.put(entry.getKey(), processJsonNode(entry.getValue())); + } + return ExprTupleValue.fromExprValueMap(values); + case STRING: + return new ExprStringValue(jsonNode.asText()); + case NUMBER: + if (jsonNode.isFloatingPointNumber()) { + return new ExprDoubleValue(jsonNode.asDouble()); + } + return new ExprIntegerValue(jsonNode.asLong()); + case BOOLEAN: + return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE; + default: + // in all other cases, return null + return LITERAL_NULL; } - if (jsonNode.isIntegralNumber()) { - return new ExprIntegerValue(jsonNode.asLong()); - } - if (jsonNode.isBoolean()) { - return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE; - } - if (jsonNode.isTextual()) { - return new ExprStringValue(jsonNode.asText()); - } - if (jsonNode.isArray()) { - List elements = new LinkedList<>(); - for (var iter = jsonNode.iterator(); iter.hasNext(); ) { - jsonNode = iter.next(); - elements.add(processJsonNode(jsonNode)); - } - return new ExprCollectionValue(elements); - } - if (jsonNode.isObject()) { - Map values = new LinkedHashMap<>(); - for (var iter = jsonNode.fields(); iter.hasNext(); ) { - Map.Entry entry = iter.next(); - values.put(entry.getKey(), processJsonNode(entry.getValue())); - } - return ExprTupleValue.fromExprValueMap(values); - } - - // in all other cases, return null - return LITERAL_NULL; } } From 018e462583fd26c2f333b7e554fe2ad309556eb8 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Mon, 20 Jan 2025 11:01:35 -0800 Subject: [PATCH 55/87] functionality implemented Signed-off-by: Kenrick Yap --- core/build.gradle | 1 + .../org/opensearch/sql/expression/DSL.java | 4 ++ .../function/BuiltinFunctionName.java | 1 + .../sql/expression/json/JsonFunctions.java | 7 +++ .../org/opensearch/sql/utils/JsonUtils.java | 63 +++++++++++++++++-- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 1 + 7 files changed, 74 insertions(+), 4 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index c5962513422..d16d1b31fb9 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -54,6 +54,7 @@ dependencies { api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + api group: 'com.jayway.jsonpath', name: 'json-path', version: '2.9.0' api group: 'com.google.code.gson', name: 'gson', version: '2.8.9' api group: 'com.tdunning', name: 't-digest', version: '3.3' api project(':common') diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index a50e3d14706..7b4aa337e03 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -687,6 +687,10 @@ public static FunctionExpression jsonValid(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.JSON_VALID, expressions); } + public static FunctionExpression jsonExtract(Expression... expressions) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON_EXTRACT, expressions); + } + public static Aggregator avg(Expression... expressions) { return aggregate(BuiltinFunctionName.AVG, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index cd309a712d4..261624a822e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -207,6 +207,7 @@ public enum BuiltinFunctionName { /** Json Functions. */ JSON_VALID(FunctionName.of("json_valid")), JSON(FunctionName.of("json")), + JSON_EXTRACT(FunctionName.of("json_extract")), /** NULL Test. */ IS_NULL(FunctionName.of("is null")), diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 75f134aa4e9..dfc4d32ef6b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -23,6 +23,7 @@ public class JsonFunctions { public void register(BuiltinFunctionRepository repository) { repository.register(jsonValid()); repository.register(jsonFunction()); + repository.register(jsonExtract()); } private DefaultFunctionResolver jsonValid() { @@ -35,4 +36,10 @@ private DefaultFunctionResolver jsonFunction() { BuiltinFunctionName.JSON.getName(), impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); } + + private DefaultFunctionResolver jsonExtract() { + return define( + BuiltinFunctionName.JSON_EXTRACT.getName(), + impl(nullMissingHandling(JsonUtils::extractJson), UNDEFINED, STRING, STRING)); + } } diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index c5f7031b13e..8471c58b3ea 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -7,10 +7,22 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.InvalidJsonException; +import com.jayway.jsonpath.JsonPath; + +import java.io.ObjectInputFilter; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; + +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.PathNotFoundException; +import com.jayway.jsonpath.spi.json.JacksonJsonProvider; +import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import lombok.experimental.UtilityClass; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDoubleValue; @@ -46,16 +58,59 @@ public static ExprValue isValidJson(ExprValue jsonExprValue) { /** Converts a JSON encoded string to an Expression object. */ public static ExprValue castJson(ExprValue json) { + JsonNode jsonNode = jsonToNode(json); + return processJsonNode(jsonNode); + } + + public static ExprValue extractJson(ExprValue json, ExprValue path) { + String jsonString = json.stringValue(); + String jsonPath = path.stringValue(); + + try { + Configuration config = Configuration.builder() + .options(Option.AS_PATH_LIST) + .build(); + List resultPaths = JsonPath.using(config).parse(jsonString).read(jsonPath); + + List elements = new LinkedList<>(); + + for (String resultPath : resultPaths) { + Object result = JsonPath.parse(jsonString).read(resultPath); + String resultJsonString = new ObjectMapper().writeValueAsString(result); + try { + elements.add(processJsonNode(jsonStringToNode(resultJsonString))); + } catch (SemanticCheckException e) { + elements.add(new ExprStringValue(resultJsonString)); + } + } + + if (elements.size() == 1) { + return elements.get(0); + } else { + return new ExprCollectionValue(elements); + } + } catch (PathNotFoundException e) { + return LITERAL_NULL; + } catch (InvalidJsonException | JsonProcessingException e) { + final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; + throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); + } + } + + private static JsonNode jsonToNode(ExprValue json) { + return jsonStringToNode(json.stringValue()); + } + + private static JsonNode jsonStringToNode(String jsonString) { ObjectMapper objectMapper = new ObjectMapper(); JsonNode jsonNode; try { - jsonNode = objectMapper.readTree(json.stringValue()); + jsonNode = objectMapper.readTree(jsonString); } catch (JsonProcessingException e) { final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; - throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); + throw new SemanticCheckException(String.format(errorFormat, jsonString, e.getMessage()), e); } - - return processJsonNode(jsonNode); + return jsonNode; } private static ExprValue processJsonNode(JsonNode jsonNode) { diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 31a668be00d..267698ee064 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -335,6 +335,7 @@ CIDRMATCH: 'CIDRMATCH'; // JSON FUNCTIONS JSON_VALID: 'JSON_VALID'; JSON: 'JSON'; +JSON_EXTRACT: 'JSON_EXTRACT'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 5d1f29614de..b9a27dcd665 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -704,6 +704,7 @@ positionFunctionName jsonFunctionName : JSON + | JSON_EXTRACT ; // operators From c6c6cc1562440f412d327a91972031cab975476a Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Tue, 21 Jan 2025 10:43:36 -0800 Subject: [PATCH 56/87] Remove conflicted files Signed-off-by: Andrew Carbonetto --- .../sql/expression/datetime/DateTimeFunctionTest.java | 2 -- .../org/opensearch/sql/expression/datetime/YearweekTest.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 115898e349e..ad15dadfb73 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1230,8 +1230,6 @@ public void testWeekFormats( expectedInteger); } - // subtracting 1 as a temporary fix for year 2024. - // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index ee4df23be4e..d944f7c85c3 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -99,8 +99,6 @@ public void testYearweekWithoutMode() { assertEquals(eval(expression), eval(expressionWithoutMode)); } - // subtracting 1 as a temporary fix for year 2024. - // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); From a5652eabeb6cf42ff71851f2cb5c5cfc5b55c02d Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Tue, 21 Jan 2025 14:51:31 -0800 Subject: [PATCH 57/87] Add doctext row Signed-off-by: Andrew Carbonetto --- docs/user/ppl/functions/json.rst | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index bbc972ba738..77d9d00f45f 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -50,12 +50,13 @@ Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY Example:: > source=json_test | where json_valid(json_string) | eval json=json(json_string) | fields test_name, json_string, json - fetched rows / total rows = 4/4 - +---------------------+------------------------------+---------------+ - | test_name | json_string | json | - |---------------------|------------------------------|---------------| - | json object | {"a":"1","b":"2"} | {a:"1",b:"2"} | - | json array | [1, 2, 3, 4] | [1,2,3,4] | - | json scalar string | "abc" | "abc" | - | json empty string | | null | - +---------------------+------------------------------+---------------+ + fetched rows / total rows = 5/5 + +---------------------+---------------------------------+-------------------------+ + | test_name | json_string | json | + |---------------------|---------------------------------|-------------------------| + | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | {a:"1",b:{c:"2",d:"3"}} | + | json object | {"a":"1","b":"2"} | {a:"1",b:"2"} | + | json array | [1, 2, 3, 4] | [1,2,3,4] | + | json scalar string | "abc" | "abc" | + | json empty string | | null | + +---------------------+---------------------------------+-------------------------+ From 2cd10a2ca9e4780328de92706803f05183f32c3d Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 21 Jan 2025 16:19:48 -0800 Subject: [PATCH 58/87] added integ-test and doc test Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/utils/JsonUtils.java | 9 ++- docs/user/ppl/functions/json.rst | 67 ++++++++++++++++--- doctest/test_data/json_test.json | 1 + .../opensearch/sql/ppl/JsonFunctionsIT.java | 36 +++++++++- integ-test/src/test/resources/json_test.json | 2 + 5 files changed, 97 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 8471c58b3ea..4e149c7ad03 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -8,21 +8,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.jayway.jsonpath.Configuration; -import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.InvalidJsonException; import com.jayway.jsonpath.JsonPath; -import java.io.ObjectInputFilter; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import com.jayway.jsonpath.Option; import com.jayway.jsonpath.PathNotFoundException; -import com.jayway.jsonpath.spi.json.JacksonJsonProvider; -import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import lombok.experimental.UtilityClass; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDoubleValue; @@ -66,6 +61,10 @@ public static ExprValue extractJson(ExprValue json, ExprValue path) { String jsonString = json.stringValue(); String jsonPath = path.stringValue(); + if (jsonString.equals("")) { + return LITERAL_NULL; + } + try { Configuration config = Configuration.builder() .options(Option.AS_PATH_LIST) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 82c0da23a67..1f2c9e7b23d 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -24,16 +24,17 @@ Example:: > source=json_test | eval is_valid = json_valid(json_string) | fields test_name, json_string, is_valid fetched rows / total rows = 6/6 - +---------------------+---------------------------------+----------+ - | test_name | json_string | is_valid | - |---------------------|---------------------------------|----------| - | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | True | - | json object | {"a":"1","b":"2"} | True | - | json array | [1, 2, 3, 4] | True | - | json scalar string | "abc" | True | - | json empty string | | True | - | json invalid object | {"invalid":"json", "string"} | False | - +---------------------+---------------------------------+----------+ + +---------------------+-------------------------------------+----------+ + | test_name | json_string | is_valid | + |---------------------|-------------------------------------|----------| + | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | True | + | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | True | + | json object | {"a":"1","b":"2"} | True | + | json array | [1, 2, 3, 4] | True | + | json scalar string | "abc" | True | + | json empty string | | True | + | json invalid object | {"invalid":"json", "string"} | False | + +---------------------+-------------------------------------+----------+ JSON ---------- @@ -59,3 +60,49 @@ Example:: | json scalar string | "abc" | "abc" | | json empty string | | null | +---------------------+------------------------------+---------------+ + +JSON_EXTRACT +____________ + +Description +>>>>>>>>>>> + +Usage: `json_extract(doc, path [, path]...)` Extracts a json value or scalar from a json document based on the path(s) specified. + +Argument type: STRING, STRING + +Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY + +- Returns a JSON array for multiple paths or if the path leads to an array. +- Return null if path is not valid. + +Example:: + + > source=json_test | where json_valid(json_string) | eval json_extract=json_extract(json_string, '$.b') | fields test_name, json_string, json_extract + fetched rows / total rows = 6/6 + +---------------------+-------------------------------------+-------------------+ + | test_name | json_string | json_extract | + |---------------------|-------------------------------------|-------------------| + | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | {c:"2",d:"3"} | + | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | [{c:"2"},{c:"3"}] | + | json object | {"a":"1","b":"2"} | 2 | + | json array | [1, 2, 3, 4] | null | + | json scalar string | "abc" | null | + | json empty string | | null | + +---------------------+-------------------------------------+-------------------+ + + > source=json_test | where test_name="json nested list" | eval json_extract=json_extract('{"a":[{"b":1},{"b":2}]}', '$.b[1].c') + fetched rows / total rows = 1/1 + +---------------------+-------------------------------------+--------------+ + | test_name | json_string | json_extract | + |---------------------|-------------------------------------|--------------| + | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | 3 | + +---------------------+-------------------------------------+--------------+ + + > source=json_test | where test_name="json nested list" | eval json_extract=json_extract('{"a":[{"b":1},{"b":2}]}', '$.b[*].c') + fetched rows / total rows = 1/1 + +---------------------+-------------------------------------+--------------+ + | test_name | json_string | json_extract | + |---------------------|-------------------------------------|--------------| + | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | [2,3] | + +---------------------+-------------------------------------+--------------+ diff --git a/doctest/test_data/json_test.json b/doctest/test_data/json_test.json index 7494fc4aa91..63e7f150115 100644 --- a/doctest/test_data/json_test.json +++ b/doctest/test_data/json_test.json @@ -1,4 +1,5 @@ {"test_name":"json nested object", "json_string":"{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"} +{"test_name":"json nested list", "json_string":"{\"a\":\"1\",\"b\":[{\"c\":\"2\"}, {\"c\":\"3\"}]}"} {"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"} {"test_name":"json array", "json_string":"[1, 2, 3, 4]"} {"test_name":"json scalar string", "json_string":"\"abc\""} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index eecf7fb338a..ef55b51685a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -40,7 +40,9 @@ public void test_json_valid() throws IOException { rows("json object"), rows("json array"), rows("json scalar string"), - rows("json empty string")); + rows("json empty string"), + rows("json nested list") + ); } @Test @@ -73,7 +75,10 @@ public void test_cast_json() throws IOException { rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), - rows("json empty string", null)); + rows("json empty string", null), + rows("json nested list", + new JSONObject(Map.of("a", "1", "b", List.of(Map.of("c", "2"), Map.of("c", "3"))))) + ); } @Test @@ -96,6 +101,31 @@ public void test_json() throws IOException { rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), - rows("json empty string", null)); + rows("json empty string", null), + rows("json nested list", + new JSONObject(Map.of("a", "1", "b", List.of(Map.of("c", "2"), Map.of("c", "3"))))) + ); + } + + @Test + public void test_json_extract() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | eval json_extract=json_extract(json_string, '$.b') | fields" + + " test_name, json_extract", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("json_extract", null, "undefined")); + verifyDataRows( + result, + rows("json nested object", new JSONObject(Map.of("c", "3"))), + rows("json object", "2"), + rows("json array", null), + rows("json scalar string", null), + rows("json empty string", null), + rows("json nested list", new JSONArray(List.of(Map.of("c","2"), Map.of("c","3")))) + ); } } diff --git a/integ-test/src/test/resources/json_test.json b/integ-test/src/test/resources/json_test.json index dae01ea4cef..1a317fb587d 100644 --- a/integ-test/src/test/resources/json_test.json +++ b/integ-test/src/test/resources/json_test.json @@ -12,3 +12,5 @@ {"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"} {"index":{"_id":"6"}} {"test_name":"json null", "json_string":null} +{"index":{"_id":"7"}} +{"test_name":"json nested list", "json_string":"{\"a\":\"1\",\"b\":[{\"c\":\"2\"}, {\"c\":\"3\"}]}"} From cd78ddd93560ed79fb6f0d5fa815b9a468027e38 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Wed, 22 Jan 2025 14:07:31 -0800 Subject: [PATCH 59/87] fixed integ tests Signed-off-by: Kenrick Yap --- .../opensearch/sql/expression/json/JsonFunctions.java | 2 +- .../main/java/org/opensearch/sql/utils/JsonUtils.java | 10 ++-------- .../java/org/opensearch/sql/ppl/JsonFunctionsIT.java | 6 ++---- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index dfc4d32ef6b..30c5cde162e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -40,6 +40,6 @@ private DefaultFunctionResolver jsonFunction() { private DefaultFunctionResolver jsonExtract() { return define( BuiltinFunctionName.JSON_EXTRACT.getName(), - impl(nullMissingHandling(JsonUtils::extractJson), UNDEFINED, STRING, STRING)); + impl(JsonUtils::extractJson, UNDEFINED, STRING, STRING)); } } diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 4e149c7ad03..5b0e0af69b9 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -72,15 +72,9 @@ public static ExprValue extractJson(ExprValue json, ExprValue path) { List resultPaths = JsonPath.using(config).parse(jsonString).read(jsonPath); List elements = new LinkedList<>(); - for (String resultPath : resultPaths) { Object result = JsonPath.parse(jsonString).read(resultPath); - String resultJsonString = new ObjectMapper().writeValueAsString(result); - try { - elements.add(processJsonNode(jsonStringToNode(resultJsonString))); - } catch (SemanticCheckException e) { - elements.add(new ExprStringValue(resultJsonString)); - } + elements.add(ExprValueUtils.fromObjectValue(result)); } if (elements.size() == 1) { @@ -90,7 +84,7 @@ public static ExprValue extractJson(ExprValue json, ExprValue path) { } } catch (PathNotFoundException e) { return LITERAL_NULL; - } catch (InvalidJsonException | JsonProcessingException e) { + } catch (InvalidJsonException e) { final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index ef55b51685a..3b15ca99e9a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -110,14 +110,12 @@ public void test_json() throws IOException { @Test public void test_json_extract() throws IOException { JSONObject result; - result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | eval json_extract=json_extract(json_string, '$.b') | fields" - + " test_name, json_extract", + "source=%s | where json_valid(json_string) | eval extracted=json_extract(json_string, '$.b') | fields test_name, extracted", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string"), schema("json_extract", null, "undefined")); + verifySchema(result, schema("test_name", null, "string"), schema("extracted", null, "undefined")); verifyDataRows( result, rows("json nested object", new JSONObject(Map.of("c", "3"))), From afb385fb059933cc86c8081da7218346158f7ec3 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 23 Jan 2025 00:23:05 -0800 Subject: [PATCH 60/87] unit tests Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/utils/JsonUtils.java | 4 + .../expression/json/JsonFunctionsTest.java | 104 +++++++++++++++++- 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 5b0e0af69b9..c6196c0c054 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.InvalidJsonException; +import com.jayway.jsonpath.InvalidPathException; import com.jayway.jsonpath.JsonPath; import java.util.LinkedHashMap; @@ -84,6 +85,9 @@ public static ExprValue extractJson(ExprValue json, ExprValue path) { } } catch (PathNotFoundException e) { return LITERAL_NULL; + } catch (InvalidPathException e) { + final String errorFormat = "JSON path '%s' is not valid. Error details: %s"; + throw new SemanticCheckException(String.format(errorFormat, path, e.getMessage()), e); } catch (InvalidJsonException e) { final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 89ea57c2e4b..cede667260d 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -16,12 +16,14 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprFloatValue; import org.opensearch.sql.data.model.ExprIntegerValue; import org.opensearch.sql.data.model.ExprLongValue; import org.opensearch.sql.data.model.ExprNullValue; @@ -32,6 +34,7 @@ import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; @ExtendWith(MockitoExtension.class) @@ -159,8 +162,107 @@ void json_returnsSemanticCheckException() { // missing bracket assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); - // mnissing quote + // missing quote assertThrows( SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); } + + @Test + void json_extract_return_object() { + List validJsonStrings = + List.of( + // test json objects are valid + "{\"a\":\"1\",\"b\":\"2\"}", + "{\"a\":1,\"b\":{\"c\":2,\"d\":3}}", + "{\"arr1\": [1,2,3], \"arr2\": [4,5,6]}", + + // test json arrays are valid + "[1, 2, 3, 4]", + "[{\"a\":1,\"b\":2}, {\"c\":3,\"d\":2}]", + + // test json scalars are valid + "\"abc\"", + "1234", + "12.34", + "true", + "false", + "null", + + // test empty string is valid + ""); + + validJsonStrings.stream() + .forEach( + str -> + assertEquals( + LITERAL_TRUE, + DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue(str)))).valueOf(), + String.format("String %s must be valid json", str))); + } + + @Test + void json_extract_search_arrays() { + Expression jsonArray = DSL.literal(ExprValueUtils.stringValue("{\"a\":[1,2.3,\"abc\",true,null,{\"c\":1},[1,2,3]]}")); + List expectedExprValue = List.of( + new ExprIntegerValue(1), + new ExprFloatValue(2.3), + new ExprStringValue("abc"), + LITERAL_TRUE, + LITERAL_NULL, + ExprTupleValue.fromExprValueMap(Map.of("c", new ExprIntegerValue(1))), + new ExprCollectionValue(List.of(new ExprIntegerValue(1), new ExprIntegerValue(2), new ExprIntegerValue(3))) + ); + + // extract specific index from JSON list + for (int i = 0 ; i < expectedExprValue.size() ; i++ ) { + String path = String.format("$.a[%d]", i); + Expression pathExpr = DSL.literal(ExprValueUtils.stringValue(path)); + FunctionExpression expression = DSL.jsonExtract(jsonArray, pathExpr); + assertEquals(expectedExprValue.get(i), expression.valueOf()); + } + + // extract * from JSON list + Expression starPath = DSL.literal(ExprValueUtils.stringValue("$.a[*]")); + FunctionExpression starExpression = DSL.castInt(DSL.jsonExtract(jsonArray, starPath)); + assertEquals( + new ExprCollectionValue(expectedExprValue), starExpression.valueOf()); + } + + @Test + void json_extract_returns_null() { + List jsonStrings = + List.of( + "{\"a\":\"1\",\"b\":\"2\"}", + "{\"a\":1,\"b\":{\"c\":2,\"d\":3}}", + "{\"arr1\": [1,2,3], \"arr2\": [4,5,6]}", + "[1, 2, 3, 4]", + "[{\"a\":1,\"b\":2}, {\"c\":3,\"d\":2}]", + "\"abc\"", + "1234", + "12.34", + "true", + "false", + "null", + ""); + + jsonStrings.stream() + .forEach( + str -> + assertEquals( + LITERAL_NULL, + DSL.jsonExtract( + DSL.literal((ExprValueUtils.stringValue(str))), + DSL.literal("$.a.path_not_found_key")).valueOf(), + String.format("JSON string %s should return null", str))); + } + + @Test + void json_returns_SemanticCheckException() { + // invalid path + assertThrows( + SemanticCheckException.class, () -> DSL.jsonExtract(DSL.literal("invalid"), DSL.literal("invalid")).valueOf()); + + // invalid json + assertThrows(SemanticCheckException.class, () -> DSL.jsonExtract(DSL.literal("{\"invalid\":\"json\", \"string\"}"), DSL.literal("invalid")).valueOf()); + } } From 794db8ab3ff1ecb930b6e0e2a2936ccebd27347a Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 23 Jan 2025 04:45:44 -0800 Subject: [PATCH 61/87] finnished unit tests Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/utils/JsonUtils.java | 10 +- .../expression/json/JsonFunctionsTest.java | 122 ++++++++++-------- .../opensearch/sql/ppl/JsonFunctionsIT.java | 24 ++-- 3 files changed, 82 insertions(+), 74 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index c6196c0c054..f3239ae25ae 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -11,14 +11,12 @@ import com.jayway.jsonpath.InvalidJsonException; import com.jayway.jsonpath.InvalidPathException; import com.jayway.jsonpath.JsonPath; - +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.PathNotFoundException; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; - -import com.jayway.jsonpath.Option; -import com.jayway.jsonpath.PathNotFoundException; import lombok.experimental.UtilityClass; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDoubleValue; @@ -67,9 +65,7 @@ public static ExprValue extractJson(ExprValue json, ExprValue path) { } try { - Configuration config = Configuration.builder() - .options(Option.AS_PATH_LIST) - .build(); + Configuration config = Configuration.builder().options(Option.AS_PATH_LIST).build(); List resultPaths = JsonPath.using(config).parse(jsonString).read(jsonPath); List elements = new LinkedList<>(); diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 5729b229a23..bb73a27fe99 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -8,7 +8,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; - import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; @@ -17,7 +16,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; - import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @@ -168,54 +166,25 @@ void json_returnsSemanticCheckException() { SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); } - @Test - void json_extract_return_object() { - List validJsonStrings = - List.of( - // test json objects are valid - "{\"a\":\"1\",\"b\":\"2\"}", - "{\"a\":1,\"b\":{\"c\":2,\"d\":3}}", - "{\"arr1\": [1,2,3], \"arr2\": [4,5,6]}", - - // test json arrays are valid - "[1, 2, 3, 4]", - "[{\"a\":1,\"b\":2}, {\"c\":3,\"d\":2}]", - - // test json scalars are valid - "\"abc\"", - "1234", - "12.34", - "true", - "false", - "null", - - // test empty string is valid - ""); - - validJsonStrings.stream() - .forEach( - str -> - assertEquals( - LITERAL_TRUE, - DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue(str)))).valueOf(), - String.format("String %s must be valid json", str))); - } - @Test void json_extract_search_arrays() { - Expression jsonArray = DSL.literal(ExprValueUtils.stringValue("{\"a\":[1,2.3,\"abc\",true,null,{\"c\":1},[1,2,3]]}")); - List expectedExprValue = List.of( - new ExprIntegerValue(1), - new ExprFloatValue(2.3), - new ExprStringValue("abc"), - LITERAL_TRUE, - LITERAL_NULL, - ExprTupleValue.fromExprValueMap(Map.of("c", new ExprIntegerValue(1))), - new ExprCollectionValue(List.of(new ExprIntegerValue(1), new ExprIntegerValue(2), new ExprIntegerValue(3))) - ); + Expression jsonArray = + DSL.literal( + ExprValueUtils.stringValue("{\"a\":[1,2.3,\"abc\",true,null,{\"c\":1},[1,2,3]]}")); + List expectedExprValue = + List.of( + new ExprIntegerValue(1), + new ExprFloatValue(2.3), + new ExprStringValue("abc"), + LITERAL_TRUE, + LITERAL_NULL, + ExprTupleValue.fromExprValueMap(Map.of("c", new ExprIntegerValue(1))), + new ExprCollectionValue( + List.of( + new ExprIntegerValue(1), new ExprIntegerValue(2), new ExprIntegerValue(3)))); // extract specific index from JSON list - for (int i = 0 ; i < expectedExprValue.size() ; i++ ) { + for (int i = 0; i < expectedExprValue.size(); i++) { String path = String.format("$.a[%d]", i); Expression pathExpr = DSL.literal(ExprValueUtils.stringValue(path)); FunctionExpression expression = DSL.jsonExtract(jsonArray, pathExpr); @@ -224,9 +193,8 @@ void json_extract_search_arrays() { // extract * from JSON list Expression starPath = DSL.literal(ExprValueUtils.stringValue("$.a[*]")); - FunctionExpression starExpression = DSL.castInt(DSL.jsonExtract(jsonArray, starPath)); - assertEquals( - new ExprCollectionValue(expectedExprValue), starExpression.valueOf()); + FunctionExpression starExpression = DSL.jsonExtract(jsonArray, starPath); + assertEquals(new ExprCollectionValue(expectedExprValue), starExpression.valueOf()); } @Test @@ -243,7 +211,6 @@ void json_extract_returns_null() { "12.34", "true", "false", - "null", ""); jsonStrings.stream() @@ -252,18 +219,63 @@ void json_extract_returns_null() { assertEquals( LITERAL_NULL, DSL.jsonExtract( - DSL.literal((ExprValueUtils.stringValue(str))), - DSL.literal("$.a.path_not_found_key")).valueOf(), + DSL.literal((ExprValueUtils.stringValue(str))), + DSL.literal("$.a.path_not_found_key")) + .valueOf(), String.format("JSON string %s should return null", str))); } @Test - void json_returns_SemanticCheckException() { + void json_extract_throws_SemanticCheckException() { // invalid path assertThrows( - SemanticCheckException.class, () -> DSL.jsonExtract(DSL.literal("invalid"), DSL.literal("invalid")).valueOf()); + SemanticCheckException.class, + () -> + DSL.jsonExtract( + DSL.literal(new ExprStringValue("{\"a\":1}")), + DSL.literal(new ExprStringValue("$a"))) + .valueOf()); // invalid json - assertThrows(SemanticCheckException.class, () -> DSL.jsonExtract(DSL.literal("{\"invalid\":\"json\", \"string\"}"), DSL.literal("invalid")).valueOf()); + assertThrows( + SemanticCheckException.class, + () -> + DSL.jsonExtract( + DSL.literal(new ExprStringValue("{\"invalid\":\"json\", \"string\"}")), + DSL.literal(new ExprStringValue("$.a"))) + .valueOf()); + } + + @Test + void json_extract_throws_ExpressionEvaluationException() { + // null json + assertThrows( + ExpressionEvaluationException.class, + () -> + DSL.jsonExtract(DSL.literal(LITERAL_NULL), DSL.literal(new ExprStringValue("$.a"))) + .valueOf()); + + // null path + assertThrows( + ExpressionEvaluationException.class, + () -> + DSL.jsonExtract( + DSL.literal(new ExprStringValue("{\"a\":1}")), DSL.literal(LITERAL_NULL)) + .valueOf()); + + // missing json + assertThrows( + ExpressionEvaluationException.class, + () -> + DSL.jsonExtract(DSL.literal(LITERAL_MISSING), DSL.literal(new ExprStringValue("$.a"))) + .valueOf()); + + // missing path + assertThrows( + ExpressionEvaluationException.class, + () -> + DSL.jsonExtract( + DSL.literal(new ExprStringValue("{\"a\":1}")), DSL.literal(LITERAL_MISSING)) + .valueOf()); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 3b15ca99e9a..afaa989a74a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -41,8 +41,7 @@ public void test_json_valid() throws IOException { rows("json array"), rows("json scalar string"), rows("json empty string"), - rows("json nested list") - ); + rows("json nested list")); } @Test @@ -76,9 +75,9 @@ public void test_cast_json() throws IOException { rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json empty string", null), - rows("json nested list", - new JSONObject(Map.of("a", "1", "b", List.of(Map.of("c", "2"), Map.of("c", "3"))))) - ); + rows( + "json nested list", + new JSONObject(Map.of("a", "1", "b", List.of(Map.of("c", "2"), Map.of("c", "3")))))); } @Test @@ -102,9 +101,9 @@ public void test_json() throws IOException { rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows("json scalar string", "abc"), rows("json empty string", null), - rows("json nested list", - new JSONObject(Map.of("a", "1", "b", List.of(Map.of("c", "2"), Map.of("c", "3"))))) - ); + rows( + "json nested list", + new JSONObject(Map.of("a", "1", "b", List.of(Map.of("c", "2"), Map.of("c", "3")))))); } @Test @@ -113,9 +112,11 @@ public void test_json_extract() throws IOException { result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | eval extracted=json_extract(json_string, '$.b') | fields test_name, extracted", + "source=%s | where json_valid(json_string) | eval" + + " extracted=json_extract(json_string, '$.b') | fields test_name, extracted", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string"), schema("extracted", null, "undefined")); + verifySchema( + result, schema("test_name", null, "string"), schema("extracted", null, "undefined")); verifyDataRows( result, rows("json nested object", new JSONObject(Map.of("c", "3"))), @@ -123,7 +124,6 @@ public void test_json_extract() throws IOException { rows("json array", null), rows("json scalar string", null), rows("json empty string", null), - rows("json nested list", new JSONArray(List.of(Map.of("c","2"), Map.of("c","3")))) - ); + rows("json nested list", new JSONArray(List.of(Map.of("c", "2"), Map.of("c", "3"))))); } } From 0f0b8d4a26fa47d2f9ae190d9050d9d87b2ab37c Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 23 Jan 2025 04:59:57 -0800 Subject: [PATCH 62/87] update doctest Signed-off-by: Kenrick Yap --- docs/user/ppl/functions/json.rst | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index ffef550c9c1..fb6a398b051 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -51,15 +51,17 @@ Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY Example:: > source=json_test | where json_valid(json_string) | eval json=json(json_string) | fields test_name, json_string, json - fetched rows / total rows = 4/4 - +---------------------+------------------------------+---------------+ - | test_name | json_string | json | - |---------------------|------------------------------|---------------| - | json object | {"a":"1","b":"2"} | {a:"1",b:"2"} | - | json array | [1, 2, 3, 4] | [1,2,3,4] | - | json scalar string | "abc" | "abc" | - | json empty string | | null | - +---------------------+------------------------------+---------------+ + fetched rows / total rows = 6/6 + +---------------------+-------------------------------------+-----------------------------+ + | test_name | json_string | json | + |---------------------|-------------------------------------|-----------------------------| + | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | {a:"1",b:{c:"2",d:"3"} | + | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | {a:"1",b:[{c:"2"},{c:"3"}]} | + | json object | {"a":"1","b":"2"} | {a:"1",b:"2"} | + | json array | [1, 2, 3, 4] | [1,2,3,4] | + | json scalar string | "abc" | "abc" | + | json empty string | | null | + +---------------------+-------------------------------------+-----------------------------+ JSON_EXTRACT ____________ From f030057b1999ba1ddc626d45c051e38d626369f6 Mon Sep 17 00:00:00 2001 From: 14yapkc1 Date: Mon, 27 Jan 2025 11:20:46 -0500 Subject: [PATCH 63/87] addessed comments Signed-off-by: 14yapkc1 --- .../org/opensearch/sql/utils/JsonUtils.java | 20 +++---------------- docs/user/ppl/functions/json.rst | 2 +- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index f3239ae25ae..b3968911f2e 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -7,11 +7,9 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.InvalidJsonException; import com.jayway.jsonpath.InvalidPathException; import com.jayway.jsonpath.JsonPath; -import com.jayway.jsonpath.Option; import com.jayway.jsonpath.PathNotFoundException; import java.util.LinkedHashMap; import java.util.LinkedList; @@ -60,25 +58,13 @@ public static ExprValue extractJson(ExprValue json, ExprValue path) { String jsonString = json.stringValue(); String jsonPath = path.stringValue(); - if (jsonString.equals("")) { + if (jsonString.isEmpty()) { return LITERAL_NULL; } try { - Configuration config = Configuration.builder().options(Option.AS_PATH_LIST).build(); - List resultPaths = JsonPath.using(config).parse(jsonString).read(jsonPath); - - List elements = new LinkedList<>(); - for (String resultPath : resultPaths) { - Object result = JsonPath.parse(jsonString).read(resultPath); - elements.add(ExprValueUtils.fromObjectValue(result)); - } - - if (elements.size() == 1) { - return elements.get(0); - } else { - return new ExprCollectionValue(elements); - } + Object results = JsonPath.parse(jsonString).read(jsonPath); + return ExprValueUtils.fromObjectValue(results); } catch (PathNotFoundException e) { return LITERAL_NULL; } catch (InvalidPathException e) { diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index fb6a398b051..864073dc011 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -69,7 +69,7 @@ ____________ Description >>>>>>>>>>> -Usage: `json_extract(doc, path [, path]...)` Extracts a json value or scalar from a json document based on the path(s) specified. +Usage: `json_extract(doc, path)` Extracts a json value or scalar from a json document based on the path(s) specified. Argument type: STRING, STRING From 2b08007279eaccb3d9d5a145338825d54013d675 Mon Sep 17 00:00:00 2001 From: 14yapkc1 Date: Tue, 28 Jan 2025 11:37:49 -0500 Subject: [PATCH 64/87] added addition edge cases for unit tests Signed-off-by: 14yapkc1 --- .../org/opensearch/sql/utils/JsonUtils.java | 7 ++++ .../expression/json/JsonFunctionsTest.java | 37 ++++++++++++++++++- docs/user/ppl/functions/json.rst | 4 +- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index b3968911f2e..c4c8a5073cf 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -54,6 +54,13 @@ public static ExprValue castJson(ExprValue json) { return processJsonNode(jsonNode); } + /** + * Extract value of JSON string at given JSON path. + * + * @param json JSON string (e.g. "{\"hello\": \"world\"}"). + * @param path JSON path (e.g. "$.hello") + * @return ExprValue of value at given path of json string. + */ public static ExprValue extractJson(ExprValue json, ExprValue path) { String jsonString = json.stringValue(); String jsonPath = path.stringValue(); diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index bb73a27fe99..89acba7cbed 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -166,11 +166,36 @@ void json_returnsSemanticCheckException() { SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); } + @Test + void json_extract_search() { + Expression jsonArray = DSL.literal(ExprValueUtils.stringValue("{\"a\":1}")); + ExprValue expectedExprValue = new ExprIntegerValue(1); + Expression pathExpr = DSL.literal(ExprValueUtils.stringValue("$.a")); + FunctionExpression expression = DSL.jsonExtract(jsonArray, pathExpr); + assertEquals(expectedExprValue, expression.valueOf()); + } + + @Test + void json_extract_search_arrays_out_of_bound() { + Expression jsonArray = DSL.literal(ExprValueUtils.stringValue("{\"a\":[1,2,3}")); + + // index out of bounds + assertThrows( + SemanticCheckException.class, + () -> DSL.jsonExtract(jsonArray, DSL.literal(new ExprStringValue("$.a[3]"))).valueOf()); + + // negative index + assertThrows( + SemanticCheckException.class, + () -> DSL.jsonExtract(jsonArray, DSL.literal(new ExprStringValue("$.a[-1]"))).valueOf()); + } + @Test void json_extract_search_arrays() { Expression jsonArray = DSL.literal( - ExprValueUtils.stringValue("{\"a\":[1,2.3,\"abc\",true,null,{\"c\":1},[1,2,3]]}")); + ExprValueUtils.stringValue( + "{\"a\":[1,2.3,\"abc\",true,null,{\"c\":{\"d\":1}},[1,2,3]]}")); List expectedExprValue = List.of( new ExprIntegerValue(1), @@ -178,7 +203,8 @@ void json_extract_search_arrays() { new ExprStringValue("abc"), LITERAL_TRUE, LITERAL_NULL, - ExprTupleValue.fromExprValueMap(Map.of("c", new ExprIntegerValue(1))), + ExprTupleValue.fromExprValueMap( + Map.of("c", ExprTupleValue.fromExprValueMap(Map.of("d", new ExprIntegerValue(1))))), new ExprCollectionValue( List.of( new ExprIntegerValue(1), new ExprIntegerValue(2), new ExprIntegerValue(3)))); @@ -191,6 +217,13 @@ void json_extract_search_arrays() { assertEquals(expectedExprValue.get(i), expression.valueOf()); } + // extract nested object + ExprValue nestedExpected = + ExprTupleValue.fromExprValueMap(Map.of("d", new ExprIntegerValue(1))); + Expression nestedPath = DSL.literal(ExprValueUtils.stringValue("$.a[5].c")); + FunctionExpression nestedExpression = DSL.jsonExtract(jsonArray, nestedPath); + assertEquals(nestedExpected, nestedExpression.valueOf()); + // extract * from JSON list Expression starPath = DSL.literal(ExprValueUtils.stringValue("$.a[*]")); FunctionExpression starExpression = DSL.jsonExtract(jsonArray, starPath); diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 864073dc011..febe606ffe1 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -69,7 +69,7 @@ ____________ Description >>>>>>>>>>> -Usage: `json_extract(doc, path)` Extracts a json value or scalar from a json document based on the path(s) specified. +Usage: `json_extract(doc, path)` Extracts a json value or scalar from a json document based on the path specified. Argument type: STRING, STRING @@ -77,6 +77,8 @@ Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY - Returns a JSON array for multiple paths or if the path leads to an array. - Return null if path is not valid. +- Throws error if `doc` or `path` is malformed. +- Throws error if `doc` or `path` is MISSING or NULL. Example:: From 6678be4bfbb0939ead3145dbe2a47a66cc4951d8 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Wed, 29 Jan 2025 11:11:36 -0500 Subject: [PATCH 65/87] addressed PR comments Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/utils/JsonUtils.java | 10 ++- .../expression/json/JsonFunctionsTest.java | 88 +++++++------------ docs/user/ppl/functions/json.rst | 8 +- 3 files changed, 43 insertions(+), 63 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 5fa148d7c17..e1cbd3a03af 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -1,9 +1,5 @@ package org.opensearch.sql.utils; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -27,6 +23,8 @@ import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.SemanticCheckException; +import static org.opensearch.sql.data.model.ExprValueUtils.*; + @UtilityClass public class JsonUtils { /** @@ -86,6 +84,10 @@ public static ExprValue castJson(ExprValue json) { * @return ExprValue of value at given path of json string. */ public static ExprValue extractJson(ExprValue json, ExprValue path) { + if (json == LITERAL_NULL || json == LITERAL_MISSING) { + return json; + } + String jsonString = json.stringValue(); String jsonPath = path.stringValue(); diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 3e57355ed70..1aef2080ede 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -193,34 +193,18 @@ void json_returnsSemanticCheckException() { @Test void json_extract_search() { - Expression jsonArray = DSL.literal(ExprValueUtils.stringValue("{\"a\":1}")); - ExprValue expectedExprValue = new ExprIntegerValue(1); - Expression pathExpr = DSL.literal(ExprValueUtils.stringValue("$.a")); - FunctionExpression expression = DSL.jsonExtract(jsonArray, pathExpr); - assertEquals(expectedExprValue, expression.valueOf()); + ExprValue expected = new ExprIntegerValue(1); + execute_extract_json(expected, "{\"a\":1}", "$.a"); } @Test void json_extract_search_arrays_out_of_bound() { - Expression jsonArray = DSL.literal(ExprValueUtils.stringValue("{\"a\":[1,2,3}")); - - // index out of bounds - assertThrows( - SemanticCheckException.class, - () -> DSL.jsonExtract(jsonArray, DSL.literal(new ExprStringValue("$.a[3]"))).valueOf()); - - // negative index - assertThrows( - SemanticCheckException.class, - () -> DSL.jsonExtract(jsonArray, DSL.literal(new ExprStringValue("$.a[-1]"))).valueOf()); + execute_extract_json(LITERAL_NULL, "{\"a\":[1,2,3]}", "$.a[4]"); } @Test void json_extract_search_arrays() { - Expression jsonArray = - DSL.literal( - ExprValueUtils.stringValue( - "{\"a\":[1,2.3,\"abc\",true,null,{\"c\":{\"d\":1}},[1,2,3]]}")); + String jsonArray = "{\"a\":[1,2.3,\"abc\",true,null,{\"c\":{\"d\":1}},[1,2,3]]}"; List expectedExprValue = List.of( new ExprIntegerValue(1), @@ -237,22 +221,17 @@ void json_extract_search_arrays() { // extract specific index from JSON list for (int i = 0; i < expectedExprValue.size(); i++) { String path = String.format("$.a[%d]", i); - Expression pathExpr = DSL.literal(ExprValueUtils.stringValue(path)); - FunctionExpression expression = DSL.jsonExtract(jsonArray, pathExpr); - assertEquals(expectedExprValue.get(i), expression.valueOf()); + execute_extract_json(expectedExprValue.get(i), jsonArray, path); } // extract nested object ExprValue nestedExpected = ExprTupleValue.fromExprValueMap(Map.of("d", new ExprIntegerValue(1))); - Expression nestedPath = DSL.literal(ExprValueUtils.stringValue("$.a[5].c")); - FunctionExpression nestedExpression = DSL.jsonExtract(jsonArray, nestedPath); - assertEquals(nestedExpected, nestedExpression.valueOf()); + execute_extract_json(nestedExpected, jsonArray, "$.a[5].c"); // extract * from JSON list - Expression starPath = DSL.literal(ExprValueUtils.stringValue("$.a[*]")); - FunctionExpression starExpression = DSL.jsonExtract(jsonArray, starPath); - assertEquals(new ExprCollectionValue(expectedExprValue), starExpression.valueOf()); + ExprValue starExpected = new ExprCollectionValue(expectedExprValue); + execute_extract_json(starExpected, jsonArray, "$.a[*]"); } @Test @@ -271,48 +250,47 @@ void json_extract_returns_null() { "false", ""); - jsonStrings.stream() - .forEach( - str -> - assertEquals( - LITERAL_NULL, - DSL.jsonExtract( - DSL.literal((ExprValueUtils.stringValue(str))), - DSL.literal("$.a.path_not_found_key")) - .valueOf(), - String.format("JSON string %s should return null", str))); + jsonStrings.forEach( + str -> execute_extract_json(LITERAL_NULL, str, "$.a.path_not_found_key") + ); + + // null json + assertEquals(LITERAL_NULL, DSL.jsonExtract(DSL.literal(LITERAL_NULL), DSL.literal(new ExprStringValue("$.a"))).valueOf()); + + // missing json + assertEquals(LITERAL_MISSING, DSL.jsonExtract(DSL.literal(LITERAL_MISSING), DSL.literal(new ExprStringValue("$.a"))).valueOf()); } @Test void json_extract_throws_SemanticCheckException() { // invalid path - assertThrows( + SemanticCheckException invalidPathError = assertThrows( SemanticCheckException.class, () -> DSL.jsonExtract( DSL.literal(new ExprStringValue("{\"a\":1}")), DSL.literal(new ExprStringValue("$a"))) .valueOf()); + assertEquals( + "JSON path '\"$a\"' is not valid. Error details: Illegal character at position 1 expected '.' or '['", + invalidPathError.getMessage()); + // invalid json - assertThrows( + SemanticCheckException invalidJsonError = assertThrows( SemanticCheckException.class, () -> DSL.jsonExtract( DSL.literal(new ExprStringValue("{\"invalid\":\"json\", \"string\"}")), DSL.literal(new ExprStringValue("$.a"))) .valueOf()); + assertEquals( + "JSON string '\"{\"invalid\":\"json\", \"string\"}\"' is not valid. Error details: net.minidev.json.parser.ParseException: Unexpected character (}) at position 26.", + invalidJsonError.getMessage()); } @Test void json_extract_throws_ExpressionEvaluationException() { - // null json - assertThrows( - ExpressionEvaluationException.class, - () -> - DSL.jsonExtract(DSL.literal(LITERAL_NULL), DSL.literal(new ExprStringValue("$.a"))) - .valueOf()); - // null path assertThrows( ExpressionEvaluationException.class, @@ -321,13 +299,6 @@ void json_extract_throws_ExpressionEvaluationException() { DSL.literal(new ExprStringValue("{\"a\":1}")), DSL.literal(LITERAL_NULL)) .valueOf()); - // missing json - assertThrows( - ExpressionEvaluationException.class, - () -> - DSL.jsonExtract(DSL.literal(LITERAL_MISSING), DSL.literal(new ExprStringValue("$.a"))) - .valueOf()); - // missing path assertThrows( ExpressionEvaluationException.class, @@ -336,4 +307,11 @@ void json_extract_throws_ExpressionEvaluationException() { DSL.literal(new ExprStringValue("{\"a\":1}")), DSL.literal(LITERAL_MISSING)) .valueOf()); } + + private static void execute_extract_json(ExprValue expected, String json, String path) { + Expression pathExpr = DSL.literal(ExprValueUtils.stringValue(path)); + Expression jsonExpr = DSL.literal(ExprValueUtils.stringValue(json)); + ExprValue actual = DSL.jsonExtract(jsonExpr, pathExpr).valueOf(); + assertEquals(expected, actual); + } } diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index febe606ffe1..c9260529bb2 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -75,10 +75,10 @@ Argument type: STRING, STRING Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY -- Returns a JSON array for multiple paths or if the path leads to an array. -- Return null if path is not valid. -- Throws error if `doc` or `path` is malformed. -- Throws error if `doc` or `path` is MISSING or NULL. +- Returns a JSON array if path points to multiple results (e.g. $.a[*]) or if the path points to an array. +- Return null if path is not valid is MISSING or NULL. +- Throws SemanticCheckException if `doc` or `path` is malformed. +- Throws ExpressionEvaluationException if `path` is missing. Example:: From e57fa21a67db9a039b8cead077df6985fe447b4e Mon Sep 17 00:00:00 2001 From: Kenrick Yap <14yapkc1@gmail.com> Date: Thu, 30 Jan 2025 11:07:35 -0500 Subject: [PATCH 66/87] fix code coverage Signed-off-by: Kenrick Yap <14yapkc1@gmail.com> --- .../org/opensearch/sql/expression/DSL.java | 4 -- .../org/opensearch/sql/utils/JsonUtils.java | 4 +- .../expression/json/JsonFunctionsTest.java | 55 +++++++++++-------- docs/user/ppl/functions/json.rst | 4 +- 4 files changed, 35 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 98afc9cbb4b..58bc9390df5 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -985,10 +985,6 @@ public static FunctionExpression utc_timestamp( return compile(functionProperties, BuiltinFunctionName.UTC_TIMESTAMP, args); } - public static FunctionExpression json_function(Expression value) { - return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); - } - @SuppressWarnings("unchecked") private static T compile( FunctionProperties functionProperties, BuiltinFunctionName bfn, Expression... args) { diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index e1cbd3a03af..b898026a081 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -1,5 +1,7 @@ package org.opensearch.sql.utils; +import static org.opensearch.sql.data.model.ExprValueUtils.*; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -23,8 +25,6 @@ import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.SemanticCheckException; -import static org.opensearch.sql.data.model.ExprValueUtils.*; - @UtilityClass public class JsonUtils { /** diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 1aef2080ede..beac451a6ec 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -250,43 +250,50 @@ void json_extract_returns_null() { "false", ""); - jsonStrings.forEach( - str -> execute_extract_json(LITERAL_NULL, str, "$.a.path_not_found_key") - ); + jsonStrings.forEach(str -> execute_extract_json(LITERAL_NULL, str, "$.a.path_not_found_key")); // null json - assertEquals(LITERAL_NULL, DSL.jsonExtract(DSL.literal(LITERAL_NULL), DSL.literal(new ExprStringValue("$.a"))).valueOf()); + assertEquals( + LITERAL_NULL, + DSL.jsonExtract(DSL.literal(LITERAL_NULL), DSL.literal(new ExprStringValue("$.a"))) + .valueOf()); // missing json - assertEquals(LITERAL_MISSING, DSL.jsonExtract(DSL.literal(LITERAL_MISSING), DSL.literal(new ExprStringValue("$.a"))).valueOf()); + assertEquals( + LITERAL_MISSING, + DSL.jsonExtract(DSL.literal(LITERAL_MISSING), DSL.literal(new ExprStringValue("$.a"))) + .valueOf()); } @Test void json_extract_throws_SemanticCheckException() { // invalid path - SemanticCheckException invalidPathError = assertThrows( - SemanticCheckException.class, - () -> - DSL.jsonExtract( - DSL.literal(new ExprStringValue("{\"a\":1}")), - DSL.literal(new ExprStringValue("$a"))) - .valueOf()); + SemanticCheckException invalidPathError = + assertThrows( + SemanticCheckException.class, + () -> + DSL.jsonExtract( + DSL.literal(new ExprStringValue("{\"a\":1}")), + DSL.literal(new ExprStringValue("$a"))) + .valueOf()); assertEquals( - "JSON path '\"$a\"' is not valid. Error details: Illegal character at position 1 expected '.' or '['", - invalidPathError.getMessage()); - + "JSON path '\"$a\"' is not valid. Error details: Illegal character at position 1 expected" + + " '.' or '['", + invalidPathError.getMessage()); // invalid json - SemanticCheckException invalidJsonError = assertThrows( - SemanticCheckException.class, - () -> - DSL.jsonExtract( - DSL.literal(new ExprStringValue("{\"invalid\":\"json\", \"string\"}")), - DSL.literal(new ExprStringValue("$.a"))) - .valueOf()); + SemanticCheckException invalidJsonError = + assertThrows( + SemanticCheckException.class, + () -> + DSL.jsonExtract( + DSL.literal(new ExprStringValue("{\"invalid\":\"json\", \"string\"}")), + DSL.literal(new ExprStringValue("$.a"))) + .valueOf()); assertEquals( - "JSON string '\"{\"invalid\":\"json\", \"string\"}\"' is not valid. Error details: net.minidev.json.parser.ParseException: Unexpected character (}) at position 26.", - invalidJsonError.getMessage()); + "JSON string '\"{\"invalid\":\"json\", \"string\"}\"' is not valid. Error details:" + + " net.minidev.json.parser.ParseException: Unexpected character (}) at position 26.", + invalidJsonError.getMessage()); } @Test diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index c9260529bb2..b75c241a530 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -69,11 +69,11 @@ ____________ Description >>>>>>>>>>> -Usage: `json_extract(doc, path)` Extracts a json value or scalar from a json document based on the path specified. +Usage: `json_extract(doc, path)` Extracts a JSON value from a json document based on the path specified. Argument type: STRING, STRING -Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY +Return type: STRING/BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY - Returns a JSON array if path points to multiple results (e.g. $.a[*]) or if the path points to an array. - Return null if path is not valid is MISSING or NULL. From 112be65095daef4fd4100a0cb173ea190922f7a6 Mon Sep 17 00:00:00 2001 From: kenrickyap <121634635+kenrickyap@users.noreply.github.com> Date: Thu, 30 Jan 2025 12:34:17 -0500 Subject: [PATCH 67/87] Update core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java Co-authored-by: Taylor Curran Signed-off-by: kenrickyap <121634635+kenrickyap@users.noreply.github.com> --- .../org/opensearch/sql/expression/json/JsonFunctionsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index beac451a6ec..9e96ff55543 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -205,7 +205,7 @@ void json_extract_search_arrays_out_of_bound() { @Test void json_extract_search_arrays() { String jsonArray = "{\"a\":[1,2.3,\"abc\",true,null,{\"c\":{\"d\":1}},[1,2,3]]}"; - List expectedExprValue = + List expectedExprValues = List.of( new ExprIntegerValue(1), new ExprFloatValue(2.3), From 306ac97d2831743bd03467e204ae52d772165173 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 30 Jan 2025 12:51:48 -0500 Subject: [PATCH 68/87] address comments Signed-off-by: Kenrick Yap --- .../sql/expression/json/JsonFunctionsTest.java | 10 ++++++---- docs/user/ppl/functions/json.rst | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 9e96ff55543..ba7d03ff169 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -290,10 +290,12 @@ void json_extract_throws_SemanticCheckException() { DSL.literal(new ExprStringValue("{\"invalid\":\"json\", \"string\"}")), DSL.literal(new ExprStringValue("$.a"))) .valueOf()); - assertEquals( - "JSON string '\"{\"invalid\":\"json\", \"string\"}\"' is not valid. Error details:" - + " net.minidev.json.parser.ParseException: Unexpected character (}) at position 26.", - invalidJsonError.getMessage()); + assertTrue( + invalidJsonError + .getMessage() + .startsWith( + "JSON string '\"{\"invalid\":\"json\", \"string\"}\"' is not valid. Error" + + " details:")); } @Test diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index b75c241a530..3e8c21a9e45 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -75,8 +75,8 @@ Argument type: STRING, STRING Return type: STRING/BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY -- Returns a JSON array if path points to multiple results (e.g. $.a[*]) or if the path points to an array. -- Return null if path is not valid is MISSING or NULL. +- Returns a JSON array if `path` points to multiple results (e.g. $.a[*]) or if the `path` points to an array. +- Return null if `path` is not valid, or if JSON `doc` is MISSING or NULL. - Throws SemanticCheckException if `doc` or `path` is malformed. - Throws ExpressionEvaluationException if `path` is missing. From 75e9cc3426a1ae8e15f74ca85e86871906d51f17 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 30 Jan 2025 14:15:23 -0500 Subject: [PATCH 69/87] fix build error Signed-off-by: Kenrick Yap --- .../opensearch/sql/expression/json/JsonFunctionsTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index ba7d03ff169..1f37e1f0db8 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -219,9 +219,9 @@ void json_extract_search_arrays() { new ExprIntegerValue(1), new ExprIntegerValue(2), new ExprIntegerValue(3)))); // extract specific index from JSON list - for (int i = 0; i < expectedExprValue.size(); i++) { + for (int i = 0; i < expectedExprValues.size(); i++) { String path = String.format("$.a[%d]", i); - execute_extract_json(expectedExprValue.get(i), jsonArray, path); + execute_extract_json(expectedExprValues.get(i), jsonArray, path); } // extract nested object @@ -230,7 +230,7 @@ void json_extract_search_arrays() { execute_extract_json(nestedExpected, jsonArray, "$.a[5].c"); // extract * from JSON list - ExprValue starExpected = new ExprCollectionValue(expectedExprValue); + ExprValue starExpected = new ExprCollectionValue(expectedExprValues); execute_extract_json(starExpected, jsonArray, "$.a[*]"); } From 80f44e28e9f4e395d4c108f760264d91a306fc41 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Fri, 31 Jan 2025 10:19:30 -0500 Subject: [PATCH 70/87] add header Signed-off-by: Kenrick Yap --- core/src/main/java/org/opensearch/sql/utils/JsonUtils.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index b898026a081..433abe76732 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.utils; import static org.opensearch.sql.data.model.ExprValueUtils.*; From 0d1cc28bdbf4dfe466b7679d199f6c27eb3eba7b Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 11 Feb 2025 20:57:02 -0800 Subject: [PATCH 71/87] addressing PR comments Signed-off-by: Kenrick Yap --- .../sql/expression/json/JsonFunctions.java | 4 +- .../org/opensearch/sql/utils/JsonUtils.java | 36 ++++++++++--- .../expression/json/JsonFunctionsTest.java | 50 +++++++++---------- docs/user/ppl/functions/json.rst | 4 +- .../sql/ppl/parser/AstExpressionBuilder.java | 11 ++++ 5 files changed, 67 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 30c5cde162e..666110b3e83 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,6 +5,7 @@ package org.opensearch.sql.expression.json; +import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; @@ -40,6 +41,7 @@ private DefaultFunctionResolver jsonFunction() { private DefaultFunctionResolver jsonExtract() { return define( BuiltinFunctionName.JSON_EXTRACT.getName(), - impl(JsonUtils::extractJson, UNDEFINED, STRING, STRING)); + impl(JsonUtils::extractJsonPaths, UNDEFINED, STRING, ARRAY), + impl(JsonUtils::extractJsonPath, UNDEFINED, STRING, STRING)); } } diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 433abe76732..3be0b900a8e 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -14,6 +14,7 @@ import com.jayway.jsonpath.InvalidPathException; import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.PathNotFoundException; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -88,7 +89,7 @@ public static ExprValue castJson(ExprValue json) { * @param path JSON path (e.g. "$.hello") * @return ExprValue of value at given path of json string. */ - public static ExprValue extractJson(ExprValue json, ExprValue path) { + public static ExprValue extractJsonPath(ExprValue json, ExprValue path) { if (json == LITERAL_NULL || json == LITERAL_MISSING) { return json; } @@ -96,21 +97,40 @@ public static ExprValue extractJson(ExprValue json, ExprValue path) { String jsonString = json.stringValue(); String jsonPath = path.stringValue(); - if (jsonString.isEmpty()) { + return extractJson(jsonString, jsonPath); + } + + public static ExprValue extractJsonPaths(ExprValue json, ExprValue paths) { + List pathList = paths.collectionValue(); + List resultList = new ArrayList<>(); + + for (ExprValue path : pathList) { + resultList.add(extractJsonPath(json, path)); + } + + return new ExprCollectionValue(resultList); + } + + private static ExprValue extractJson(String json, String path) { + if (json.isEmpty() || json.equals("null")) { return LITERAL_NULL; } try { - Object results = JsonPath.parse(jsonString).read(jsonPath); + Object results = JsonPath.parse(json).read(path); return ExprValueUtils.fromObjectValue(results); - } catch (PathNotFoundException e) { + } catch (PathNotFoundException ignored) { return LITERAL_NULL; - } catch (InvalidPathException e) { + } catch (InvalidPathException invalidPathException) { final String errorFormat = "JSON path '%s' is not valid. Error details: %s"; - throw new SemanticCheckException(String.format(errorFormat, path, e.getMessage()), e); - } catch (InvalidJsonException e) { + throw new SemanticCheckException( + String.format(errorFormat, path, invalidPathException.getMessage()), + invalidPathException); + } catch (InvalidJsonException invalidJsonException) { final String errorFormat = "JSON string '%s' is not valid. Error details: %s"; - throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); + throw new SemanticCheckException( + String.format(errorFormat, json, invalidJsonException.getMessage()), + invalidJsonException); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 1f37e1f0db8..1fff41609e5 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -197,11 +197,6 @@ void json_extract_search() { execute_extract_json(expected, "{\"a\":1}", "$.a"); } - @Test - void json_extract_search_arrays_out_of_bound() { - execute_extract_json(LITERAL_NULL, "{\"a\":[1,2,3]}", "$.a[4]"); - } - @Test void json_extract_search_arrays() { String jsonArray = "{\"a\":[1,2.3,\"abc\",true,null,{\"c\":{\"d\":1}},[1,2,3]]}"; @@ -252,17 +247,20 @@ void json_extract_returns_null() { jsonStrings.forEach(str -> execute_extract_json(LITERAL_NULL, str, "$.a.path_not_found_key")); + // null string literal + assertEquals(LITERAL_NULL, DSL.jsonExtract(DSL.literal("null"), DSL.literal("$.a")).valueOf()); + // null json assertEquals( - LITERAL_NULL, - DSL.jsonExtract(DSL.literal(LITERAL_NULL), DSL.literal(new ExprStringValue("$.a"))) - .valueOf()); + LITERAL_NULL, DSL.jsonExtract(DSL.literal(LITERAL_NULL), DSL.literal("$.a")).valueOf()); // missing json assertEquals( LITERAL_MISSING, - DSL.jsonExtract(DSL.literal(LITERAL_MISSING), DSL.literal(new ExprStringValue("$.a"))) - .valueOf()); + DSL.jsonExtract(DSL.literal(LITERAL_MISSING), DSL.literal("$.a")).valueOf()); + + // array out of bounds + execute_extract_json(LITERAL_NULL, "{\"a\":[1,2,3]}", "$.a[4]"); } @Test @@ -271,13 +269,9 @@ void json_extract_throws_SemanticCheckException() { SemanticCheckException invalidPathError = assertThrows( SemanticCheckException.class, - () -> - DSL.jsonExtract( - DSL.literal(new ExprStringValue("{\"a\":1}")), - DSL.literal(new ExprStringValue("$a"))) - .valueOf()); + () -> DSL.jsonExtract(DSL.literal("{\"a\":1}"), DSL.literal("$a")).valueOf()); assertEquals( - "JSON path '\"$a\"' is not valid. Error details: Illegal character at position 1 expected" + "JSON path '$a' is not valid. Error details: Illegal character at position 1 expected" + " '.' or '['", invalidPathError.getMessage()); @@ -287,14 +281,13 @@ void json_extract_throws_SemanticCheckException() { SemanticCheckException.class, () -> DSL.jsonExtract( - DSL.literal(new ExprStringValue("{\"invalid\":\"json\", \"string\"}")), - DSL.literal(new ExprStringValue("$.a"))) + DSL.literal("{\"invalid\":\"json\", \"string\"}"), DSL.literal("$.a")) .valueOf()); assertTrue( invalidJsonError .getMessage() .startsWith( - "JSON string '\"{\"invalid\":\"json\", \"string\"}\"' is not valid. Error" + "JSON string '{\"invalid\":\"json\", \"string\"}' is not valid. Error" + " details:")); } @@ -303,18 +296,21 @@ void json_extract_throws_ExpressionEvaluationException() { // null path assertThrows( ExpressionEvaluationException.class, - () -> - DSL.jsonExtract( - DSL.literal(new ExprStringValue("{\"a\":1}")), DSL.literal(LITERAL_NULL)) - .valueOf()); + () -> DSL.jsonExtract(DSL.literal("{\"a\":1}"), DSL.literal(LITERAL_NULL)).valueOf()); // missing path assertThrows( ExpressionEvaluationException.class, - () -> - DSL.jsonExtract( - DSL.literal(new ExprStringValue("{\"a\":1}")), DSL.literal(LITERAL_MISSING)) - .valueOf()); + () -> DSL.jsonExtract(DSL.literal("{\"a\":1}"), DSL.literal(LITERAL_MISSING)).valueOf()); + } + + @Test + void json_extract_search_list_of_paths() { + final String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + execute_extract_json(LITERAL_NULL, objectJson, "($.foo, $bar2)"); } private static void execute_extract_json(ExprValue expected, String json, String path) { diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 3e8c21a9e45..fe4c92b3998 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -75,14 +75,14 @@ Argument type: STRING, STRING Return type: STRING/BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY -- Returns a JSON array if `path` points to multiple results (e.g. $.a[*]) or if the `path` points to an array. +- Returns an ARRAY if `path` points to multiple results (e.g. $.a[*]) or if the `path` points to an array. - Return null if `path` is not valid, or if JSON `doc` is MISSING or NULL. - Throws SemanticCheckException if `doc` or `path` is malformed. - Throws ExpressionEvaluationException if `path` is missing. Example:: - > source=json_test | where json_valid(json_string) | eval json_extract=json_extract(json_string, '$.b') | fields test_name, json_string, json_extract + os> source=json_test | where json_valid(json_string) | eval json_extract=json_extract(json_string, '$.b') | fields test_name, json_string, json_extract fetched rows / total rows = 6/6 +---------------------+-------------------------------------+-------------------+ | test_name | json_string | json_extract | diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 5a7522683a1..7dc89a7b251 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -390,6 +390,17 @@ public UnresolvedExpression visitSpanClause(SpanClauseContext ctx) { return new Span(visit(ctx.fieldExpression()), visit(ctx.value), SpanUnit.of(unit)); } + @Override + public UnresolvedExpression visitJsonExtract( + OpenSearchPPLParser.JsonExtractFunctionCallContext ctx) {} + + @Override + public UnresolvedExpression visitJsonPathString(OpenSearchPPLParser.JsonPathStringContext ctx) {} + + @Override + public List visitJsonPathList( + OpenSearchPPLParser.JsonPathListContext ctx) {} + private QualifiedName visitIdentifiers(List ctx) { return new QualifiedName( ctx.stream() From b6ae5babcf266dd94906b7b723644019747dfba4 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Wed, 12 Feb 2025 10:02:00 -0800 Subject: [PATCH 72/87] added multi path use case Signed-off-by: Kenrick Yap --- .../sql/expression/json/JsonFunctions.java | 6 ++-- .../org/opensearch/sql/utils/JsonUtils.java | 34 +++++++++---------- .../expression/json/JsonFunctionsTest.java | 10 +++++- docs/user/ppl/functions/json.rst | 15 ++++++-- .../sql/ppl/parser/AstExpressionBuilder.java | 11 ------ 5 files changed, 41 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 666110b3e83..a9aa4998973 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,7 +5,6 @@ package org.opensearch.sql.expression.json; -import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; @@ -41,7 +40,8 @@ private DefaultFunctionResolver jsonFunction() { private DefaultFunctionResolver jsonExtract() { return define( BuiltinFunctionName.JSON_EXTRACT.getName(), - impl(JsonUtils::extractJsonPaths, UNDEFINED, STRING, ARRAY), - impl(JsonUtils::extractJsonPath, UNDEFINED, STRING, STRING)); + impl(JsonUtils::extractJson, UNDEFINED, STRING, STRING), + impl(JsonUtils::extractJson, UNDEFINED, STRING, STRING, STRING), + impl(JsonUtils::extractJson, UNDEFINED, STRING, STRING, STRING, STRING)); } } diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 3be0b900a8e..75e68cb18a5 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -86,32 +86,32 @@ public static ExprValue castJson(ExprValue json) { * Extract value of JSON string at given JSON path. * * @param json JSON string (e.g. "{\"hello\": \"world\"}"). - * @param path JSON path (e.g. "$.hello") + * @param paths list of JSON path (e.g. "$.hello") * @return ExprValue of value at given path of json string. */ - public static ExprValue extractJsonPath(ExprValue json, ExprValue path) { - if (json == LITERAL_NULL || json == LITERAL_MISSING) { - return json; - } - - String jsonString = json.stringValue(); - String jsonPath = path.stringValue(); + public static ExprValue extractJson(ExprValue json, ExprValue... paths) { + List resultList = new ArrayList<>(); - return extractJson(jsonString, jsonPath); - } + for (ExprValue path : paths) { + System.out.println("Processing path: " + path); + if (json == LITERAL_NULL || json == LITERAL_MISSING) { + return json; + } - public static ExprValue extractJsonPaths(ExprValue json, ExprValue paths) { - List pathList = paths.collectionValue(); - List resultList = new ArrayList<>(); + String jsonString = json.stringValue(); + String jsonPath = path.stringValue(); - for (ExprValue path : pathList) { - resultList.add(extractJsonPath(json, path)); + resultList.add(extractJsonPath(jsonString, jsonPath)); } - return new ExprCollectionValue(resultList); + if (resultList.size() == 1) { + return resultList.getFirst(); + } else { + return new ExprCollectionValue(resultList); + } } - private static ExprValue extractJson(String json, String path) { + private static ExprValue extractJsonPath(String json, String path) { if (json.isEmpty() || json.equals("null")) { return LITERAL_NULL; } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 1fff41609e5..f8b9aa53547 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -310,7 +310,15 @@ void json_extract_search_list_of_paths() { "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; - execute_extract_json(LITERAL_NULL, objectJson, "($.foo, $bar2)"); + ExprValue expected = + new ExprCollectionValue( + List.of(new ExprStringValue("foo"), new ExprFloatValue(12.34), LITERAL_NULL)); + Expression pathExpr1 = DSL.literal(ExprValueUtils.stringValue("$.foo")); + Expression pathExpr2 = DSL.literal(ExprValueUtils.stringValue("$.bar2")); + Expression pathExpr3 = DSL.literal(ExprValueUtils.stringValue("$.potato")); + Expression jsonExpr = DSL.literal(ExprValueUtils.stringValue(objectJson)); + ExprValue actual = DSL.jsonExtract(jsonExpr, pathExpr1, pathExpr2, pathExpr3).valueOf(); + assertEquals(expected, actual); } private static void execute_extract_json(ExprValue expected, String json, String path) { diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index fe4c92b3998..0d9e63224a1 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -69,12 +69,13 @@ ____________ Description >>>>>>>>>>> -Usage: `json_extract(doc, path)` Extracts a JSON value from a json document based on the path specified. +Usage: `json_extract(doc, path[, path])` Extracts a JSON value from a json document based on the path specified. Argument type: STRING, STRING Return type: STRING/BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY +- Up to 3 paths can be provided, and results of each `path` with be returned in an ARRAY. - Returns an ARRAY if `path` points to multiple results (e.g. $.a[*]) or if the `path` points to an array. - Return null if `path` is not valid, or if JSON `doc` is MISSING or NULL. - Throws SemanticCheckException if `doc` or `path` is malformed. @@ -95,7 +96,7 @@ Example:: | json empty string | | null | +---------------------+-------------------------------------+-------------------+ - > source=json_test | where test_name="json nested list" | eval json_extract=json_extract('{"a":[{"b":1},{"b":2}]}', '$.b[1].c') + os> source=json_test | where test_name="json nested list" | eval json_extract=json_extract('{"a":[{"b":1},{"b":2}]}', '$.b[1].c') fetched rows / total rows = 1/1 +---------------------+-------------------------------------+--------------+ | test_name | json_string | json_extract | @@ -103,10 +104,18 @@ Example:: | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | 3 | +---------------------+-------------------------------------+--------------+ - > source=json_test | where test_name="json nested list" | eval json_extract=json_extract('{"a":[{"b":1},{"b":2}]}', '$.b[*].c') + os> source=json_test | where test_name="json nested list" | eval json_extract=json_extract('{"a":[{"b":1},{"b":2}]}', '$.b[*].c') fetched rows / total rows = 1/1 +---------------------+-------------------------------------+--------------+ | test_name | json_string | json_extract | |---------------------|-------------------------------------|--------------| | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | [2,3] | +---------------------+-------------------------------------+--------------+ + + os> source=json_test | where test_name="json nested list" | eval json_extract=json_extract('{"a":[{"b":1},{"b":2}]}', '$.a', '$.b[*].c') + fetched rows / total rows = 1/1 + +---------------------+-------------------------------------+--------------+ + | test_name | json_string | json_extract | + |---------------------|-------------------------------------|--------------| + | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | [1,[2,3]] | + +---------------------+-------------------------------------+--------------+ diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 7dc89a7b251..5a7522683a1 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -390,17 +390,6 @@ public UnresolvedExpression visitSpanClause(SpanClauseContext ctx) { return new Span(visit(ctx.fieldExpression()), visit(ctx.value), SpanUnit.of(unit)); } - @Override - public UnresolvedExpression visitJsonExtract( - OpenSearchPPLParser.JsonExtractFunctionCallContext ctx) {} - - @Override - public UnresolvedExpression visitJsonPathString(OpenSearchPPLParser.JsonPathStringContext ctx) {} - - @Override - public List visitJsonPathList( - OpenSearchPPLParser.JsonPathListContext ctx) {} - private QualifiedName visitIdentifiers(List ctx) { return new QualifiedName( ctx.stream() From aa8b81e1c5f5eae7710c7ed0f81f38ba388db222 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Wed, 12 Feb 2025 10:30:27 -0800 Subject: [PATCH 73/87] linting Signed-off-by: Kenrick Yap --- .../org/opensearch/sql/utils/JsonUtils.java | 1 - .../expression/json/JsonFunctionsTest.java | 54 +++++++------------ 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 8f8f569f823..3d3a95f0e7b 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -13,7 +13,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; - import com.jayway.jsonpath.InvalidJsonException; import com.jayway.jsonpath.InvalidPathException; import com.jayway.jsonpath.JsonPath; diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 23b3f5fefc7..59e0104c443 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -22,10 +22,7 @@ import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDoubleValue; -<<<<<<< HEAD import org.opensearch.sql.data.model.ExprFloatValue; -======= ->>>>>>> main import org.opensearch.sql.data.model.ExprIntegerValue; import org.opensearch.sql.data.model.ExprLongValue; import org.opensearch.sql.data.model.ExprNullValue; @@ -44,16 +41,6 @@ public class JsonFunctionsTest { @Test public void json_valid_returns_false() { -<<<<<<< HEAD - assertEquals( - LITERAL_FALSE, - DSL.jsonValid(DSL.literal(ExprValueUtils.stringValue("{\"invalid\":\"json\", \"string\"}"))) - .valueOf()); - assertEquals( - LITERAL_FALSE, DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue("abc")))).valueOf()); - assertEquals(LITERAL_FALSE, DSL.jsonValid(DSL.literal((LITERAL_NULL))).valueOf()); - assertEquals(LITERAL_FALSE, DSL.jsonValid(DSL.literal((LITERAL_MISSING))).valueOf()); -======= List expressions = List.of( DSL.literal(LITERAL_MISSING), // missing returns false @@ -78,7 +65,6 @@ public void json_valid_returns_false() { LITERAL_FALSE, DSL.jsonValid(expr).valueOf(), "Expected FALSE when calling jsonValid with " + expr)); ->>>>>>> main } @Test @@ -118,11 +104,7 @@ public void json_valid_returns_true() { str -> assertEquals( LITERAL_TRUE, -<<<<<<< HEAD - DSL.jsonValid(DSL.literal((ExprValueUtils.stringValue(str)))).valueOf(), -======= DSL.jsonValid(DSL.literal(str)).valueOf(), ->>>>>>> main String.format("String %s must be valid json", str))); } @@ -215,27 +197,27 @@ void json_returnsScalar() { @Test void json_returnsSemanticCheckException() { List expressions = - List.of( - DSL.literal("invalid"), // invalid type - DSL.literal("{{[}}"), // missing bracket - DSL.literal("[}"), // missing bracket - DSL.literal("}"), // missing bracket - DSL.literal("\"missing quote"), // missing quote - DSL.literal("abc"), // not a type - DSL.literal("97ab"), // not a type - DSL.literal("{1, 2, 3, 4}"), // invalid object - DSL.literal("{123: 1, true: 2, null: 3}"), // invalid object - DSL.literal("{\"invalid\":\"json\", \"string\"}"), // invalid object - DSL.literal("[\"a\": 1, \"b\": 2]") // invalid array + List.of( + DSL.literal("invalid"), // invalid type + DSL.literal("{{[}}"), // missing bracket + DSL.literal("[}"), // missing bracket + DSL.literal("}"), // missing bracket + DSL.literal("\"missing quote"), // missing quote + DSL.literal("abc"), // not a type + DSL.literal("97ab"), // not a type + DSL.literal("{1, 2, 3, 4}"), // invalid object + DSL.literal("{123: 1, true: 2, null: 3}"), // invalid object + DSL.literal("{\"invalid\":\"json\", \"string\"}"), // invalid object + DSL.literal("[\"a\": 1, \"b\": 2]") // invalid array ); expressions.stream() - .forEach( - expr -> - assertThrows( - SemanticCheckException.class, - () -> DSL.castJson(expr).valueOf(), - "Expected to throw SemanticCheckException when calling castJson with " + expr)); + .forEach( + expr -> + assertThrows( + SemanticCheckException.class, + () -> DSL.castJson(expr).valueOf(), + "Expected to throw SemanticCheckException when calling castJson with " + expr)); // invalid type assertThrows( From ec6ff5e8901703e9afbc025260c20e94cf029657 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Wed, 12 Feb 2025 11:00:08 -0800 Subject: [PATCH 74/87] fixing doc tests Signed-off-by: Kenrick Yap --- docs/user/ppl/functions/json.rst | 92 ++++++++++++++++---------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 79488d5fd0c..323bdb9c170 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -24,17 +24,17 @@ Example:: os> source=json_test | eval is_valid = json_valid(json_string) | fields test_name, json_string, is_valid fetched rows / total rows = 7/7 - +---------------------+-------------------------------------+----------+ - | test_name | json_string | is_valid | - |---------------------|-------------------------------------|----------| - | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | True | - | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | True | - | json object | {"a":"1","b":"2"} | True | - | json array | [1, 2, 3, 4] | True | - | json scalar string | "abc" | True | - | json empty string | | True | - | json invalid object | {"invalid":"json", "string"} | False | - +---------------------+-------------------------------------+----------+ + +---------------------+--------------------------------------+----------+ + | test_name | json_string | is_valid | + |---------------------+--------------------------------------+----------| + | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | True | + | json nested list | {"a":"1","b":[{"c":"2"}, {"c":"3"}]} | True | + | json object | {"a":"1","b":"2"} | True | + | json array | [1, 2, 3, 4] | True | + | json scalar string | "abc" | True | + | json empty string | | True | + | json invalid object | {"invalid":"json", "string"} | False | + +---------------------+--------------------------------------+----------+ JSON ---------- @@ -52,16 +52,16 @@ Example:: os> source=json_test | where json_valid(json_string) | eval json=json(json_string) | fields test_name, json_string, json fetched rows / total rows = 6/6 - +---------------------+-------------------------------------+-----------------------------+ - | test_name | json_string | json | - |---------------------|-------------------------------------|-----------------------------| - | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | {a:"1",b:{c:"2",d:"3"} | - | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | {a:"1",b:[{c:"2"},{c:"3"}]} | - | json object | {"a":"1","b":"2"} | {a:"1",b:"2"} | - | json array | [1, 2, 3, 4] | [1,2,3,4] | - | json scalar string | "abc" | "abc" | - | json empty string | | null | - +---------------------+-------------------------------------+-----------------------------+ + +--------------------+--------------------------------------+-------------------------------------------+ + | test_name | json_string | json | + |--------------------+--------------------------------------+-------------------------------------------| + | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | {'a': '1', 'b': {'c': '2', 'd': '3'}} | + | json nested list | {"a":"1","b":[{"c":"2"}, {"c":"3"}]} | {'a': '1', 'b': [{'c': '2'}, {'c': '3'}]} | + | json object | {"a":"1","b":"2"} | {'a': '1', 'b': '2'} | + | json array | [1, 2, 3, 4] | [1,2,3,4] | + | json scalar string | "abc" | abc | + | json empty string | | null | + +--------------------+--------------------------------------+-------------------------------------------+ JSON_EXTRACT ____________ @@ -85,37 +85,37 @@ Example:: os> source=json_test | where json_valid(json_string) | eval json_extract=json_extract(json_string, '$.b') | fields test_name, json_string, json_extract fetched rows / total rows = 6/6 - +---------------------+-------------------------------------+-------------------+ - | test_name | json_string | json_extract | - |---------------------|-------------------------------------|-------------------| - | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | {c:"2",d:"3"} | - | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | [{c:"2"},{c:"3"}] | - | json object | {"a":"1","b":"2"} | 2 | - | json array | [1, 2, 3, 4] | null | - | json scalar string | "abc" | null | - | json empty string | | null | - +---------------------+-------------------------------------+-------------------+ + +--------------------+--------------------------------------+-------------------------+ + | test_name | json_string | json_extract | + |--------------------+--------------------------------------+-------------------------| + | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | {'c': '2', 'd': '3'} | + | json nested list | {"a":"1","b":[{"c":"2"}, {"c":"3"}]} | [{'c': '2'},{'c': '3'}] | + | json object | {"a":"1","b":"2"} | 2 | + | json array | [1, 2, 3, 4] | null | + | json scalar string | "abc" | null | + | json empty string | | null | + +--------------------+--------------------------------------+-------------------------+ os> source=json_test | where test_name="json nested list" | eval json_extract=json_extract(json_string, '$.b[1].c') | fields test_name, json_string, json_extract fetched rows / total rows = 1/1 - +---------------------+-------------------------------------+--------------+ - | test_name | json_string | json_extract | - |---------------------|-------------------------------------|--------------| - | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | 3 | - +---------------------+-------------------------------------+--------------+ + +------------------+--------------------------------------+--------------+ + | test_name | json_string | json_extract | + |------------------+--------------------------------------+--------------| + | json nested list | {"a":"1","b":[{"c":"2"}, {"c":"3"}]} | 3 | + +------------------+--------------------------------------+--------------+ os> source=json_test | where test_name="json nested list" | eval json_extract=json_extract(json_string, '$.b[*].c') | fields test_name, json_string, json_extract fetched rows / total rows = 1/1 - +---------------------+-------------------------------------+--------------+ - | test_name | json_string | json_extract | - |---------------------|-------------------------------------|--------------| - | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | [2,3] | - +---------------------+-------------------------------------+--------------+ + +------------------+--------------------------------------+--------------+ + | test_name | json_string | json_extract | + |------------------+--------------------------------------+--------------| + | json nested list | {"a":"1","b":[{"c":"2"}, {"c":"3"}]} | [2,3] | + +------------------+--------------------------------------+--------------+ os> source=json_test | where test_name="json nested list" | eval json_extract=json_extract(json_string, '$.a', '$.b[*].c') | fields test_name, json_string, json_extract fetched rows / total rows = 1/1 - +---------------------+-------------------------------------+--------------+ - | test_name | json_string | json_extract | - |---------------------|-------------------------------------|--------------| - | json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | [1,[2,3]] | - +---------------------+-------------------------------------+--------------+ + +------------------+--------------------------------------+--------------+ + | test_name | json_string | json_extract | + |------------------+--------------------------------------+--------------| + | json nested list | {"a":"1","b":[{"c":"2"}, {"c":"3"}]} | [1,[2,3]] | + +------------------+--------------------------------------+--------------+ From 95e996b42aa4abef8b28f82e91bd059db1eefaca Mon Sep 17 00:00:00 2001 From: kenrickyap <121634635+kenrickyap@users.noreply.github.com> Date: Thu, 13 Feb 2025 14:47:54 -0800 Subject: [PATCH 75/87] Update core/src/main/java/org/opensearch/sql/utils/JsonUtils.java Co-authored-by: Taylor Curran Signed-off-by: kenrickyap <121634635+kenrickyap@users.noreply.github.com> --- core/src/main/java/org/opensearch/sql/utils/JsonUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 3d3a95f0e7b..a372bad8cda 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -97,7 +97,7 @@ public static ExprValue extractJson(ExprValue json, ExprValue... paths) { for (ExprValue path : paths) { System.out.println("Processing path: " + path); - if (json == LITERAL_NULL || json == LITERAL_MISSING) { + if (json.isNull() || json.isMissing()) { return json; } From 527415e9e0d427286f7bb6669c3bef41234488de Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Mon, 20 Jan 2025 14:55:06 -0800 Subject: [PATCH 76/87] Update jsonSet Signed-off-by: Andy Kwok --- .../sql/expression/function/BuiltinFunctionName.java | 1 + .../opensearch/sql/expression/json/JsonFunctions.java | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 4fccb95dedc..bebf57d393b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -208,6 +208,7 @@ public enum BuiltinFunctionName { JSON_VALID(FunctionName.of("json_valid")), JSON(FunctionName.of("json")), JSON_EXTRACT(FunctionName.of("json_extract")), + JSON_SET(FunctionName.of("json_set")), /** GEOSPATIAL Functions. */ GEOIP(FunctionName.of("geoip")), diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index a9aa4998973..ea6144db363 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -24,6 +24,7 @@ public void register(BuiltinFunctionRepository repository) { repository.register(jsonValid()); repository.register(jsonFunction()); repository.register(jsonExtract()); + repository.register(jsonSet()); } private DefaultFunctionResolver jsonValid() { @@ -44,4 +45,12 @@ private DefaultFunctionResolver jsonExtract() { impl(JsonUtils::extractJson, UNDEFINED, STRING, STRING, STRING), impl(JsonUtils::extractJson, UNDEFINED, STRING, STRING, STRING, STRING)); } + + + private DefaultFunctionResolver jsonSet() { + return define( + BuiltinFunctionName.JSON_SET.getName(), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRING)); + } + } From aefa9cb1704860576d16e7ba98fde842fb63be9c Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Mon, 20 Jan 2025 16:49:55 -0800 Subject: [PATCH 77/87] Provide primitive support Signed-off-by: Andy Kwok --- .../sql/expression/json/JsonFunctions.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index ea6144db363..26de9a89fe4 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,8 +5,14 @@ package org.opensearch.sql.expression.json; +import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.SHORT; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; @@ -50,7 +56,14 @@ private DefaultFunctionResolver jsonExtract() { private DefaultFunctionResolver jsonSet() { return define( BuiltinFunctionName.JSON_SET.getName(), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRING)); + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRING), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, LONG), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, SHORT), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, INTEGER), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, DOUBLE), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, BOOLEAN), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, ARRAY), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRUCT)); } } From 3fabe171c31a7c7e160734bc88bfb5625bd04df0 Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Mon, 27 Jan 2025 14:50:10 -0800 Subject: [PATCH 78/87] Manual test Signed-off-by: Andy Kwok --- async-query-core/src/main/antlr/OpenSearchPPLLexer.g4 | 2 +- async-query-core/src/main/antlr/OpenSearchPPLParser.g4 | 2 +- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/async-query-core/src/main/antlr/OpenSearchPPLLexer.g4 b/async-query-core/src/main/antlr/OpenSearchPPLLexer.g4 index cb323f7942c..f1df48bb697 100644 --- a/async-query-core/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/async-query-core/src/main/antlr/OpenSearchPPLLexer.g4 @@ -387,7 +387,7 @@ JSON_VALID: 'JSON_VALID'; //JSON_APPEND: 'JSON_APPEND'; //JSON_DELETE: 'JSON_DELETE'; //JSON_EXTEND: 'JSON_EXTEND'; -//JSON_SET: 'JSON_SET'; +JSON_SET: 'JSON_SET'; //JSON_ARRAY_ALL_MATCH: 'JSON_ARRAY_ALL_MATCH'; //JSON_ARRAY_ANY_MATCH: 'JSON_ARRAY_ANY_MATCH'; //JSON_ARRAY_FILTER: 'JSON_ARRAY_FILTER'; diff --git a/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 b/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 index 133cf64be58..a6ecd35e962 100644 --- a/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 +++ b/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 @@ -875,7 +875,7 @@ jsonFunctionName // | JSON_APPEND // | JSON_DELETE // | JSON_EXTEND -// | JSON_SET + | JSON_SET // | JSON_ARRAY_ALL_MATCH // | JSON_ARRAY_ANY_MATCH // | JSON_ARRAY_FILTER diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index a338e4fa3b7..aa69d566f80 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -335,6 +335,7 @@ CIDRMATCH: 'CIDRMATCH'; JSON_VALID: 'JSON_VALID'; JSON: 'JSON'; JSON_EXTRACT: 'JSON_EXTRACT'; +JSON_SET: 'JSON_SET'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 54a5e7e57cc..5ee2df900b0 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -670,6 +670,7 @@ conditionFunctionName | ISNOTNULL | CIDRMATCH | JSON_VALID + | JSON_SET ; // flow control function return non-boolean value From 46183d90ceaa3cfe9e7fa90836b5973b996dd7e1 Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Mon, 27 Jan 2025 15:19:04 -0800 Subject: [PATCH 79/87] IT tests Signed-off-by: Andy Kwok --- .../opensearch/sql/ppl/JsonFunctionsIT.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 536f1964359..626b4d4a2d9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -217,4 +217,36 @@ public void test_json_extract() throws IOException { rows("json empty string", null), rows("json nested list", new JSONArray(List.of(Map.of("c", "2"), Map.of("c", "3"))))); } + + @Test + public void test_json_set() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | eval updated=json_set(json_string, \"$.c.innerProperty\", \"test_value\" | fields" + + " test_name, updated", + TEST_INDEX_JSON_TEST)); + verifySchema(result, + schema("test_name", null, "string"), + schema("updated", null, "undefined")); + verifyDataRows( + result, + rows( + "json nested object", + new JSONObject(Map.of("a", "1", + "b", Map.of("c", "3"), + "d", List.of(1, 2, 3), + "c", Map.of("innerProperty", "test_value")))), + rows("json object", new JSONObject(Map.of( + "a", "1", + "b", "2", + "c", Map.of("innerProperty", "test_value")))), + rows("json array", null), + rows("json scalar string", null), + rows("json empty string", null), + rows("json invalid string", null), + rows("json null", null)); + } } From afd7a31e49e05fc39a2d1f9bffdbe39823754371 Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Tue, 28 Jan 2025 00:37:11 -0800 Subject: [PATCH 80/87] IT test Signed-off-by: Andy Kwok --- .../org/opensearch/sql/ppl/JsonFunctionsIT.java | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 626b4d4a2d9..b321d64438f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -225,7 +225,7 @@ public void test_json_set() throws IOException { result = executeQuery( String.format( - "source=%s | eval updated=json_set(json_string, \"$.c.innerProperty\", \"test_value\" | fields" + "source=%s | eval updated=json_set(json_string, \\\"$.c.innerProperty\\\", \\\"test_value\\\") | fields" + " test_name, updated", TEST_INDEX_JSON_TEST)); verifySchema(result, @@ -233,20 +233,13 @@ public void test_json_set() throws IOException { schema("updated", null, "undefined")); verifyDataRows( result, - rows( - "json nested object", - new JSONObject(Map.of("a", "1", - "b", Map.of("c", "3"), - "d", List.of(1, 2, 3), - "c", Map.of("innerProperty", "test_value")))), - rows("json object", new JSONObject(Map.of( - "a", "1", - "b", "2", - "c", Map.of("innerProperty", "test_value")))), + rows("json nested object", + "{\"a\":\"1\",\"b\":{\"c\":\"3\"},\"d\":[1,2,3],\"c\":{\"innerProperty\":\"test_value\"}}"), + rows("json object", "{\"a\":\"1\",\"b\":\"2\",\"c\":{\"innerProperty\":\"test_value\"}}"), rows("json array", null), rows("json scalar string", null), rows("json empty string", null), - rows("json invalid string", null), + rows("json invalid object", null), rows("json null", null)); } } From 6daca82389776ff371d6d5aa72c94f89cc39a334 Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Fri, 31 Jan 2025 10:47:44 -0800 Subject: [PATCH 81/87] Code review Signed-off-by: Andy Kwok --- .../src/main/antlr/OpenSearchPPLLexer.g4 | 2 +- .../src/main/antlr/OpenSearchPPLParser.g4 | 2 +- .../sql/expression/json/JsonFunctions.java | 20 +++++------ .../opensearch/sql/ppl/JsonFunctionsIT.java | 33 +++++++++---------- 4 files changed, 27 insertions(+), 30 deletions(-) diff --git a/async-query-core/src/main/antlr/OpenSearchPPLLexer.g4 b/async-query-core/src/main/antlr/OpenSearchPPLLexer.g4 index f1df48bb697..4ed6738b22e 100644 --- a/async-query-core/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/async-query-core/src/main/antlr/OpenSearchPPLLexer.g4 @@ -384,10 +384,10 @@ TO_JSON_STRING: 'TO_JSON_STRING'; JSON_EXTRACT: 'JSON_EXTRACT'; JSON_KEYS: 'JSON_KEYS'; JSON_VALID: 'JSON_VALID'; +JSON_SET: 'JSON_SET'; //JSON_APPEND: 'JSON_APPEND'; //JSON_DELETE: 'JSON_DELETE'; //JSON_EXTEND: 'JSON_EXTEND'; -JSON_SET: 'JSON_SET'; //JSON_ARRAY_ALL_MATCH: 'JSON_ARRAY_ALL_MATCH'; //JSON_ARRAY_ANY_MATCH: 'JSON_ARRAY_ANY_MATCH'; //JSON_ARRAY_FILTER: 'JSON_ARRAY_FILTER'; diff --git a/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 b/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 index a6ecd35e962..c471d9d6d9c 100644 --- a/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 +++ b/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 @@ -872,10 +872,10 @@ jsonFunctionName | JSON_EXTRACT | JSON_KEYS | JSON_VALID + | JSON_SET // | JSON_APPEND // | JSON_DELETE // | JSON_EXTEND - | JSON_SET // | JSON_ARRAY_ALL_MATCH // | JSON_ARRAY_ANY_MATCH // | JSON_ARRAY_FILTER diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 26de9a89fe4..e8c1f6e08e5 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -52,18 +52,16 @@ private DefaultFunctionResolver jsonExtract() { impl(JsonUtils::extractJson, UNDEFINED, STRING, STRING, STRING, STRING)); } - private DefaultFunctionResolver jsonSet() { return define( - BuiltinFunctionName.JSON_SET.getName(), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRING), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, LONG), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, SHORT), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, INTEGER), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, DOUBLE), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, BOOLEAN), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, ARRAY), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRUCT)); + BuiltinFunctionName.JSON_SET.getName(), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRING), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, LONG), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, SHORT), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, INTEGER), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, DOUBLE), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, BOOLEAN), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, ARRAY), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRUCT)); } - } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index b321d64438f..268e2af47dc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -223,23 +223,22 @@ public void test_json_set() throws IOException { JSONObject result; result = - executeQuery( - String.format( - "source=%s | eval updated=json_set(json_string, \\\"$.c.innerProperty\\\", \\\"test_value\\\") | fields" - + " test_name, updated", - TEST_INDEX_JSON_TEST)); - verifySchema(result, - schema("test_name", null, "string"), - schema("updated", null, "undefined")); + executeQuery( + String.format( + "source=%s | eval updated=json_set(json_string, \\\"$.c.innerProperty\\\"," + + " \\\"test_value\\\") | fields test_name, updated", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("updated", null, "undefined")); verifyDataRows( - result, - rows("json nested object", - "{\"a\":\"1\",\"b\":{\"c\":\"3\"},\"d\":[1,2,3],\"c\":{\"innerProperty\":\"test_value\"}}"), - rows("json object", "{\"a\":\"1\",\"b\":\"2\",\"c\":{\"innerProperty\":\"test_value\"}}"), - rows("json array", null), - rows("json scalar string", null), - rows("json empty string", null), - rows("json invalid object", null), - rows("json null", null)); + result, + rows( + "json nested object", + "{\"a\":\"1\",\"b\":{\"c\":\"3\"},\"d\":[1,2,3],\"c\":{\"innerProperty\":\"test_value\"}}"), + rows("json object", "{\"a\":\"1\",\"b\":\"2\",\"c\":{\"innerProperty\":\"test_value\"}}"), + rows("json array", null), + rows("json scalar string", null), + rows("json empty string", null), + rows("json invalid object", null), + rows("json null", null)); } } From b1cb3aae08eb57b67b8d08dcbc0b4329ae116f9a Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Fri, 31 Jan 2025 11:03:09 -0800 Subject: [PATCH 82/87] Fix doc test Signed-off-by: Andy Kwok --- .../org/opensearch/sql/expression/json/JsonFunctions.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index e8c1f6e08e5..3e103c81208 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -7,8 +7,11 @@ import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.BYTE; +import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.IP; import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.SHORT; import static org.opensearch.sql.data.type.ExprCoreType.STRING; @@ -62,6 +65,9 @@ private DefaultFunctionResolver jsonSet() { impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, DOUBLE), impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, BOOLEAN), impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, ARRAY), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRUCT)); + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRUCT), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, BYTE), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, DATE), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, IP)); } } From 741322a372ea14ed73d66da5f77272838b9bc274 Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Tue, 4 Feb 2025 12:33:31 -0800 Subject: [PATCH 83/87] Update supported types Signed-off-by: Andy Kwok --- .../sql/expression/json/JsonFunctions.java | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 3e103c81208..9f92847b234 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -10,12 +10,16 @@ import static org.opensearch.sql.data.type.ExprCoreType.BYTE; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.INTERVAL; import static org.opensearch.sql.data.type.ExprCoreType.IP; import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.SHORT; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; +import static org.opensearch.sql.data.type.ExprCoreType.TIME; +import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; @@ -57,17 +61,31 @@ private DefaultFunctionResolver jsonExtract() { private DefaultFunctionResolver jsonSet() { return define( - BuiltinFunctionName.JSON_SET.getName(), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRING), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, LONG), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, SHORT), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, INTEGER), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, DOUBLE), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, BOOLEAN), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, ARRAY), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRUCT), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, BYTE), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, DATE), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, IP)); + BuiltinFunctionName.JSON_SET.getName(), + + // Numeric types + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, BYTE), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, SHORT), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, INTEGER), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, LONG), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, FLOAT), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, DOUBLE), + + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRING), + + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, BOOLEAN), + + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, DATE), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, TIME), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, TIMESTAMP), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, INTERVAL), + + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, IP), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRUCT), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, ARRAY) + + + + ); } } From 8d26f6b557ff11cfd251eb6dd8f678bf0e1af620 Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Wed, 5 Feb 2025 13:10:13 -0800 Subject: [PATCH 84/87] Code comments Signed-off-by: Andy Kwok --- .../sql/expression/json/JsonFunctions.java | 44 ++++++++----------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 9f92847b234..9b3a712f778 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -61,31 +61,23 @@ private DefaultFunctionResolver jsonExtract() { private DefaultFunctionResolver jsonSet() { return define( - BuiltinFunctionName.JSON_SET.getName(), - - // Numeric types - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, BYTE), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, SHORT), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, INTEGER), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, LONG), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, FLOAT), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, DOUBLE), - - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRING), - - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, BOOLEAN), - - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, DATE), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, TIME), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, TIMESTAMP), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, INTERVAL), - - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, IP), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRUCT), - impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, ARRAY) - - - - ); + BuiltinFunctionName.JSON_SET.getName(), + + // Numeric types + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, BYTE), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, SHORT), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, INTEGER), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, LONG), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, FLOAT), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, DOUBLE), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRING), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, BOOLEAN), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, DATE), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, TIME), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, TIMESTAMP), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, INTERVAL), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, IP), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, STRUCT), + impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, ARRAY)); } } From c5b4da6d0558c4c44a3ff0fcd4708ea28dc6a0bc Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Fri, 7 Feb 2025 12:44:31 -0800 Subject: [PATCH 85/87] Update the test-case Signed-off-by: Andy Kwok --- .../java/org/opensearch/sql/expression/json/JsonFunctions.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 9b3a712f778..8ecd5aa2f46 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -62,8 +62,6 @@ private DefaultFunctionResolver jsonExtract() { private DefaultFunctionResolver jsonSet() { return define( BuiltinFunctionName.JSON_SET.getName(), - - // Numeric types impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, BYTE), impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, SHORT), impl(nullMissingHandling(JsonUtils::setJson), UNDEFINED, STRING, STRING, INTEGER), From b44d1a18e426979d4ce64fa7658426427c095030 Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Thu, 13 Feb 2025 21:30:33 -0800 Subject: [PATCH 86/87] Rebase Signed-off-by: Andy Kwok --- .../org/opensearch/sql/expression/DSL.java | 4 + .../org/opensearch/sql/utils/JsonUtils.java | 60 +++++ .../expression/json/JsonFunctionsTest.java | 251 ++++++++++++++++++ docs/user/ppl/functions/json.rst | 29 ++ 4 files changed, 344 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index f284820639b..b3f0cd522f3 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -691,6 +691,10 @@ public static FunctionExpression jsonExtract(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.JSON_EXTRACT, expressions); } + public static FunctionExpression jsonSet(Expression... expressions) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON_SET, expressions); + } + public static FunctionExpression stringToJson(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); } diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index a372bad8cda..ec18075152a 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -13,9 +13,12 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.InvalidJsonException; import com.jayway.jsonpath.InvalidPathException; import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; import com.jayway.jsonpath.PathNotFoundException; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -167,4 +170,61 @@ private static ExprValue processJsonNode(JsonNode jsonNode) { return LITERAL_NULL; } } + + /** + * Perform an upsert operation against the incoming jsonString value with provided jsonPath and + * value. + * + * @param json jsonObject in String format. + * @param path upsert reference in the form of JsonPath. + * @param valueToInsert value to be added + * @return JsonString after the upsert operation. + */ + public static ExprValue setJson(ExprValue json, ExprValue path, ExprValue valueToInsert) { + + String jsonString = json.stringValue(); + String jsonPathString = path.stringValue(); + Object valueToInsertObj = valueToInsert.value(); + Configuration conf = + Configuration.defaultConfiguration().addOptions(Option.SUPPRESS_EXCEPTIONS); + try { + JsonPath jsonPath = JsonPath.compile(jsonPathString); + DocumentContext docContext = JsonPath.using(conf).parse(jsonString); + Object readResult = docContext.read(jsonPath); + if (readResult == null) { + recursiveCreate(docContext, jsonPathString, valueToInsertObj); + } else { + docContext.set(jsonPathString, valueToInsertObj); + } + return new ExprStringValue(docContext.jsonString()); + + } catch (InvalidPathException e) { + final String errorFormat = "JSON path '%s' is not valid. Error details: %s"; + throw new SemanticCheckException(String.format(errorFormat, path, e.getMessage()), e); + + } catch (InvalidJsonException e) { + final String errorFormat = "JSON object '%s' is not valid. Error details: %s"; + throw new SemanticCheckException(String.format(errorFormat, json, e.getMessage()), e); + } + } + + /** + * Helper method to handle recursive scenario. + * + * @param docContext incoming Json in Java object form. + * @param path path in String to perform insertion. + * @param value value to be inserted with given path. + */ + private static DocumentContext recursiveCreate( + DocumentContext docContext, String path, Object value) { + final int pos = path.lastIndexOf('.'); + final String parent = path.substring(0, pos); + final String current = path.substring(pos + 1); + // Attempt to read the current path as it is, trigger the recursive in case of deep insert. + if (docContext.read(parent) == null) { + recursiveCreate(docContext, parent, new LinkedHashMap<>()); + } + return docContext.put(parent, current, value); + } + } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 59e0104c443..dd331cce4d2 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -21,12 +21,14 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprFloatValue; import org.opensearch.sql.data.model.ExprIntegerValue; import org.opensearch.sql.data.model.ExprLongValue; import org.opensearch.sql.data.model.ExprNullValue; import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; @@ -39,6 +41,10 @@ @ExtendWith(MockitoExtension.class) public class JsonFunctionsTest { + + private static final String JsonSetTestData = + "{\"members\":[{\"name\":\"Alice\",\"age\":19,\"phoneNumbers\":[{\"home\":\"alice_home_landline\"},{\"work\":\"alice_work_phone\"}]},{\"name\":\"Ben\",\"age\":30,\"phoneNumbers\":[{\"home\":\"ben_home_landline\"},{\"work\":\"ben_work_phone\"}]}]}"; + @Test public void json_valid_returns_false() { List expressions = @@ -361,6 +367,251 @@ void json_extract_search_list_of_paths() { assertEquals(expected, actual); } + + @Test + void json_set_InsertByte() { + FunctionExpression functionExpression = + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal((byte) 'a')); + assertEquals("{\"test\":97}", functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_InsertShort() { + FunctionExpression functionExpression = + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(Short.valueOf("123"))); + assertEquals("{\"test\":123}", functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_InsertInt() { + FunctionExpression functionExpression = + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(123)); + assertEquals("{\"test\":123}", functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_InsertLong() { + FunctionExpression functionExpression = + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(123L)); + assertEquals("{\"test\":123}", functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_InsertFloat() { + FunctionExpression functionExpression = + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(123.123F)); + assertEquals("{\"test\":123.123}", functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_InsertDouble() { + FunctionExpression functionExpression = + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(123.123)); + assertEquals("{\"test\":123.123}", functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_InsertString() { + FunctionExpression functionExpression = + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal("test_value")); + assertEquals("{\"test\":\"test_value\"}", functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_InsertBoolean() { + FunctionExpression functionExpression = + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(Boolean.TRUE)); + assertEquals("{\"test\":true}", functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_InsertDate() { + FunctionExpression functionExpression = + DSL.jsonSet( + DSL.literal("{}"), + DSL.literal("$.test"), + DSL.date(DSL.literal(new ExprDateValue("2020-08-17")))); + assertEquals("{\"test\":\"2020-08-17\"}", functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_InsertTime() { + FunctionExpression functionExpression = + DSL.jsonSet( + DSL.literal("{}"), + DSL.literal("$.test"), + DSL.time(DSL.literal(new ExprTimeValue("01:01:01")))); + assertEquals("{\"test\":\"01:01:01\"}", functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_InsertTimestamp() { + FunctionExpression functionExpression = + DSL.jsonSet( + DSL.literal("{}"), + DSL.literal("$.test"), + DSL.timestamp(DSL.literal("2008-05-15 22:00:00"))); + assertEquals("{\"test\":\"2008-05-15 22:00:00\"}", functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_InsertInterval() { + FunctionExpression functionExpression = + DSL.jsonSet( + DSL.literal("{}"), + DSL.literal("$.test"), + DSL.interval(DSL.literal(1), DSL.literal("second"))); + assertEquals("{\"test\":{\"seconds\":1}}", functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_InsertIp() { + FunctionExpression functionExpression = + DSL.jsonSet( + DSL.literal("{}"), DSL.literal("$.test"), DSL.castIp(DSL.literal("192.168.1.1"))); + assertEquals("{\"test\":\"192.168.1.1\"}", functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_InsertMap() { + FunctionExpression functionExpression = + DSL.jsonSet( + DSL.literal("{}"), + DSL.literal("$.test"), + DSL.literal( + ExprTupleValue.fromExprValueMap(Map.of("name", new ExprStringValue("alice"))))); + assertEquals("{\"test\":{\"name\":\"alice\"}}", functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_InsertArray() { + FunctionExpression functionExpression = + DSL.jsonSet( + DSL.literal("{}"), + DSL.literal("$.test"), + DSL.literal( + new ExprCollectionValue( + List.of(new ExprStringValue("Alice"), new ExprStringValue("Ben"))))); + assertEquals("{\"test\":[\"Alice\",\"Ben\"]}", functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_insert_invalid_path() { + FunctionExpression functionExpression = + DSL.jsonSet( + DSL.literal("{\"members\":[{\"name\":\"alice\"}]}"), + DSL.literal("$$$$$$$$$"), + DSL.literal("18")); + assertThrows(SemanticCheckException.class, functionExpression::valueOf); + } + + @Test + void json_set_insert_invalid_jsonObject() { + FunctionExpression functionExpression = + DSL.jsonSet(DSL.literal("[xxxx}}}}}"), DSL.literal("$.test"), DSL.literal("18")); + assertThrows(SemanticCheckException.class, functionExpression::valueOf); + } + + @Test + void json_set_noMatch_property() { + FunctionExpression functionExpression = + DSL.jsonSet( + DSL.literal("{\"members\":[{\"name\":\"alice\"}]}"), + DSL.literal("$.members[0].age.innerAge"), + DSL.literal("18")); + assertEquals( + "{\"members\":[{\"name\":\"alice\",\"age\":{\"innerAge\":\"18\"}}]}", + functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_noMatch_array() { + FunctionExpression functionExpression = + DSL.jsonSet( + DSL.literal("{\"members\":[{\"name\":\"alice\"}]}"), + DSL.literal("$.members[0].age.innerArray"), + DSL.literal( + new ExprCollectionValue( + List.of(new ExprStringValue("18"), new ExprStringValue("20"))))); + assertEquals( + "{\"members\":[{\"name\":\"alice\",\"age\":{\"innerArray\":[\"18\",\"20\"]}}]}", + functionExpression.valueOf().stringValue()); + } + + /** + * In the case of jsonPath hit single match on property, it should overwrite the existing value, + * regardless of the value type (Array, numeric....etc.) + */ + @Test + void json_set_singleMatch_property() { + + FunctionExpression functionExpression = + DSL.jsonSet( + DSL.literal(JsonSetTestData), + DSL.literal("$.members[0].name"), + DSL.literal(new ExprStringValue("Alice Spring"))); + assertEquals( + "{\"members\":[{\"name\":\"Alice" + + " Spring\",\"age\":19,\"phoneNumbers\":[{\"home\":\"alice_home_landline\"},{\"work\":\"alice_work_phone\"}]},{\"name\":\"Ben\",\"age\":30,\"phoneNumbers\":[{\"home\":\"ben_home_landline\"},{\"work\":\"ben_work_phone\"}]}]}", + functionExpression.valueOf().stringValue()); + } + + /** + * In the case of jsonPath hit single match on property, it should overwrite the existing value, + * regardless of the value type (Array, numeric....etc.) + */ + @Test + void json_set_singleMatch_array() { + + FunctionExpression functionExpression = + DSL.jsonSet( + DSL.literal(JsonSetTestData), + DSL.literal("$.members[0].phoneNumbers"), + DSL.literal( + new ExprCollectionValue( + List.of( + ExprTupleValue.fromExprValueMap( + Map.of("home", new ExprStringValue("alice_new_landline"))), + ExprTupleValue.fromExprValueMap( + Map.of("work", new ExprStringValue("alice_new_work_phone"))))))); + assertEquals( + "{\"members\":[{\"name\":\"Alice\",\"age\":19,\"phoneNumbers\":[{\"home\":\"alice_new_landline\"},{\"work\":\"alice_new_work_phone\"}]},{\"name\":\"Ben\",\"age\":30,\"phoneNumbers\":[{\"home\":\"ben_home_landline\"},{\"work\":\"ben_work_phone\"}]}]}", + functionExpression.valueOf().stringValue()); + } + + /** The handling would stay identical regardless of single match || multiple matches. */ + @Test + void json_set_multiMatches_property() { + + FunctionExpression functionExpression = + DSL.jsonSet( + DSL.literal(JsonSetTestData), + DSL.literal("$.members..age"), + DSL.literal(new ExprLongValue(25))); + assertEquals( + "{\"members\":[{\"name\":\"Alice\",\"age\":25,\"phoneNumbers\":[{\"home\":\"alice_home_landline\"},{\"work\":\"alice_work_phone\"}]},{\"name\":\"Ben\",\"age\":25,\"phoneNumbers\":[{\"home\":\"ben_home_landline\"},{\"work\":\"ben_work_phone\"}]}]}", + functionExpression.valueOf().stringValue()); + } + + @Test + void json_set_multiMatches_array() { + + FunctionExpression functionExpression = + DSL.jsonSet( + DSL.literal(JsonSetTestData), + DSL.literal("$.members..phoneNumbers"), + DSL.literal( + new ExprCollectionValue( + List.of( + ExprTupleValue.fromExprValueMap( + Map.of("home", new ExprStringValue("generic_new_landline"))), + ExprTupleValue.fromExprValueMap( + Map.of("work", new ExprStringValue("generic_new_work_phone"))))))); + assertEquals( + "{\"members\":[{\"name\":\"Alice\",\"age\":19,\"phoneNumbers\":[{\"home\":\"generic_new_landline\"},{\"work\":\"generic_new_work_phone\"}]},{\"name\":\"Ben\",\"age\":30,\"phoneNumbers\":[{\"home\":\"generic_new_landline\"},{\"work\":\"generic_new_work_phone\"}]}]}", + functionExpression.valueOf().stringValue()); + } + private static void execute_extract_json(ExprValue expected, String json, String path) { Expression pathExpr = DSL.literal(ExprValueUtils.stringValue(path)); Expression jsonExpr = DSL.literal(ExprValueUtils.stringValue(json)); diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 323bdb9c170..4241ef04601 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -119,3 +119,32 @@ Example:: |------------------+--------------------------------------+--------------| | json nested list | {"a":"1","b":[{"c":"2"}, {"c":"3"}]} | [1,[2,3]] | +------------------+--------------------------------------+--------------+ + + + +JSON_SET +---------- + +Description +>>>>>>>>>>> + +Usage: `json_set(json_string, json_path, value)` Perform value insertion or override with provided Json path and value. Returns the updated JSON object if valid, null otherwise. + +Argument type: STRING, STRING, BYTE/SHORT/INTEGER/LONG/FLOAT/DOUBLE/STRING/BOOLEAN/DATE/TIME/TIMESTAMP/INTERVAL/IP/STRUCT/ARRAY + +Return type: STRING + +Example:: + + os> source=json_test | eval updated=json_set(json_string, "$.c.innerProperty", "test_value") | fields test_name, updated + fetched rows / total rows = 6/6 + +---------------------+--------------------------------------------------------------------+ + | test_name | updated | + |---------------------+--------------------------------------------------------------------| + | json nested object | {"a":"1","b":{"c":"2","d":"3"},"c":{"innerProperty":"test_value"}} | + | json object | {"a":"1","b":"2","c":{"innerProperty":"test_value"}} | + | json array | null | + | json scalar string | null | + | json empty string | null | + | json invalid object | null | + +---------------------+--------------------------------------------------------------------+ \ No newline at end of file From 85c23f93ce1d98e23487981398895f7c7a5ab8da Mon Sep 17 00:00:00 2001 From: Andy Kwok Date: Thu, 13 Feb 2025 21:32:17 -0800 Subject: [PATCH 87/87] Code style Signed-off-by: Andy Kwok --- .../org/opensearch/sql/utils/JsonUtils.java | 6 +- .../expression/json/JsonFunctionsTest.java | 189 +++++++++--------- 2 files changed, 96 insertions(+), 99 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index ec18075152a..89fbaf27070 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -6,7 +6,6 @@ package org.opensearch.sql.utils; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; @@ -186,7 +185,7 @@ public static ExprValue setJson(ExprValue json, ExprValue path, ExprValue valueT String jsonPathString = path.stringValue(); Object valueToInsertObj = valueToInsert.value(); Configuration conf = - Configuration.defaultConfiguration().addOptions(Option.SUPPRESS_EXCEPTIONS); + Configuration.defaultConfiguration().addOptions(Option.SUPPRESS_EXCEPTIONS); try { JsonPath jsonPath = JsonPath.compile(jsonPathString); DocumentContext docContext = JsonPath.using(conf).parse(jsonString); @@ -216,7 +215,7 @@ public static ExprValue setJson(ExprValue json, ExprValue path, ExprValue valueT * @param value value to be inserted with given path. */ private static DocumentContext recursiveCreate( - DocumentContext docContext, String path, Object value) { + DocumentContext docContext, String path, Object value) { final int pos = path.lastIndexOf('.'); final String parent = path.substring(0, pos); final String current = path.substring(pos + 1); @@ -226,5 +225,4 @@ private static DocumentContext recursiveCreate( } return docContext.put(parent, current, value); } - } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index dd331cce4d2..985096b3b96 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -43,7 +43,7 @@ public class JsonFunctionsTest { private static final String JsonSetTestData = - "{\"members\":[{\"name\":\"Alice\",\"age\":19,\"phoneNumbers\":[{\"home\":\"alice_home_landline\"},{\"work\":\"alice_work_phone\"}]},{\"name\":\"Ben\",\"age\":30,\"phoneNumbers\":[{\"home\":\"ben_home_landline\"},{\"work\":\"ben_work_phone\"}]}]}"; + "{\"members\":[{\"name\":\"Alice\",\"age\":19,\"phoneNumbers\":[{\"home\":\"alice_home_landline\"},{\"work\":\"alice_work_phone\"}]},{\"name\":\"Ben\",\"age\":30,\"phoneNumbers\":[{\"home\":\"ben_home_landline\"},{\"work\":\"ben_work_phone\"}]}]}"; @Test public void json_valid_returns_false() { @@ -367,175 +367,174 @@ void json_extract_search_list_of_paths() { assertEquals(expected, actual); } - @Test void json_set_InsertByte() { FunctionExpression functionExpression = - DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal((byte) 'a')); + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal((byte) 'a')); assertEquals("{\"test\":97}", functionExpression.valueOf().stringValue()); } @Test void json_set_InsertShort() { FunctionExpression functionExpression = - DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(Short.valueOf("123"))); + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(Short.valueOf("123"))); assertEquals("{\"test\":123}", functionExpression.valueOf().stringValue()); } @Test void json_set_InsertInt() { FunctionExpression functionExpression = - DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(123)); + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(123)); assertEquals("{\"test\":123}", functionExpression.valueOf().stringValue()); } @Test void json_set_InsertLong() { FunctionExpression functionExpression = - DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(123L)); + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(123L)); assertEquals("{\"test\":123}", functionExpression.valueOf().stringValue()); } @Test void json_set_InsertFloat() { FunctionExpression functionExpression = - DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(123.123F)); + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(123.123F)); assertEquals("{\"test\":123.123}", functionExpression.valueOf().stringValue()); } @Test void json_set_InsertDouble() { FunctionExpression functionExpression = - DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(123.123)); + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(123.123)); assertEquals("{\"test\":123.123}", functionExpression.valueOf().stringValue()); } @Test void json_set_InsertString() { FunctionExpression functionExpression = - DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal("test_value")); + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal("test_value")); assertEquals("{\"test\":\"test_value\"}", functionExpression.valueOf().stringValue()); } @Test void json_set_InsertBoolean() { FunctionExpression functionExpression = - DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(Boolean.TRUE)); + DSL.jsonSet(DSL.literal("{}"), DSL.literal("$.test"), DSL.literal(Boolean.TRUE)); assertEquals("{\"test\":true}", functionExpression.valueOf().stringValue()); } @Test void json_set_InsertDate() { FunctionExpression functionExpression = - DSL.jsonSet( - DSL.literal("{}"), - DSL.literal("$.test"), - DSL.date(DSL.literal(new ExprDateValue("2020-08-17")))); + DSL.jsonSet( + DSL.literal("{}"), + DSL.literal("$.test"), + DSL.date(DSL.literal(new ExprDateValue("2020-08-17")))); assertEquals("{\"test\":\"2020-08-17\"}", functionExpression.valueOf().stringValue()); } @Test void json_set_InsertTime() { FunctionExpression functionExpression = - DSL.jsonSet( - DSL.literal("{}"), - DSL.literal("$.test"), - DSL.time(DSL.literal(new ExprTimeValue("01:01:01")))); + DSL.jsonSet( + DSL.literal("{}"), + DSL.literal("$.test"), + DSL.time(DSL.literal(new ExprTimeValue("01:01:01")))); assertEquals("{\"test\":\"01:01:01\"}", functionExpression.valueOf().stringValue()); } @Test void json_set_InsertTimestamp() { FunctionExpression functionExpression = - DSL.jsonSet( - DSL.literal("{}"), - DSL.literal("$.test"), - DSL.timestamp(DSL.literal("2008-05-15 22:00:00"))); + DSL.jsonSet( + DSL.literal("{}"), + DSL.literal("$.test"), + DSL.timestamp(DSL.literal("2008-05-15 22:00:00"))); assertEquals("{\"test\":\"2008-05-15 22:00:00\"}", functionExpression.valueOf().stringValue()); } @Test void json_set_InsertInterval() { FunctionExpression functionExpression = - DSL.jsonSet( - DSL.literal("{}"), - DSL.literal("$.test"), - DSL.interval(DSL.literal(1), DSL.literal("second"))); + DSL.jsonSet( + DSL.literal("{}"), + DSL.literal("$.test"), + DSL.interval(DSL.literal(1), DSL.literal("second"))); assertEquals("{\"test\":{\"seconds\":1}}", functionExpression.valueOf().stringValue()); } @Test void json_set_InsertIp() { FunctionExpression functionExpression = - DSL.jsonSet( - DSL.literal("{}"), DSL.literal("$.test"), DSL.castIp(DSL.literal("192.168.1.1"))); + DSL.jsonSet( + DSL.literal("{}"), DSL.literal("$.test"), DSL.castIp(DSL.literal("192.168.1.1"))); assertEquals("{\"test\":\"192.168.1.1\"}", functionExpression.valueOf().stringValue()); } @Test void json_set_InsertMap() { FunctionExpression functionExpression = - DSL.jsonSet( - DSL.literal("{}"), - DSL.literal("$.test"), - DSL.literal( - ExprTupleValue.fromExprValueMap(Map.of("name", new ExprStringValue("alice"))))); + DSL.jsonSet( + DSL.literal("{}"), + DSL.literal("$.test"), + DSL.literal( + ExprTupleValue.fromExprValueMap(Map.of("name", new ExprStringValue("alice"))))); assertEquals("{\"test\":{\"name\":\"alice\"}}", functionExpression.valueOf().stringValue()); } @Test void json_set_InsertArray() { FunctionExpression functionExpression = - DSL.jsonSet( - DSL.literal("{}"), - DSL.literal("$.test"), - DSL.literal( - new ExprCollectionValue( - List.of(new ExprStringValue("Alice"), new ExprStringValue("Ben"))))); + DSL.jsonSet( + DSL.literal("{}"), + DSL.literal("$.test"), + DSL.literal( + new ExprCollectionValue( + List.of(new ExprStringValue("Alice"), new ExprStringValue("Ben"))))); assertEquals("{\"test\":[\"Alice\",\"Ben\"]}", functionExpression.valueOf().stringValue()); } @Test void json_set_insert_invalid_path() { FunctionExpression functionExpression = - DSL.jsonSet( - DSL.literal("{\"members\":[{\"name\":\"alice\"}]}"), - DSL.literal("$$$$$$$$$"), - DSL.literal("18")); + DSL.jsonSet( + DSL.literal("{\"members\":[{\"name\":\"alice\"}]}"), + DSL.literal("$$$$$$$$$"), + DSL.literal("18")); assertThrows(SemanticCheckException.class, functionExpression::valueOf); } @Test void json_set_insert_invalid_jsonObject() { FunctionExpression functionExpression = - DSL.jsonSet(DSL.literal("[xxxx}}}}}"), DSL.literal("$.test"), DSL.literal("18")); + DSL.jsonSet(DSL.literal("[xxxx}}}}}"), DSL.literal("$.test"), DSL.literal("18")); assertThrows(SemanticCheckException.class, functionExpression::valueOf); } @Test void json_set_noMatch_property() { FunctionExpression functionExpression = - DSL.jsonSet( - DSL.literal("{\"members\":[{\"name\":\"alice\"}]}"), - DSL.literal("$.members[0].age.innerAge"), - DSL.literal("18")); + DSL.jsonSet( + DSL.literal("{\"members\":[{\"name\":\"alice\"}]}"), + DSL.literal("$.members[0].age.innerAge"), + DSL.literal("18")); assertEquals( - "{\"members\":[{\"name\":\"alice\",\"age\":{\"innerAge\":\"18\"}}]}", - functionExpression.valueOf().stringValue()); + "{\"members\":[{\"name\":\"alice\",\"age\":{\"innerAge\":\"18\"}}]}", + functionExpression.valueOf().stringValue()); } @Test void json_set_noMatch_array() { FunctionExpression functionExpression = - DSL.jsonSet( - DSL.literal("{\"members\":[{\"name\":\"alice\"}]}"), - DSL.literal("$.members[0].age.innerArray"), - DSL.literal( - new ExprCollectionValue( - List.of(new ExprStringValue("18"), new ExprStringValue("20"))))); + DSL.jsonSet( + DSL.literal("{\"members\":[{\"name\":\"alice\"}]}"), + DSL.literal("$.members[0].age.innerArray"), + DSL.literal( + new ExprCollectionValue( + List.of(new ExprStringValue("18"), new ExprStringValue("20"))))); assertEquals( - "{\"members\":[{\"name\":\"alice\",\"age\":{\"innerArray\":[\"18\",\"20\"]}}]}", - functionExpression.valueOf().stringValue()); + "{\"members\":[{\"name\":\"alice\",\"age\":{\"innerArray\":[\"18\",\"20\"]}}]}", + functionExpression.valueOf().stringValue()); } /** @@ -546,14 +545,14 @@ void json_set_noMatch_array() { void json_set_singleMatch_property() { FunctionExpression functionExpression = - DSL.jsonSet( - DSL.literal(JsonSetTestData), - DSL.literal("$.members[0].name"), - DSL.literal(new ExprStringValue("Alice Spring"))); + DSL.jsonSet( + DSL.literal(JsonSetTestData), + DSL.literal("$.members[0].name"), + DSL.literal(new ExprStringValue("Alice Spring"))); assertEquals( - "{\"members\":[{\"name\":\"Alice" - + " Spring\",\"age\":19,\"phoneNumbers\":[{\"home\":\"alice_home_landline\"},{\"work\":\"alice_work_phone\"}]},{\"name\":\"Ben\",\"age\":30,\"phoneNumbers\":[{\"home\":\"ben_home_landline\"},{\"work\":\"ben_work_phone\"}]}]}", - functionExpression.valueOf().stringValue()); + "{\"members\":[{\"name\":\"Alice" + + " Spring\",\"age\":19,\"phoneNumbers\":[{\"home\":\"alice_home_landline\"},{\"work\":\"alice_work_phone\"}]},{\"name\":\"Ben\",\"age\":30,\"phoneNumbers\":[{\"home\":\"ben_home_landline\"},{\"work\":\"ben_work_phone\"}]}]}", + functionExpression.valueOf().stringValue()); } /** @@ -564,19 +563,19 @@ void json_set_singleMatch_property() { void json_set_singleMatch_array() { FunctionExpression functionExpression = - DSL.jsonSet( - DSL.literal(JsonSetTestData), - DSL.literal("$.members[0].phoneNumbers"), - DSL.literal( - new ExprCollectionValue( - List.of( - ExprTupleValue.fromExprValueMap( - Map.of("home", new ExprStringValue("alice_new_landline"))), - ExprTupleValue.fromExprValueMap( - Map.of("work", new ExprStringValue("alice_new_work_phone"))))))); + DSL.jsonSet( + DSL.literal(JsonSetTestData), + DSL.literal("$.members[0].phoneNumbers"), + DSL.literal( + new ExprCollectionValue( + List.of( + ExprTupleValue.fromExprValueMap( + Map.of("home", new ExprStringValue("alice_new_landline"))), + ExprTupleValue.fromExprValueMap( + Map.of("work", new ExprStringValue("alice_new_work_phone"))))))); assertEquals( - "{\"members\":[{\"name\":\"Alice\",\"age\":19,\"phoneNumbers\":[{\"home\":\"alice_new_landline\"},{\"work\":\"alice_new_work_phone\"}]},{\"name\":\"Ben\",\"age\":30,\"phoneNumbers\":[{\"home\":\"ben_home_landline\"},{\"work\":\"ben_work_phone\"}]}]}", - functionExpression.valueOf().stringValue()); + "{\"members\":[{\"name\":\"Alice\",\"age\":19,\"phoneNumbers\":[{\"home\":\"alice_new_landline\"},{\"work\":\"alice_new_work_phone\"}]},{\"name\":\"Ben\",\"age\":30,\"phoneNumbers\":[{\"home\":\"ben_home_landline\"},{\"work\":\"ben_work_phone\"}]}]}", + functionExpression.valueOf().stringValue()); } /** The handling would stay identical regardless of single match || multiple matches. */ @@ -584,32 +583,32 @@ void json_set_singleMatch_array() { void json_set_multiMatches_property() { FunctionExpression functionExpression = - DSL.jsonSet( - DSL.literal(JsonSetTestData), - DSL.literal("$.members..age"), - DSL.literal(new ExprLongValue(25))); + DSL.jsonSet( + DSL.literal(JsonSetTestData), + DSL.literal("$.members..age"), + DSL.literal(new ExprLongValue(25))); assertEquals( - "{\"members\":[{\"name\":\"Alice\",\"age\":25,\"phoneNumbers\":[{\"home\":\"alice_home_landline\"},{\"work\":\"alice_work_phone\"}]},{\"name\":\"Ben\",\"age\":25,\"phoneNumbers\":[{\"home\":\"ben_home_landline\"},{\"work\":\"ben_work_phone\"}]}]}", - functionExpression.valueOf().stringValue()); + "{\"members\":[{\"name\":\"Alice\",\"age\":25,\"phoneNumbers\":[{\"home\":\"alice_home_landline\"},{\"work\":\"alice_work_phone\"}]},{\"name\":\"Ben\",\"age\":25,\"phoneNumbers\":[{\"home\":\"ben_home_landline\"},{\"work\":\"ben_work_phone\"}]}]}", + functionExpression.valueOf().stringValue()); } @Test void json_set_multiMatches_array() { FunctionExpression functionExpression = - DSL.jsonSet( - DSL.literal(JsonSetTestData), - DSL.literal("$.members..phoneNumbers"), - DSL.literal( - new ExprCollectionValue( - List.of( - ExprTupleValue.fromExprValueMap( - Map.of("home", new ExprStringValue("generic_new_landline"))), - ExprTupleValue.fromExprValueMap( - Map.of("work", new ExprStringValue("generic_new_work_phone"))))))); + DSL.jsonSet( + DSL.literal(JsonSetTestData), + DSL.literal("$.members..phoneNumbers"), + DSL.literal( + new ExprCollectionValue( + List.of( + ExprTupleValue.fromExprValueMap( + Map.of("home", new ExprStringValue("generic_new_landline"))), + ExprTupleValue.fromExprValueMap( + Map.of("work", new ExprStringValue("generic_new_work_phone"))))))); assertEquals( - "{\"members\":[{\"name\":\"Alice\",\"age\":19,\"phoneNumbers\":[{\"home\":\"generic_new_landline\"},{\"work\":\"generic_new_work_phone\"}]},{\"name\":\"Ben\",\"age\":30,\"phoneNumbers\":[{\"home\":\"generic_new_landline\"},{\"work\":\"generic_new_work_phone\"}]}]}", - functionExpression.valueOf().stringValue()); + "{\"members\":[{\"name\":\"Alice\",\"age\":19,\"phoneNumbers\":[{\"home\":\"generic_new_landline\"},{\"work\":\"generic_new_work_phone\"}]},{\"name\":\"Ben\",\"age\":30,\"phoneNumbers\":[{\"home\":\"generic_new_landline\"},{\"work\":\"generic_new_work_phone\"}]}]}", + functionExpression.valueOf().stringValue()); } private static void execute_extract_json(ExprValue expected, String json, String path) {